Add VictoriaMetrics + Grafana DNS monitoring over the mesh

control runs VictoriaMetrics (loopback) and Grafana; every machine exports node metrics and the nameservers export Knot stats (mod-stats + knot-exporter). Scraping and the Grafana UI ride the ZeroTier mesh only, scoped by nftables to the mesh /88; the public side stays closed by the Hetzner cloud firewall. The provisioned DNS dashboard includes a per-zone SOA serial table to catch primary/secondary drift. ZeroTier ULAs are centralised in mesh-hosts.nix.
2026-06-17 10:17:27 +07:00
parent 63446173bc
commit 33ac7e106b
8 changed files with 402 additions and 2 deletions
@@ -0,0 +1,117 @@
+# Monitoring server, imported by control only: VictoriaMetrics (TSDB + scraper)
+# and Grafana. VictoriaMetrics binds loopback (only Grafana, on the same host,
+# reads it). Grafana is reachable over the ZeroTier mesh, scoped by the firewall
+# rule at the bottom; the Hetzner cloud firewall keeps it off the public net.
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  mesh = import ../mesh-hosts.nix;
+  vmPort = 8428;
+  grafanaPort = 3000;
+  controlV6 = mesh.hosts.control;
+
+  # A single scrape target with a friendly instance label. IPv6 mesh addresses
+  # must be bracketed for Prometheus-style targets.
+  target = name: addr: port: {
+    targets = [ "${addr}:${toString port}" ];
+    labels.instance = name;
+  };
+  v6 = addr: "[${addr}]";
+
+  adminPasswordFile = config.clan.core.vars.generators.grafana-admin.files."password".path;
+in
+{
+  services.victoriametrics = {
+    enable = true;
+    listenAddress = "127.0.0.1:${toString vmPort}";
+    retentionPeriod = "180d";
+    prometheusConfig = {
+      global.scrape_interval = "30s";
+      scrape_configs = [
+        {
+          job_name = "node";
+          static_configs = [
+            # control scrapes its own node_exporter over loopback so host metrics
+            # survive even if the mesh is down; ns1/ns2 are scraped over the mesh.
+            (target "control" "127.0.0.1" 9100)
+            (target "ns1" (v6 mesh.hosts.ns1) 9100)
+            (target "ns2" (v6 mesh.hosts.ns2) 9100)
+          ];
+        }
+        {
+          job_name = "knot";
+          static_configs = [
+            (target "ns1" (v6 mesh.hosts.ns1) 9433)
+            (target "ns2" (v6 mesh.hosts.ns2) 9433)
+          ];
+        }
+      ];
+    };
+  };
+
+  # Admin password generated once and stored as a clan secret. Retrieve with:
+  #   clan vars get control grafana-admin/password
+  clan.core.vars.generators.grafana-admin = {
+    files."password" = {
+      secret = true;
+      owner = "grafana";
+      group = "grafana";
+    };
+    runtimeInputs = [ pkgs.openssl ];
+    script = ''
+      openssl rand -base64 24 | tr -d "\n" > "$out"/password
+    '';
+  };
+
+  services.grafana = {
+    enable = true;
+    settings = {
+      server = {
+        http_addr = "::";
+        http_port = grafanaPort;
+        root_url = "http://${v6 controlV6}:${toString grafanaPort}/";
+      };
+      security = {
+        admin_user = "admin";
+        admin_password = "$__file{${adminPasswordFile}}";
+      };
+      "auth.anonymous".enabled = false;
+      users.allow_sign_up = false;
+    };
+    provision = {
+      enable = true;
+      datasources.settings = {
+        apiVersion = 1;
+        datasources = [
+          {
+            name = "VictoriaMetrics";
+            type = "prometheus";
+            uid = "victoriametrics";
+            access = "proxy";
+            url = "http://127.0.0.1:${toString vmPort}";
+            isDefault = true;
+          }
+        ];
+      };
+      dashboards.settings = {
+        apiVersion = 1;
+        providers = [
+          {
+            name = "cnx";
+            options.path = ./dashboards;
+            options.foldersFromFilesStructure = false;
+          }
+        ];
+      };
+    };
+  };
+
+  # Grafana reachable only from the ZeroTier mesh (admin laptops + servers).
+  networking.firewall.extraInputRules = ''
+    ip6 saddr ${mesh.subnet} tcp dport ${toString grafanaPort} accept
+  '';
+}