Add VictoriaMetrics + Grafana DNS monitoring over the mesh

control runs VictoriaMetrics (loopback) and Grafana; every machine exports node metrics and the nameservers export Knot stats (mod-stats + knot-exporter). Scraping and the Grafana UI ride the ZeroTier mesh only, scoped by nftables to the mesh /88; the public side stays closed by the Hetzner cloud firewall. The provisioned DNS dashboard includes a per-zone SOA serial table to catch primary/secondary drift. ZeroTier ULAs are centralised in mesh-hosts.nix.
2026-06-17 10:17:27 +07:00
parent 63446173bc
commit 33ac7e106b
8 changed files with 402 additions and 2 deletions
@@ -0,0 +1,93 @@
+# Metric exporters, imported by every machine. Host metrics everywhere; Knot DNS
+# metrics on the nameservers. Everything is reachable only over the ZeroTier mesh
+# (see the firewall rule at the bottom); the public side is already closed by the
+# Hetzner cloud firewall.
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  mesh = import ../mesh-hosts.nix;
+  knotEnabled = config.services.knot.enable;
+  # node_exporter on every host; knot-exporter only where Knot runs.
+  ports = [ 9100 ] ++ lib.optional knotEnabled 9433;
+in
+{
+  # extraInputRules (the mesh-scoped accept below) needs the nftables firewall
+  # backend. allowedTCP/UDPPorts used elsewhere (53, 9993) translate unchanged.
+  networking.nftables.enable = true;
+
+  # Host metrics: CPU, memory, disk, network, systemd unit state.
+  services.prometheus.exporters.node = {
+    enable = true;
+    # Listen on all interfaces (incl. the v6 mesh). We deliberately do NOT bind
+    # to the ZeroTier ULA: the node module renders --web.listen-address without
+    # IPv6 brackets, and binding a single ULA would also race ZeroTier bring-up
+    # at boot. Reachability is constrained by the firewall rule instead.
+    listenAddress = "";
+    port = 9100;
+    enabledCollectors = [ "systemd" ];
+  };
+
+  # Knot DNS metrics. The exporter reads Knot's control socket, so it runs as the
+  # knot user; mod-stats (below) populates the query/response counters it exports.
+  systemd.services.knot-exporter = lib.mkIf knotEnabled {
+    description = "Prometheus exporter for Knot DNS";
+    after = [ "knot.service" ];
+    wants = [ "knot.service" ];
+    wantedBy = [ "multi-user.target" ];
+    serviceConfig = {
+      ExecStart = lib.concatStringsSep " " [
+        "${pkgs.prometheus-knot-exporter}/bin/knot-exporter"
+        "--web-listen-addr ::"
+        "--web-listen-port 9433"
+        "--knot-library-path ${pkgs.knot-dns.out}/lib/libknot.so"
+        "--knot-socket-path /run/knot/knot.sock"
+      ];
+      User = "knot";
+      Group = "knot";
+      Restart = "on-failure";
+      RestartSec = 5;
+      NoNewPrivileges = true;
+      ProtectSystem = "strict";
+      ProtectHome = true;
+      PrivateTmp = true;
+      ProtectKernelTunables = true;
+      ProtectControlGroups = true;
+      RestrictAddressFamilies = [
+        "AF_INET"
+        "AF_INET6"
+        "AF_UNIX"
+      ];
+    };
+  };
+
+  # mod-stats: per-query/zone counters exposed over the control socket and read
+  # by knot-exporter. Loaded as a global module on the default template so it
+  # applies to every zone. (Merges with the zone/acl/policy settings elsewhere.)
+  services.knot.settings = lib.mkIf knotEnabled {
+    "mod-stats" = [
+      {
+        id = "default";
+        "request-protocol" = true;
+        "server-operation" = true;
+        "response-code" = true;
+        "query-type" = true;
+        "reply-nodata" = true;
+      }
+    ];
+    template = [
+      {
+        id = "default";
+        global-module = [ "mod-stats/default" ];
+      }
+    ];
+  };
+
+  # Scrape ports reachable only from the ZeroTier mesh.
+  networking.firewall.extraInputRules = ''
+    ip6 saddr ${mesh.subnet} tcp dport { ${lib.concatMapStringsSep ", " toString ports} } accept
+  '';
+}