Add gateway diagnostics: iperf3, CLI toolkit, periodic WAN speed test

iperf3 serves throughput tests from every VLAN and the mesh; an hourly
librespeed run feeds speedtest_* metrics through node_exporter's textfile
collector, and vmalert flags download rates below half the link's own
7-day median so ISP degradation surfaces without per-site thresholds.

docs/src/gateways.md

@@ -21,6 +21,8 @@ Naming: `gw-<city>-<n>`, e.g. `gw-cnx-1`.
 | Bans         | CrowdSec + nftables bouncer (sshd log parsing)                                                                                                                                                |
 | Omada        | Optional per site: TP-Link Omada controller as a podman container                                                                                                                             |
 | Proxy        | Optional: Caddy reverse proxy for internal services under `*.<site><n>.cnx.network` with a real Let's Encrypt wildcard (DNS-01 against ns1)                                                   |
+| Diagnostics  | iperf3 server on 5201, reachable from every VLAN and the mesh (`iperf3 -c <router-vlan-addr>`); CLI toolkit: tcpdump, mtr, ethtool, conntrack, kdig, iftop, librespeed-cli                    |
+| Speed test   | Hourly librespeed run (`speedtest.timer`) → `speedtest_*` metrics via node_exporter; vmalert flags download < 50% of the link's own 7-day median (`WANSpeedDegraded`)                         |
 | Management   | ZeroTier mesh: SSH, node_exporter, journald upload — like the fleet                                                                                                                           |
 | Storage      | Single-disk ZFS (zstd, `/var` its own dataset for snapshots)                                                                                                                                  |
 
@@ -124,6 +126,10 @@ cnx.router.proxy = {
   port's `bridgeVLANs` tagging and that the switch trunk carries the VLAN.
 - **DNS/blocklist issues**: Blocky metrics are in VictoriaMetrics (job
   `blocky`); `journalctl -u blocky` on the gateway.
+- **Slow internet reported / `WANSpeedDegraded` firing**: check the
+  `speedtest_download_mbps` / `speedtest_ping_ms` trend in VictoriaMetrics,
+  then `librespeed-cli` and `mtr <isp-gw>` on the gateway to separate ISP
+  degradation from local saturation (`iftop` on ppp0 shows who's eating it).
 - **Banned yourself**: `cscli decisions list` / `cscli decisions delete --ip <ip>`
   on the gateway. The ZeroTier mesh `/88` is whitelisted at the parser stage
   (`cnx/mesh-whitelist`), so admin access over the mesh can never be banned.

modules/monitoring/alerts.nix

@@ -59,6 +59,31 @@ in
           }
         ];
       }
+      {
+        # WAN quality on the site gateways, fed by the hourly speedtest timer
+        # (modules/router/speedtest.nix) via the node_exporter textfile
+        # collector. Degradation is judged against the link's own 7-day median,
+        # so no per-site/per-ISP-plan threshold needs maintaining.
+        name = "wan";
+        rules = [
+          {
+            alert = "WANSpeedDegraded";
+            expr = "speedtest_download_mbps < 0.5 * quantile_over_time(0.5, speedtest_download_mbps[7d])";
+            for = "3h";
+            labels.severity = "warning";
+            annotations.summary = "WAN download on {{ $labels.instance }} is below half its 7-day median";
+            annotations.description = "Sustained for 3h (several test runs) — likely ISP degradation, line fault, or saturation. Compare speedtest_upload_mbps/speedtest_ping_ms and run `librespeed-cli` manually on the gateway.";
+          }
+          {
+            alert = "WANSpeedTestFailing";
+            expr = "speedtest_success == 0";
+            for = "3h";
+            labels.severity = "warning";
+            annotations.summary = "Speed tests on {{ $labels.instance }} have been failing for 3h";
+            annotations.description = "librespeed-cli cannot complete a test while other traffic may still flow; check `journalctl -u speedtest` on the gateway.";
+          }
+        ];
+      }
       {
         name = "backup";
         rules = [

modules/router/default.nix

@@ -11,6 +11,7 @@
 {
   config,
   lib,
+  pkgs,
   ...
 }:
 let
@@ -85,6 +86,8 @@ in
     ./crowdsec.nix
     ./omada.nix
     ./proxy.nix
+    ./iperf.nix
+    ./speedtest.nix
   ];
 
   options.cnx.router = {
@@ -152,6 +155,19 @@ in
       }
     ];
 
+    # Router diagnostics toolkit: packets (tcpdump), path (mtr), link
+    # negotiation (ethtool), NAT state (conntrack), DNS (kdig), per-flow
+    # bandwidth (iftop), WAN throughput (librespeed-cli; iperf3 covers LAN).
+    environment.systemPackages = with pkgs; [
+      tcpdump
+      mtr
+      ethtool
+      conntrack-tools
+      knot-dns
+      iftop
+      librespeed-cli
+    ];
+
     networking.useNetworkd = true;
     networking.useDHCP = false;
     systemd.network.enable = true;

modules/router/iperf.nix

@@ -0,0 +1,28 @@
+# iperf3 server on every gateway, for throughput testing from any LAN segment
+# (e.g. validating AP/switch links: `iperf3 -c 10.<siteId>.<vlan>.1`) and from
+# admin machines over the mesh. Never reachable from the WAN (default-deny).
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.cnx.router;
+  mesh = import ../mesh-hosts.nix { inherit config lib; };
+  vlanIfs = lib.mapAttrsToList (name: _: "vlan-${name}") cfg.vlans;
+in
+{
+  config = lib.mkIf cfg.enable {
+    services.iperf3.enable = true;
+
+    networking.firewall.interfaces = lib.genAttrs vlanIfs (_: {
+      allowedTCPPorts = [ 5201 ];
+      allowedUDPPorts = [ 5201 ];
+    });
+
+    networking.firewall.extraInputRules = ''
+      ip6 saddr ${mesh.subnet} tcp dport 5201 accept comment "iperf3 over the mesh"
+      ip6 saddr ${mesh.subnet} udp dport 5201 accept comment "iperf3 over the mesh"
+    '';
+  };
+}

modules/router/speedtest.nix

@@ -0,0 +1,67 @@
+# Periodic WAN speed test so ISP degradation shows up as a trend instead of a
+# complaint. A timer runs librespeed-cli and writes the results as Prometheus
+# metrics into node_exporter's textfile collector — they ride the existing
+# 9100 scrape to VictoriaMetrics, where alerts.nix compares each run against
+# the link's own 7-day median (no per-site threshold to maintain).
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.cnx.router;
+  textfileDir = "/var/lib/speedtest";
+in
+{
+  options.cnx.router.speedtest.interval = lib.mkOption {
+    type = lib.types.str;
+    default = "hourly";
+    description = "systemd OnCalendar spec for the WAN speed test.";
+  };
+
+  config = lib.mkIf cfg.enable {
+    services.prometheus.exporters.node.extraFlags = [
+      "--collector.textfile.directory=${textfileDir}"
+    ];
+
+    systemd.services.speedtest = {
+      description = "WAN speed test to Prometheus textfile metrics";
+      after = [ "network-online.target" ];
+      wants = [ "network-online.target" ];
+      path = [
+        pkgs.librespeed-cli
+        pkgs.jq
+      ];
+      serviceConfig = {
+        Type = "oneshot";
+        StateDirectory = "speedtest";
+        # One test at boot would race PPPoE and log a spurious failure.
+        ExecCondition = "${pkgs.iproute2}/bin/ip link show ppp0";
+      };
+      script = ''
+        tmp="${textfileDir}/.speedtest.prom.tmp"
+        if result=$(librespeed-cli --json); then
+          jq -r '.[0]
+            | "speedtest_download_mbps \(.download)",
+              "speedtest_upload_mbps \(.upload)",
+              "speedtest_ping_ms \(.ping)",
+              "speedtest_jitter_ms \(.jitter)",
+              "speedtest_success 1"' <<<"$result" > "$tmp"
+        else
+          echo "speedtest_success 0" > "$tmp"
+        fi
+        mv "$tmp" "${textfileDir}/speedtest.prom"
+      '';
+    };
+
+    systemd.timers.speedtest = {
+      wantedBy = [ "timers.target" ];
+      timerConfig = {
+        OnCalendar = cfg.speedtest.interval;
+        RandomizedDelaySec = "10m";
+        Persistent = true;
+      };
+    };
+  };
+}