Add gateway diagnostics: iperf3, CLI toolkit, periodic WAN speed test

iperf3 serves throughput tests from every VLAN and the mesh; an hourly librespeed run feeds speedtest_* metrics through node_exporter's textfile collector, and vmalert flags download rates below half the link's own 7-day median so ISP degradation surfaces without per-site thresholds.
2026-07-31 10:44:52 +07:00
parent b11ff75ca6
commit d8d4a686eb
5 changed files with 142 additions and 0 deletions
@@ -0,0 +1,67 @@
+# Periodic WAN speed test so ISP degradation shows up as a trend instead of a
+# complaint. A timer runs librespeed-cli and writes the results as Prometheus
+# metrics into node_exporter's textfile collector — they ride the existing
+# 9100 scrape to VictoriaMetrics, where alerts.nix compares each run against
+# the link's own 7-day median (no per-site threshold to maintain).
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.cnx.router;
+  textfileDir = "/var/lib/speedtest";
+in
+{
+  options.cnx.router.speedtest.interval = lib.mkOption {
+    type = lib.types.str;
+    default = "hourly";
+    description = "systemd OnCalendar spec for the WAN speed test.";
+  };
+
+  config = lib.mkIf cfg.enable {
+    services.prometheus.exporters.node.extraFlags = [
+      "--collector.textfile.directory=${textfileDir}"
+    ];
+
+    systemd.services.speedtest = {
+      description = "WAN speed test to Prometheus textfile metrics";
+      after = [ "network-online.target" ];
+      wants = [ "network-online.target" ];
+      path = [
+        pkgs.librespeed-cli
+        pkgs.jq
+      ];
+      serviceConfig = {
+        Type = "oneshot";
+        StateDirectory = "speedtest";
+        # One test at boot would race PPPoE and log a spurious failure.
+        ExecCondition = "${pkgs.iproute2}/bin/ip link show ppp0";
+      };
+      script = ''
+        tmp="${textfileDir}/.speedtest.prom.tmp"
+        if result=$(librespeed-cli --json); then
+          jq -r '.[0]
+            | "speedtest_download_mbps \(.download)",
+              "speedtest_upload_mbps \(.upload)",
+              "speedtest_ping_ms \(.ping)",
+              "speedtest_jitter_ms \(.jitter)",
+              "speedtest_success 1"' <<<"$result" > "$tmp"
+        else
+          echo "speedtest_success 0" > "$tmp"
+        fi
+        mv "$tmp" "${textfileDir}/speedtest.prom"
+      '';
+    };
+
+    systemd.timers.speedtest = {
+      wantedBy = [ "timers.target" ];
+      timerConfig = {
+        OnCalendar = cfg.speedtest.interval;
+        RandomizedDelaySec = "10m";
+        Persistent = true;
+      };
+    };
+  };
+}