From d8d4a686eb46dcdb46660d8ee156dbcd7813a949 Mon Sep 17 00:00:00 2001 From: Berwn Date: Fri, 31 Jul 2026 10:44:52 +0700 Subject: [PATCH] Add gateway diagnostics: iperf3, CLI toolkit, periodic WAN speed test iperf3 serves throughput tests from every VLAN and the mesh; an hourly librespeed run feeds speedtest_* metrics through node_exporter's textfile collector, and vmalert flags download rates below half the link's own 7-day median so ISP degradation surfaces without per-site thresholds. --- docs/src/gateways.md | 6 ++++ modules/monitoring/alerts.nix | 25 +++++++++++++ modules/router/default.nix | 16 +++++++++ modules/router/iperf.nix | 28 +++++++++++++++ modules/router/speedtest.nix | 67 +++++++++++++++++++++++++++++++++++ 5 files changed, 142 insertions(+) create mode 100644 modules/router/iperf.nix create mode 100644 modules/router/speedtest.nix diff --git a/docs/src/gateways.md b/docs/src/gateways.md index 1efe78d..b9916fb 100644 --- a/docs/src/gateways.md +++ b/docs/src/gateways.md @@ -21,6 +21,8 @@ Naming: `gw--`, e.g. `gw-cnx-1`. | Bans | CrowdSec + nftables bouncer (sshd log parsing) | | Omada | Optional per site: TP-Link Omada controller as a podman container | | Proxy | Optional: Caddy reverse proxy for internal services under `*..cnx.network` with a real Let's Encrypt wildcard (DNS-01 against ns1) | +| Diagnostics | iperf3 server on 5201, reachable from every VLAN and the mesh (`iperf3 -c `); CLI toolkit: tcpdump, mtr, ethtool, conntrack, kdig, iftop, librespeed-cli | +| Speed test | Hourly librespeed run (`speedtest.timer`) → `speedtest_*` metrics via node_exporter; vmalert flags download < 50% of the link's own 7-day median (`WANSpeedDegraded`) | | Management | ZeroTier mesh: SSH, node_exporter, journald upload — like the fleet | | Storage | Single-disk ZFS (zstd, `/var` its own dataset for snapshots) | @@ -124,6 +126,10 @@ cnx.router.proxy = { port's `bridgeVLANs` tagging and that the switch trunk carries the VLAN. - **DNS/blocklist issues**: Blocky metrics are in VictoriaMetrics (job `blocky`); `journalctl -u blocky` on the gateway. +- **Slow internet reported / `WANSpeedDegraded` firing**: check the + `speedtest_download_mbps` / `speedtest_ping_ms` trend in VictoriaMetrics, + then `librespeed-cli` and `mtr ` on the gateway to separate ISP + degradation from local saturation (`iftop` on ppp0 shows who's eating it). - **Banned yourself**: `cscli decisions list` / `cscli decisions delete --ip ` on the gateway. The ZeroTier mesh `/88` is whitelisted at the parser stage (`cnx/mesh-whitelist`), so admin access over the mesh can never be banned. diff --git a/modules/monitoring/alerts.nix b/modules/monitoring/alerts.nix index cc419a7..d85bfbf 100644 --- a/modules/monitoring/alerts.nix +++ b/modules/monitoring/alerts.nix @@ -59,6 +59,31 @@ in } ]; } + { + # WAN quality on the site gateways, fed by the hourly speedtest timer + # (modules/router/speedtest.nix) via the node_exporter textfile + # collector. Degradation is judged against the link's own 7-day median, + # so no per-site/per-ISP-plan threshold needs maintaining. + name = "wan"; + rules = [ + { + alert = "WANSpeedDegraded"; + expr = "speedtest_download_mbps < 0.5 * quantile_over_time(0.5, speedtest_download_mbps[7d])"; + for = "3h"; + labels.severity = "warning"; + annotations.summary = "WAN download on {{ $labels.instance }} is below half its 7-day median"; + annotations.description = "Sustained for 3h (several test runs) — likely ISP degradation, line fault, or saturation. Compare speedtest_upload_mbps/speedtest_ping_ms and run `librespeed-cli` manually on the gateway."; + } + { + alert = "WANSpeedTestFailing"; + expr = "speedtest_success == 0"; + for = "3h"; + labels.severity = "warning"; + annotations.summary = "Speed tests on {{ $labels.instance }} have been failing for 3h"; + annotations.description = "librespeed-cli cannot complete a test while other traffic may still flow; check `journalctl -u speedtest` on the gateway."; + } + ]; + } { name = "backup"; rules = [ diff --git a/modules/router/default.nix b/modules/router/default.nix index 4ddf576..c7564c7 100644 --- a/modules/router/default.nix +++ b/modules/router/default.nix @@ -11,6 +11,7 @@ { config, lib, + pkgs, ... }: let @@ -85,6 +86,8 @@ in ./crowdsec.nix ./omada.nix ./proxy.nix + ./iperf.nix + ./speedtest.nix ]; options.cnx.router = { @@ -152,6 +155,19 @@ in } ]; + # Router diagnostics toolkit: packets (tcpdump), path (mtr), link + # negotiation (ethtool), NAT state (conntrack), DNS (kdig), per-flow + # bandwidth (iftop), WAN throughput (librespeed-cli; iperf3 covers LAN). + environment.systemPackages = with pkgs; [ + tcpdump + mtr + ethtool + conntrack-tools + knot-dns + iftop + librespeed-cli + ]; + networking.useNetworkd = true; networking.useDHCP = false; systemd.network.enable = true; diff --git a/modules/router/iperf.nix b/modules/router/iperf.nix new file mode 100644 index 0000000..6889bd1 --- /dev/null +++ b/modules/router/iperf.nix @@ -0,0 +1,28 @@ +# iperf3 server on every gateway, for throughput testing from any LAN segment +# (e.g. validating AP/switch links: `iperf3 -c 10...1`) and from +# admin machines over the mesh. Never reachable from the WAN (default-deny). +{ + config, + lib, + ... +}: +let + cfg = config.cnx.router; + mesh = import ../mesh-hosts.nix { inherit config lib; }; + vlanIfs = lib.mapAttrsToList (name: _: "vlan-${name}") cfg.vlans; +in +{ + config = lib.mkIf cfg.enable { + services.iperf3.enable = true; + + networking.firewall.interfaces = lib.genAttrs vlanIfs (_: { + allowedTCPPorts = [ 5201 ]; + allowedUDPPorts = [ 5201 ]; + }); + + networking.firewall.extraInputRules = '' + ip6 saddr ${mesh.subnet} tcp dport 5201 accept comment "iperf3 over the mesh" + ip6 saddr ${mesh.subnet} udp dport 5201 accept comment "iperf3 over the mesh" + ''; + }; +} diff --git a/modules/router/speedtest.nix b/modules/router/speedtest.nix new file mode 100644 index 0000000..678ba5a --- /dev/null +++ b/modules/router/speedtest.nix @@ -0,0 +1,67 @@ +# Periodic WAN speed test so ISP degradation shows up as a trend instead of a +# complaint. A timer runs librespeed-cli and writes the results as Prometheus +# metrics into node_exporter's textfile collector — they ride the existing +# 9100 scrape to VictoriaMetrics, where alerts.nix compares each run against +# the link's own 7-day median (no per-site threshold to maintain). +{ + config, + lib, + pkgs, + ... +}: +let + cfg = config.cnx.router; + textfileDir = "/var/lib/speedtest"; +in +{ + options.cnx.router.speedtest.interval = lib.mkOption { + type = lib.types.str; + default = "hourly"; + description = "systemd OnCalendar spec for the WAN speed test."; + }; + + config = lib.mkIf cfg.enable { + services.prometheus.exporters.node.extraFlags = [ + "--collector.textfile.directory=${textfileDir}" + ]; + + systemd.services.speedtest = { + description = "WAN speed test to Prometheus textfile metrics"; + after = [ "network-online.target" ]; + wants = [ "network-online.target" ]; + path = [ + pkgs.librespeed-cli + pkgs.jq + ]; + serviceConfig = { + Type = "oneshot"; + StateDirectory = "speedtest"; + # One test at boot would race PPPoE and log a spurious failure. + ExecCondition = "${pkgs.iproute2}/bin/ip link show ppp0"; + }; + script = '' + tmp="${textfileDir}/.speedtest.prom.tmp" + if result=$(librespeed-cli --json); then + jq -r '.[0] + | "speedtest_download_mbps \(.download)", + "speedtest_upload_mbps \(.upload)", + "speedtest_ping_ms \(.ping)", + "speedtest_jitter_ms \(.jitter)", + "speedtest_success 1"' <<<"$result" > "$tmp" + else + echo "speedtest_success 0" > "$tmp" + fi + mv "$tmp" "${textfileDir}/speedtest.prom" + ''; + }; + + systemd.timers.speedtest = { + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = cfg.speedtest.interval; + RandomizedDelaySec = "10m"; + Persistent = true; + }; + }; + }; +}