Add gateway diagnostics: iperf3, CLI toolkit, periodic WAN speed test
iperf3 serves throughput tests from every VLAN and the mesh; an hourly librespeed run feeds speedtest_* metrics through node_exporter's textfile collector, and vmalert flags download rates below half the link's own 7-day median so ISP degradation surfaces without per-site thresholds.
This commit is contained in:
Berwn
@@ -21,6 +21,8 @@ Naming: `gw-<city>-<n>`, e.g. `gw-cnx-1`.
|
||||
| Bans | CrowdSec + nftables bouncer (sshd log parsing) |
|
||||
| Omada | Optional per site: TP-Link Omada controller as a podman container |
|
||||
| Proxy | Optional: Caddy reverse proxy for internal services under `*.<site><n>.cnx.network` with a real Let's Encrypt wildcard (DNS-01 against ns1) |
|
||||
| Diagnostics | iperf3 server on 5201, reachable from every VLAN and the mesh (`iperf3 -c <router-vlan-addr>`); CLI toolkit: tcpdump, mtr, ethtool, conntrack, kdig, iftop, librespeed-cli |
|
||||
| Speed test | Hourly librespeed run (`speedtest.timer`) → `speedtest_*` metrics via node_exporter; vmalert flags download < 50% of the link's own 7-day median (`WANSpeedDegraded`) |
|
||||
| Management | ZeroTier mesh: SSH, node_exporter, journald upload — like the fleet |
|
||||
| Storage | Single-disk ZFS (zstd, `/var` its own dataset for snapshots) |
|
||||
|
||||
@@ -124,6 +126,10 @@ cnx.router.proxy = {
|
||||
port's `bridgeVLANs` tagging and that the switch trunk carries the VLAN.
|
||||
- **DNS/blocklist issues**: Blocky metrics are in VictoriaMetrics (job
|
||||
`blocky`); `journalctl -u blocky` on the gateway.
|
||||
- **Slow internet reported / `WANSpeedDegraded` firing**: check the
|
||||
`speedtest_download_mbps` / `speedtest_ping_ms` trend in VictoriaMetrics,
|
||||
then `librespeed-cli` and `mtr <isp-gw>` on the gateway to separate ISP
|
||||
degradation from local saturation (`iftop` on ppp0 shows who's eating it).
|
||||
- **Banned yourself**: `cscli decisions list` / `cscli decisions delete --ip <ip>`
|
||||
on the gateway. The ZeroTier mesh `/88` is whitelisted at the parser stage
|
||||
(`cnx/mesh-whitelist`), so admin access over the mesh can never be banned.
|
||||
|
||||
@@ -59,6 +59,31 @@ in
|
||||
}
|
||||
];
|
||||
}
|
||||
{
|
||||
# WAN quality on the site gateways, fed by the hourly speedtest timer
|
||||
# (modules/router/speedtest.nix) via the node_exporter textfile
|
||||
# collector. Degradation is judged against the link's own 7-day median,
|
||||
# so no per-site/per-ISP-plan threshold needs maintaining.
|
||||
name = "wan";
|
||||
rules = [
|
||||
{
|
||||
alert = "WANSpeedDegraded";
|
||||
expr = "speedtest_download_mbps < 0.5 * quantile_over_time(0.5, speedtest_download_mbps[7d])";
|
||||
for = "3h";
|
||||
labels.severity = "warning";
|
||||
annotations.summary = "WAN download on {{ $labels.instance }} is below half its 7-day median";
|
||||
annotations.description = "Sustained for 3h (several test runs) — likely ISP degradation, line fault, or saturation. Compare speedtest_upload_mbps/speedtest_ping_ms and run `librespeed-cli` manually on the gateway.";
|
||||
}
|
||||
{
|
||||
alert = "WANSpeedTestFailing";
|
||||
expr = "speedtest_success == 0";
|
||||
for = "3h";
|
||||
labels.severity = "warning";
|
||||
annotations.summary = "Speed tests on {{ $labels.instance }} have been failing for 3h";
|
||||
annotations.description = "librespeed-cli cannot complete a test while other traffic may still flow; check `journalctl -u speedtest` on the gateway.";
|
||||
}
|
||||
];
|
||||
}
|
||||
{
|
||||
name = "backup";
|
||||
rules = [
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
let
|
||||
@@ -85,6 +86,8 @@ in
|
||||
./crowdsec.nix
|
||||
./omada.nix
|
||||
./proxy.nix
|
||||
./iperf.nix
|
||||
./speedtest.nix
|
||||
];
|
||||
|
||||
options.cnx.router = {
|
||||
@@ -152,6 +155,19 @@ in
|
||||
}
|
||||
];
|
||||
|
||||
# Router diagnostics toolkit: packets (tcpdump), path (mtr), link
|
||||
# negotiation (ethtool), NAT state (conntrack), DNS (kdig), per-flow
|
||||
# bandwidth (iftop), WAN throughput (librespeed-cli; iperf3 covers LAN).
|
||||
environment.systemPackages = with pkgs; [
|
||||
tcpdump
|
||||
mtr
|
||||
ethtool
|
||||
conntrack-tools
|
||||
knot-dns
|
||||
iftop
|
||||
librespeed-cli
|
||||
];
|
||||
|
||||
networking.useNetworkd = true;
|
||||
networking.useDHCP = false;
|
||||
systemd.network.enable = true;
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
# iperf3 server on every gateway, for throughput testing from any LAN segment
|
||||
# (e.g. validating AP/switch links: `iperf3 -c 10.<siteId>.<vlan>.1`) and from
|
||||
# admin machines over the mesh. Never reachable from the WAN (default-deny).
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
...
|
||||
}:
|
||||
let
|
||||
cfg = config.cnx.router;
|
||||
mesh = import ../mesh-hosts.nix { inherit config lib; };
|
||||
vlanIfs = lib.mapAttrsToList (name: _: "vlan-${name}") cfg.vlans;
|
||||
in
|
||||
{
|
||||
config = lib.mkIf cfg.enable {
|
||||
services.iperf3.enable = true;
|
||||
|
||||
networking.firewall.interfaces = lib.genAttrs vlanIfs (_: {
|
||||
allowedTCPPorts = [ 5201 ];
|
||||
allowedUDPPorts = [ 5201 ];
|
||||
});
|
||||
|
||||
networking.firewall.extraInputRules = ''
|
||||
ip6 saddr ${mesh.subnet} tcp dport 5201 accept comment "iperf3 over the mesh"
|
||||
ip6 saddr ${mesh.subnet} udp dport 5201 accept comment "iperf3 over the mesh"
|
||||
'';
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
# Periodic WAN speed test so ISP degradation shows up as a trend instead of a
|
||||
# complaint. A timer runs librespeed-cli and writes the results as Prometheus
|
||||
# metrics into node_exporter's textfile collector — they ride the existing
|
||||
# 9100 scrape to VictoriaMetrics, where alerts.nix compares each run against
|
||||
# the link's own 7-day median (no per-site threshold to maintain).
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
let
|
||||
cfg = config.cnx.router;
|
||||
textfileDir = "/var/lib/speedtest";
|
||||
in
|
||||
{
|
||||
options.cnx.router.speedtest.interval = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "hourly";
|
||||
description = "systemd OnCalendar spec for the WAN speed test.";
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
services.prometheus.exporters.node.extraFlags = [
|
||||
"--collector.textfile.directory=${textfileDir}"
|
||||
];
|
||||
|
||||
systemd.services.speedtest = {
|
||||
description = "WAN speed test to Prometheus textfile metrics";
|
||||
after = [ "network-online.target" ];
|
||||
wants = [ "network-online.target" ];
|
||||
path = [
|
||||
pkgs.librespeed-cli
|
||||
pkgs.jq
|
||||
];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
StateDirectory = "speedtest";
|
||||
# One test at boot would race PPPoE and log a spurious failure.
|
||||
ExecCondition = "${pkgs.iproute2}/bin/ip link show ppp0";
|
||||
};
|
||||
script = ''
|
||||
tmp="${textfileDir}/.speedtest.prom.tmp"
|
||||
if result=$(librespeed-cli --json); then
|
||||
jq -r '.[0]
|
||||
| "speedtest_download_mbps \(.download)",
|
||||
"speedtest_upload_mbps \(.upload)",
|
||||
"speedtest_ping_ms \(.ping)",
|
||||
"speedtest_jitter_ms \(.jitter)",
|
||||
"speedtest_success 1"' <<<"$result" > "$tmp"
|
||||
else
|
||||
echo "speedtest_success 0" > "$tmp"
|
||||
fi
|
||||
mv "$tmp" "${textfileDir}/speedtest.prom"
|
||||
'';
|
||||
};
|
||||
|
||||
systemd.timers.speedtest = {
|
||||
wantedBy = [ "timers.target" ];
|
||||
timerConfig = {
|
||||
OnCalendar = cfg.speedtest.interval;
|
||||
RandomizedDelaySec = "10m";
|
||||
Persistent = true;
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user