Compare commits
20
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a97f3f82e5 | ||
|
|
a46b83cbc7 | ||
|
|
5642a0f6fc | ||
|
|
df389266f6 | ||
|
|
81626a75db | ||
|
|
8171207e9c | ||
|
|
53c03af2a0 | ||
|
|
1a56a2d24b | ||
|
|
e316ec56ce | ||
|
|
559115e52b | ||
|
|
9cfcf91b51 | ||
|
|
570fe64497 | ||
|
|
2ac4215237 | ||
|
|
d51a334ec6 | ||
|
|
b81c0aaa89 | ||
|
|
fcd8e55024 | ||
|
|
19e1acda51 | ||
|
|
f5b6b4b55e | ||
|
|
3646d43ac5 | ||
|
|
d8d4a686eb |
@@ -1,3 +1,4 @@
|
||||
{ inputs, self, ... }:
|
||||
let
|
||||
hosts = import ./modules/hosts.nix;
|
||||
|
||||
@@ -17,83 +18,192 @@ let
|
||||
};
|
||||
in
|
||||
{
|
||||
# Ensure this is unique among all clans you want to use.
|
||||
meta.name = "cnx-network-clan";
|
||||
meta.domain = "cnx-network.internal";
|
||||
clan = {
|
||||
# Ensure this is unique among all clans you want to use.
|
||||
meta.name = "cnx-network-clan";
|
||||
meta.domain = "cnx-network.internal";
|
||||
|
||||
inventory.machines = fleet;
|
||||
specialArgs = { inherit inputs self; };
|
||||
|
||||
inventory.instances = {
|
||||
# Customize nixpkgs
|
||||
# pkgsForSystem =
|
||||
# system:
|
||||
# import nixpkgs {
|
||||
# inherit system;
|
||||
# config = {
|
||||
# allowUnfree = true;
|
||||
# };
|
||||
# overlays = [];
|
||||
# };
|
||||
secrets.age.plugins = [
|
||||
"age-plugin-yubikey"
|
||||
"age-plugin-fido2-hmac"
|
||||
];
|
||||
|
||||
# Admin SSH keys + root password, split per the clan-core migration off
|
||||
# the deprecated `admin` service (sshd handles keys, users the password).
|
||||
sshd = {
|
||||
roles.server.tags.all = { };
|
||||
roles.server.settings.authorizedKeys = {
|
||||
"berwn" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIENAjhGQGraQoAjJzsomKP8GAmQPeGL1rNRNHgRcLqtT";
|
||||
"kurogeek" =
|
||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEcZ/p1Ofa9liwIzPWzNtONhJ7+FUWd2lCz33r81t8+w kurogeek@kurogeek";
|
||||
inventory.machines = fleet;
|
||||
|
||||
inventory.instances = {
|
||||
|
||||
# Admin SSH keys + root password, split per the clan-core migration off
|
||||
# the deprecated `admin` service (sshd handles keys, users the password).
|
||||
sshd = {
|
||||
roles.server.tags.all = { };
|
||||
roles.server.settings.authorizedKeys = {
|
||||
"berwn" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIENAjhGQGraQoAjJzsomKP8GAmQPeGL1rNRNHgRcLqtT";
|
||||
"kurogeek" =
|
||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEcZ/p1Ofa9liwIzPWzNtONhJ7+FUWd2lCz33r81t8+w kurogeek@kurogeek";
|
||||
};
|
||||
};
|
||||
|
||||
gw-router = {
|
||||
module = {
|
||||
name = "gw-router";
|
||||
input = "self";
|
||||
};
|
||||
roles.default.machines."gw-cnx-1" = {
|
||||
settings = {
|
||||
wan = {
|
||||
interface = "enp1s0";
|
||||
vlanId = null;
|
||||
pppInterface = "enp1s0";
|
||||
};
|
||||
vlans = {
|
||||
mgmt = {
|
||||
id = 10;
|
||||
address = "10.1.10.1";
|
||||
prefixLength = 24;
|
||||
subnet = "10.1.10.0/24";
|
||||
dhcp.fixIPs.storinator01 = {
|
||||
hwAddress = "7c:c2:55:e0:d6:40";
|
||||
ipAddress = "10.1.10.53";
|
||||
};
|
||||
};
|
||||
lan = {
|
||||
id = 20;
|
||||
address = "10.1.20.1";
|
||||
prefixLength = 24;
|
||||
subnet = "10.1.20.0/24";
|
||||
};
|
||||
iot = {
|
||||
id = 40;
|
||||
address = "10.1.40.1";
|
||||
prefixLength = 24;
|
||||
subnet = "10.1.40.0/24";
|
||||
};
|
||||
voip = {
|
||||
id = 50;
|
||||
address = "10.1.50.1";
|
||||
prefixLength = 24;
|
||||
subnet = "10.1.50.0/24";
|
||||
};
|
||||
dmz = {
|
||||
id = 60;
|
||||
address = "10.1.60.1";
|
||||
prefixLength = 24;
|
||||
subnet = "10.1.60.0/24";
|
||||
};
|
||||
unit1 = {
|
||||
id = 110;
|
||||
address = "10.1.110.1";
|
||||
prefixLength = 24;
|
||||
subnet = "10.1.110.0/24";
|
||||
};
|
||||
unit2 = {
|
||||
id = 120;
|
||||
address = "10.1.120.1";
|
||||
prefixLength = 24;
|
||||
subnet = "10.1.120.0/24";
|
||||
};
|
||||
unit3 = {
|
||||
id = 130;
|
||||
address = "10.1.130.1";
|
||||
prefixLength = 24;
|
||||
subnet = "10.1.130.0/24";
|
||||
};
|
||||
unit4 = {
|
||||
id = 140;
|
||||
address = "10.1.140.1";
|
||||
prefixLength = 24;
|
||||
subnet = "10.1.140.0/24";
|
||||
};
|
||||
unit5 = {
|
||||
id = 150;
|
||||
address = "10.1.150.1";
|
||||
prefixLength = 24;
|
||||
subnet = "10.1.150.0/24";
|
||||
dhcp.fixIPs.newt = {
|
||||
hwAddress = "7c:d3:0a:21:58:0b";
|
||||
ipAddress = "10.1.150.22";
|
||||
};
|
||||
};
|
||||
};
|
||||
trunkPorts = [ "enp2s0" ];
|
||||
accessPorts = {
|
||||
enp4s0.vlanId = 10;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
root-user = {
|
||||
module = {
|
||||
name = "users";
|
||||
input = "clan-core";
|
||||
};
|
||||
roles.default.tags.all = { };
|
||||
roles.default.settings = {
|
||||
user = "root";
|
||||
prompt = false; # auto-generate, like the old admin service
|
||||
share = false; # per-machine password, not fleet-wide
|
||||
};
|
||||
};
|
||||
|
||||
zerotier = {
|
||||
roles.controller.machines."control" = { };
|
||||
roles.peer.tags.all = { };
|
||||
# External members admitted by ZeroTier node id (stable per device).
|
||||
# Inventory machines are auto-accepted; this is only for peers outside the
|
||||
# clan. Node id comes from `zerotier-cli info` on the joining device.
|
||||
roles.controller.settings.allowedIds = [
|
||||
"8802c8d7e0" # alex-nixos
|
||||
"2bd36db8cc" # kurogeek-thinkpad
|
||||
];
|
||||
};
|
||||
|
||||
tor = {
|
||||
roles.server.tags.nixos = { };
|
||||
};
|
||||
|
||||
# Direct SSH to public IPs — clan's priority-1 connection path, with the
|
||||
# ZeroTier mesh and Tor kept as automatic fallbacks. Raw IPs (from
|
||||
# modules/hosts.nix, not the ns1/ns2 DNS names) so reaching these hosts never
|
||||
# depends on their own DNS being up.
|
||||
internet.roles.default.machines = builtins.mapAttrs (_: h: {
|
||||
settings.host = h.ipv4;
|
||||
}) hosts;
|
||||
|
||||
# Recovery root password for console access when a machine fails to boot.
|
||||
emergency-access = {
|
||||
roles.default.tags.nixos = { };
|
||||
};
|
||||
|
||||
# Encrypted, deduplicating backups. control hosts the repos; ns1 is the
|
||||
# only client, backing up its declared clan.core.state (the Knot DNSSEC
|
||||
# keystore) over the mesh. Repo lives at /var/lib/borgbackup/ns1 on control.
|
||||
# Cross-host so an ns1 loss is recoverable; repokey encryption means control
|
||||
# never holds plaintext. Run `clan vars generate ns1` (YubiKey) before deploy.
|
||||
borgbackup = {
|
||||
roles.server.machines.control = { };
|
||||
roles.client.machines.ns1 = { };
|
||||
};
|
||||
};
|
||||
|
||||
root-user = {
|
||||
module = {
|
||||
name = "users";
|
||||
input = "clan-core";
|
||||
};
|
||||
roles.default.tags.all = { };
|
||||
roles.default.settings = {
|
||||
user = "root";
|
||||
prompt = false; # auto-generate, like the old admin service
|
||||
share = false; # per-machine password, not fleet-wide
|
||||
};
|
||||
};
|
||||
|
||||
zerotier = {
|
||||
roles.controller.machines."control" = { };
|
||||
roles.peer.tags.all = { };
|
||||
# External members admitted by ZeroTier node id (stable per device).
|
||||
# Inventory machines are auto-accepted; this is only for peers outside the
|
||||
# clan. Node id comes from `zerotier-cli info` on the joining device.
|
||||
roles.controller.settings.allowedIds = [
|
||||
"8802c8d7e0" # alex-nixos
|
||||
"2bd36db8cc" # kurogeek-thinkpad
|
||||
];
|
||||
};
|
||||
|
||||
tor = {
|
||||
roles.server.tags.nixos = { };
|
||||
};
|
||||
|
||||
# Direct SSH to public IPs — clan's priority-1 connection path, with the
|
||||
# ZeroTier mesh and Tor kept as automatic fallbacks. Raw IPs (from
|
||||
# modules/hosts.nix, not the ns1/ns2 DNS names) so reaching these hosts never
|
||||
# depends on their own DNS being up.
|
||||
internet.roles.default.machines = builtins.mapAttrs (_: h: {
|
||||
settings.host = h.ipv4;
|
||||
}) hosts;
|
||||
|
||||
# Recovery root password for console access when a machine fails to boot.
|
||||
emergency-access = {
|
||||
roles.default.tags.nixos = { };
|
||||
};
|
||||
|
||||
# Encrypted, deduplicating backups. control hosts the repos; ns1 is the
|
||||
# only client, backing up its declared clan.core.state (the Knot DNSSEC
|
||||
# keystore) over the mesh. Repo lives at /var/lib/borgbackup/ns1 on control.
|
||||
# Cross-host so an ns1 loss is recoverable; repokey encryption means control
|
||||
# never holds plaintext. Run `clan vars generate ns1` (YubiKey) before deploy.
|
||||
borgbackup = {
|
||||
roles.server.machines.control = { };
|
||||
roles.client.machines.ns1 = { };
|
||||
};
|
||||
# Fleet-wide baseline applied to every machine. Secrets minted by
|
||||
# `clan vars generate` are encrypted for the admins group from the very
|
||||
# first run — generating before this took effect is what forced the
|
||||
# re-encryption dance (`clan vars fix`) on gw-cnx-1.
|
||||
machines = builtins.mapAttrs (_: _: {
|
||||
clan.core.sops.defaultGroups = [ "admins" ];
|
||||
}) fleet;
|
||||
};
|
||||
|
||||
# Fleet-wide baseline applied to every machine. Secrets minted by
|
||||
# `clan vars generate` are encrypted for the admins group from the very
|
||||
# first run — generating before this took effect is what forced the
|
||||
# re-encryption dance (`clan vars fix`) on gw-cnx-1.
|
||||
machines = builtins.mapAttrs (_: _: {
|
||||
clan.core.sops.defaultGroups = [ "admins" ];
|
||||
}) fleet;
|
||||
}
|
||||
|
||||
+14
-4
@@ -13,7 +13,7 @@ Naming: `gw-<city>-<n>`, e.g. `gw-cnx-1`.
|
||||
| Function | Implementation |
|
||||
| ------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| WAN | PPPoE (`pppd`), per-site ISP credentials via clan vars prompts; `wan.vlanId` when the ISP tags the session (AIS: 10); `wan.macAddress` to clone the old router's MAC if the ISP has it pinned |
|
||||
| LAN | VLAN-filtering bridge `br0` over the trunk ports (networkd) |
|
||||
| LAN | VLAN-filtering bridge `br0` over the trunk ports (networkd); `accessPorts` pin a port untagged to one VLAN — convention: the last copper port is an untagged `mgmt` recovery port |
|
||||
| Firewall/NAT | nftables: default-deny WAN, no inter-VLAN, MSS clamp, v4 NAT |
|
||||
| DHCP | Kea, one subnet per VLAN |
|
||||
| DNS | Blocky (blocklist resolver), metrics on :4000 scraped by control |
|
||||
@@ -21,6 +21,8 @@ Naming: `gw-<city>-<n>`, e.g. `gw-cnx-1`.
|
||||
| Bans | CrowdSec + nftables bouncer (sshd log parsing) |
|
||||
| Omada | Optional per site: TP-Link Omada controller as a podman container |
|
||||
| Proxy | Optional: Caddy reverse proxy for internal services under `*.<site><n>.cnx.network` with a real Let's Encrypt wildcard (DNS-01 against ns1) |
|
||||
| Diagnostics | iperf3 server on 5201, reachable from every VLAN and the mesh (`iperf3 -c <router-vlan-addr>`); CLI toolkit: tcpdump, mtr, ethtool, conntrack, kdig, iftop, librespeed-cli |
|
||||
| Speed test | Hourly librespeed run (`speedtest.timer`) → `speedtest_*` metrics via node_exporter; vmalert flags download < 50% of the link's own 7-day median (`WANSpeedDegraded`) |
|
||||
| Management | ZeroTier mesh: SSH, node_exporter, journald upload — like the fleet |
|
||||
| Storage | Single-disk ZFS (zstd, `/var` its own dataset for snapshots) |
|
||||
|
||||
@@ -40,9 +42,13 @@ should also shorten `dhcp.leaseTime` (default 86400 s) so the pool recycles.
|
||||
First user: `gw-cnx-2` (site 2) runs the public WiFi — guest VLAN 30 at
|
||||
`10.2.128.0/22`, pool `10.2.128.100 – 10.2.131.250`, `dhcp.leaseTime = 3600`.
|
||||
|
||||
| Site | siteId | mgmt | lan |
|
||||
| ---- | ------ | -------------- | -------------- |
|
||||
| cnx | 1 | `10.1.10.0/24` | `10.1.20.0/24` |
|
||||
| Site | siteId | mgmt | lan | site-specific VLANs |
|
||||
| ---- | ------ | -------------- | -------------- | --------------------------------------------------------------- |
|
||||
| cnx | 1 | `10.1.10.0/24` | `10.1.20.0/24` | iot 40, voip 50, dmz 60, unit1–5 110/120/130/140/150 (all /24s) |
|
||||
|
||||
Static DHCP leases are declared per VLAN via `dhcp.reservations` (attribute
|
||||
name = hostname, plus `hwAddress`/`ipAddress`); park them outside the pool,
|
||||
in the `.2–.99` infra range or `.200–.254`.
|
||||
|
||||
Trust model: mgmt → everything; other VLANs → router DNS/DHCP + internet only
|
||||
(no inter-VLAN); WAN → nothing inbound; mesh → SSH, metrics, Omada UI.
|
||||
@@ -124,6 +130,10 @@ cnx.router.proxy = {
|
||||
port's `bridgeVLANs` tagging and that the switch trunk carries the VLAN.
|
||||
- **DNS/blocklist issues**: Blocky metrics are in VictoriaMetrics (job
|
||||
`blocky`); `journalctl -u blocky` on the gateway.
|
||||
- **Slow internet reported / `WANSpeedDegraded` firing**: check the
|
||||
`speedtest_download_mbps` / `speedtest_ping_ms` trend in VictoriaMetrics,
|
||||
then `librespeed-cli` and `mtr <isp-gw>` on the gateway to separate ISP
|
||||
degradation from local saturation (`iftop` on ppp0 shows who's eating it).
|
||||
- **Banned yourself**: `cscli decisions list` / `cscli decisions delete --ip <ip>`
|
||||
on the gateway. The ZeroTier mesh `/88` is whitelisted at the parser stage
|
||||
(`cnx/mesh-whitelist`), so admin access over the mesh can never be banned.
|
||||
|
||||
@@ -52,6 +52,10 @@ there is picked up):
|
||||
- **CNX Uptime** (`uptime.json`) — per-host up/down status, current uptime,
|
||||
availability over the selected window, and up/down history. Label-driven, so
|
||||
every scraped host appears automatically.
|
||||
- **CNX WAN Speed** (`wan.json`) — per-gateway WAN speed from the hourly
|
||||
librespeed runs: latest download/upload/ping, throughput history with each
|
||||
link's 7-day median as reference, latency/jitter, and test status. See
|
||||
[Site gateways](./gateways.md).
|
||||
- **parsedmarc** — DMARC aggregate/forensic report viewer. Auto-provisioned by
|
||||
the `parsedmarc` module (not from `dashboards/`); reads its own Elasticsearch
|
||||
datasource, not VictoriaMetrics. See [DMARC reports](#dmarc-reports) below.
|
||||
|
||||
Generated
+4
@@ -247,6 +247,10 @@
|
||||
"root": {
|
||||
"inputs": {
|
||||
"clan-core": "clan-core",
|
||||
"flake-parts": [
|
||||
"clan-core",
|
||||
"flake-parts"
|
||||
],
|
||||
"nixos-mailserver": "nixos-mailserver",
|
||||
"nixpkgs": [
|
||||
"clan-core",
|
||||
|
||||
@@ -7,67 +7,43 @@
|
||||
inputs.nixos-mailserver.url = "gitlab:simple-nixos-mailserver/nixos-mailserver/nixos-26.05";
|
||||
inputs.nixos-mailserver.inputs.nixpkgs.follows = "nixpkgs";
|
||||
|
||||
inputs.flake-parts.follows = "clan-core/flake-parts";
|
||||
|
||||
outputs =
|
||||
{
|
||||
inputs@{
|
||||
self,
|
||||
clan-core,
|
||||
nixpkgs,
|
||||
flake-parts,
|
||||
...
|
||||
}@inputs:
|
||||
}:
|
||||
let
|
||||
# Usage see: https://docs.clan.lol
|
||||
clan = clan-core.lib.clan {
|
||||
inherit self;
|
||||
imports = [ ./clan.nix ];
|
||||
specialArgs = { inherit inputs; };
|
||||
|
||||
# Customize nixpkgs
|
||||
# pkgsForSystem =
|
||||
# system:
|
||||
# import nixpkgs {
|
||||
# inherit system;
|
||||
# config = {
|
||||
# allowUnfree = true;
|
||||
# };
|
||||
# overlays = [];
|
||||
# };
|
||||
secrets.age.plugins = [
|
||||
"age-plugin-yubikey"
|
||||
"age-plugin-fido2-hmac"
|
||||
];
|
||||
};
|
||||
|
||||
in
|
||||
flake-parts.lib.mkFlake { inherit inputs; } {
|
||||
systems = [
|
||||
"x86_64-linux"
|
||||
"aarch64-linux"
|
||||
"aarch64-darwin"
|
||||
"x86_64-darwin"
|
||||
];
|
||||
forAllSystems = nixpkgs.lib.genAttrs systems;
|
||||
pkgsFor = system: clan-core.inputs.nixpkgs.legacyPackages.${system};
|
||||
treefmtFor = system: inputs.treefmt-nix.lib.evalModule (pkgsFor system) ./fmt.nix;
|
||||
in
|
||||
{
|
||||
inherit (clan.config) nixosConfigurations nixosModules clanInternals;
|
||||
clan = clan.config;
|
||||
|
||||
# `nix fmt` and the `nix flake check` formatting gate.
|
||||
formatter = forAllSystems (system: (treefmtFor system).config.build.wrapper);
|
||||
checks = forAllSystems (system: {
|
||||
formatting = (treefmtFor system).config.build.check self;
|
||||
});
|
||||
imports = [
|
||||
inputs.clan-core.flakeModules.default
|
||||
|
||||
# Add the Clan cli tool to the dev shell.
|
||||
# Use "nix develop" to enter the dev shell.
|
||||
devShells = forAllSystems (
|
||||
system:
|
||||
./clan.nix
|
||||
./modules/clan/flake-module.nix
|
||||
];
|
||||
|
||||
perSystem =
|
||||
{ system, ... }:
|
||||
let
|
||||
pkgs = clan-core.inputs.nixpkgs.legacyPackages.${system};
|
||||
treefmtEval = inputs.treefmt-nix.lib.evalModule pkgs ./fmt.nix;
|
||||
clanCli = clan-core.packages.${system}.clan-cli;
|
||||
# `clan machines update a b c` normally runs machines in parallel,
|
||||
# which interleaves their output and buries the YubiKey PIN prompts.
|
||||
# This wrapper (first in PATH) runs them one at a time instead; any
|
||||
# flags fall through to the real CLI untouched.
|
||||
clanSequential = (pkgsFor system).writeShellScriptBin "clan" ''
|
||||
clanSequential = pkgs.writeShellScriptBin "clan" ''
|
||||
if [ "$#" -gt 3 ] && [ "$1" = machines ] && [ "$2" = update ]; then
|
||||
shift 2
|
||||
for arg in "$@"; do
|
||||
@@ -84,14 +60,19 @@
|
||||
'';
|
||||
in
|
||||
{
|
||||
default = (pkgsFor system).mkShell {
|
||||
# `nix fmt` and the `nix flake check` formatting gate.
|
||||
formatter = treefmtEval.config.build.wrapper;
|
||||
checks.formatting = treefmtEval.config.build.check self;
|
||||
|
||||
# Add the Clan cli tool to the dev shell.
|
||||
# Use "nix develop" to enter the dev shell.
|
||||
devShells.default = pkgs.mkShell {
|
||||
packages = [
|
||||
clanSequential
|
||||
clanCli
|
||||
(treefmtFor system).config.build.wrapper
|
||||
treefmtEval.config.build.wrapper
|
||||
];
|
||||
};
|
||||
}
|
||||
);
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
@@ -14,6 +14,9 @@
|
||||
},
|
||||
"web01": {
|
||||
"installedAt": 1781983723
|
||||
},
|
||||
"gw-cnx-1": {
|
||||
"installedAt": 1785494267
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,7 @@
|
||||
{ config, lib, ... }:
|
||||
{
|
||||
imports = [
|
||||
../../modules/router
|
||||
# ../../modules/router
|
||||
../../modules/monitoring/exporters.nix
|
||||
(import ../../modules/dns/acme-gw-secret.nix "gw-cnx-1")
|
||||
];
|
||||
@@ -20,34 +20,70 @@
|
||||
builtins.hashString "sha256" config.networking.hostName
|
||||
);
|
||||
|
||||
cnx.router = {
|
||||
enable = true;
|
||||
site = "cnx";
|
||||
siteId = 1;
|
||||
wan.interface = "enp1s0";
|
||||
wan.vlanId = 10; # AIS delivers PPPoE tagged on VLAN 10
|
||||
trunkPorts = [
|
||||
"enp2s0"
|
||||
"enp3s0"
|
||||
"enp4s0"
|
||||
];
|
||||
vlans = {
|
||||
mgmt.id = 10; # 10.1.10.0/24 — APs, switches, Omada, admin
|
||||
lan.id = 20; # 10.1.20.0/24 — trusted clients
|
||||
};
|
||||
# This site runs the Omada controller for its APs/switches.
|
||||
omada.enable = true;
|
||||
# cnx.router = {
|
||||
# enable = true;
|
||||
# site = "cnx";
|
||||
# siteId = 1;
|
||||
# wan.interface = "enp1s0";
|
||||
# wan.vlanId = null; # this ISP runs PPPoE untagged on the port
|
||||
# trunkPorts = [
|
||||
# "enp2s0"
|
||||
# # "enp3s0" # STAGING: serves as the uplink below until cutover
|
||||
# ];
|
||||
# # Dedicated on-site recovery port: untagged mgmt, always available even
|
||||
# # if the switch config is broken.
|
||||
# accessPorts.enp4s0 = "mgmt";
|
||||
# # Replaces the newedge.house OPNsense box; renumbered to the fleet
|
||||
# # convention (10.1.<vlanId>.0/24, router .1, pool .100-.199). The old
|
||||
# # untagged LAN becomes tagged mgmt — infra switch ports get PVID 10.
|
||||
# vlans = {
|
||||
# mgmt = {
|
||||
# id = 10; # 10.1.10.0/24 — servers, APs, switches, Omada, admin
|
||||
# dhcp.reservations.storinator01 = {
|
||||
# hwAddress = "7c:c2:55:e0:d6:40";
|
||||
# ipAddress = "10.1.10.53";
|
||||
# };
|
||||
# };
|
||||
# lan.id = 20; # 10.1.20.0/24 — trusted clients
|
||||
# iot.id = 40; # 10.1.40.0/24
|
||||
# voip.id = 50; # 10.1.50.0/24
|
||||
# dmz.id = 60; # 10.1.60.0/24
|
||||
# unit1.id = 110; # 10.1.110.0/24
|
||||
# unit2.id = 120; # 10.1.120.0/24
|
||||
# unit3.id = 130; # 10.1.130.0/24
|
||||
# unit4.id = 140; # 10.1.140.0/24
|
||||
# unit5 = {
|
||||
# id = 150; # 10.1.150.0/24
|
||||
# dhcp.reservations.newt = {
|
||||
# hwAddress = "7c:d3:0a:21:58:0b";
|
||||
# ipAddress = "10.1.150.22";
|
||||
# };
|
||||
# };
|
||||
# };
|
||||
# # This site runs the Omada controller for its APs/switches.
|
||||
# omada.enable = true;
|
||||
#
|
||||
# # Internal reverse proxy: real wildcard cert *.cnx1.cnx.network; Blocky
|
||||
# # resolves the names to the router's LAN address for mgmt+lan clients.
|
||||
# proxy = {
|
||||
# enable = true;
|
||||
# services.omada = {
|
||||
# # Omada's UI is HTTPS with a self-signed cert on the host network.
|
||||
# backend = "https://127.0.0.1:8043";
|
||||
# insecureSkipVerify = true;
|
||||
# };
|
||||
# };
|
||||
# };
|
||||
|
||||
# Internal reverse proxy: real wildcard cert *.cnx1.cnx.network; Blocky
|
||||
# resolves the names to the router's LAN address for mgmt+lan clients.
|
||||
proxy = {
|
||||
enable = true;
|
||||
services.omada = {
|
||||
# Omada's UI is HTTPS with a self-signed cert on the host network.
|
||||
backend = "https://127.0.0.1:8043";
|
||||
insecureSkipVerify = true;
|
||||
};
|
||||
};
|
||||
# STAGING (remove at cutover, and restore enp3s0 to trunkPorts): DHCP-client
|
||||
# uplink into the existing OPNsense LAN so the box has internet + mesh while
|
||||
# it runs alongside the old router. Default-deny firewall on this interface
|
||||
# (it's in no VLAN zone); PPPoE simply retries until the WAN port is cabled.
|
||||
# Do NOT connect the trunk ports to the production switch while staging —
|
||||
# Kea on tag 10 would fight the OPNsense LAN DHCP in one broadcast domain.
|
||||
systemd.network.networks."05-staging" = {
|
||||
matchConfig.Name = "enp3s0";
|
||||
networkConfig.DHCP = "ipv4";
|
||||
};
|
||||
|
||||
time.timeZone = "Etc/GMT-7"; # UTC+7 (Thailand, fixed offset, no DST)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,26 @@
|
||||
{
|
||||
inputs,
|
||||
...
|
||||
}:
|
||||
{
|
||||
imports =
|
||||
let
|
||||
# Get all subdirectories in the current directory
|
||||
dirContents = builtins.readDir ./.;
|
||||
|
||||
# Filter to include only directories that have a flake-module.nix file
|
||||
# and exclude special directories like 'result'
|
||||
validModuleDirs = builtins.filter (
|
||||
name:
|
||||
name != "result"
|
||||
&& dirContents.${name} == "directory"
|
||||
&& builtins.pathExists (./. + "/${name}/flake-module.nix")
|
||||
) (builtins.attrNames dirContents);
|
||||
|
||||
# Create import paths for each valid directory
|
||||
imports = (map (name: ./. + "/${name}/flake-module.nix") validModuleDirs) ++ [
|
||||
inputs.clan-core.flakeModules.testModule
|
||||
];
|
||||
in
|
||||
imports;
|
||||
}
|
||||
@@ -0,0 +1,293 @@
|
||||
{ ... }:
|
||||
{
|
||||
_class = "clan.service";
|
||||
manifest.name = "gw-router";
|
||||
manifest.description = "A gateway router service to configure most of a router features";
|
||||
manifest.readme = "A gateway router service to configure most of a router features";
|
||||
manifest.categories = [ "System" ];
|
||||
|
||||
roles.default = {
|
||||
description = "Site gateway router role";
|
||||
|
||||
interface =
|
||||
{ lib, config, ... }:
|
||||
let
|
||||
vlanModule =
|
||||
{ ... }:
|
||||
{
|
||||
options = {
|
||||
id = lib.mkOption {
|
||||
type = lib.types.ints.between 1 4094;
|
||||
description = "802.1Q VLAN id.";
|
||||
};
|
||||
address = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
example = "10.0.10.1";
|
||||
description = "Router address on this VLAN.";
|
||||
};
|
||||
prefixLength = lib.mkOption {
|
||||
type = lib.types.ints.between 8 30;
|
||||
default = 24;
|
||||
};
|
||||
subnet = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
example = "10.0.10.0/24";
|
||||
description = "The VLAN's network in CIDR form (must contain `address`).";
|
||||
};
|
||||
dhcp = {
|
||||
enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
};
|
||||
pool = {
|
||||
from = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
example = "10.0.10.100";
|
||||
};
|
||||
to = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
example = "10.0.10.199";
|
||||
};
|
||||
};
|
||||
leaseTime = lib.mkOption {
|
||||
type = lib.types.ints.positive;
|
||||
default = 86400;
|
||||
description = ''
|
||||
Lease validity in seconds. Lower it for high-churn networks,
|
||||
e.g. public-WiFi guest VLANs (3600-7200), so the pool recycles.
|
||||
'';
|
||||
};
|
||||
fixedIPs = lib.mkOption {
|
||||
type = lib.types.attrsOf (
|
||||
lib.types.submodule {
|
||||
options = {
|
||||
hwAddress = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
example = "aa:bb:cc:dd:ee:ff";
|
||||
description = "Client MAC address.";
|
||||
};
|
||||
ipAddress = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "Fixed address handed to this client (inside the VLAN's subnet, outside the pool).";
|
||||
};
|
||||
};
|
||||
}
|
||||
);
|
||||
default = { };
|
||||
description = "Static DHCP leases; the attribute name becomes the client's hostname.";
|
||||
};
|
||||
};
|
||||
allowWAN = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
description = "Whether clients on this VLAN may reach the internet.";
|
||||
};
|
||||
};
|
||||
};
|
||||
in
|
||||
{
|
||||
options = {
|
||||
wan = {
|
||||
interface = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "Physical WAN port the PPPoE session runs on.";
|
||||
};
|
||||
|
||||
vlanId = lib.mkOption {
|
||||
type = lib.types.nullOr (lib.types.ints.between 1 4094);
|
||||
default = null;
|
||||
description = ''
|
||||
802.1Q tag the ISP requires for the PPPoE session (AIS Thailand: 10);
|
||||
null for untagged PPPoE directly on the port. Unrelated to the LAN
|
||||
VLANs — this tag exists only on the WAN port.
|
||||
'';
|
||||
};
|
||||
|
||||
macAddress = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
example = "aa:bb:cc:dd:ee:ff";
|
||||
description = ''
|
||||
Spoofed MAC for the WAN port, e.g. to keep the MAC the ISP has
|
||||
pinned (cloned from the old router). null keeps the hardware MAC.
|
||||
'';
|
||||
};
|
||||
|
||||
pppInterface = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
internal = true;
|
||||
readOnly = true;
|
||||
default = if config.wan.vlanId == null then config.wan.interface else "wan-vlan";
|
||||
description = "Interface pppd dials on (the WAN port or its ISP VLAN).";
|
||||
};
|
||||
};
|
||||
|
||||
vlans = lib.mkOption {
|
||||
type = lib.types.attrsOf (lib.types.submodule vlanModule);
|
||||
description = "VLANs setup for this router";
|
||||
};
|
||||
|
||||
trunkPorts = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
description = "LAN ports carrying all VLANs tagged (incl. any 10G SFP+ ports).";
|
||||
};
|
||||
|
||||
accessPorts = lib.mkOption {
|
||||
type = lib.types.attrsOf (
|
||||
lib.types.submodule {
|
||||
options = {
|
||||
vlanId = lib.mkOption {
|
||||
type = lib.types.int;
|
||||
description = "Untagged traffic in from the device gets tagged VLANs inside the bridge, and VLANs traffic going back out to the device gets untagged, so the device itself never has to know VLANs exist.";
|
||||
};
|
||||
};
|
||||
}
|
||||
);
|
||||
description = ''
|
||||
Ports acting as untagged access ports on a single VLAN (port name ->
|
||||
VLAN id). Frames are untagged on the wire; the bridge tags them with
|
||||
the VLAN's PVID. Use for an always-available on-site mgmt port.
|
||||
'';
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
perInstance =
|
||||
{ settings, ... }:
|
||||
{
|
||||
nixosModule =
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
...
|
||||
}:
|
||||
let
|
||||
vlanIf = name: "vlan=${name}";
|
||||
in
|
||||
{
|
||||
|
||||
networking.useNetworkd = true;
|
||||
networking.useDHCP = false;
|
||||
systemd.network.enable = true;
|
||||
|
||||
systemd.network.netdevs = {
|
||||
"20-br0" = {
|
||||
netdevConfig = {
|
||||
Name = "br0";
|
||||
Kind = "bridge";
|
||||
};
|
||||
bridgeConfig = {
|
||||
VLANFiltering = true;
|
||||
};
|
||||
};
|
||||
}
|
||||
// lib.optionalAttrs (settings.wan.vlanId != null) {
|
||||
"15-wan-lan" = {
|
||||
netdevConfig = {
|
||||
Name = "wan-vlan";
|
||||
Kind = "vlan";
|
||||
};
|
||||
vlanConfig.Id = settings.wan.vlanId;
|
||||
};
|
||||
}
|
||||
// lib.mapAttrs' (
|
||||
name: vlan:
|
||||
lib.nameValuePair "30-${vlanIf name}" {
|
||||
netdevConfig = {
|
||||
Name = vlanIf name;
|
||||
Kind = "vlan";
|
||||
};
|
||||
vlanConfig.Id = vlan.id;
|
||||
}
|
||||
) settings.vlans;
|
||||
|
||||
systemd.network.networks =
|
||||
let
|
||||
allVLANs = lib.mapAttrsToList (_: vlan: { VLAN = vlan.id; }) settings.vlans;
|
||||
in
|
||||
{
|
||||
"10-wan" = {
|
||||
matchConfig.Name = settings.wan.interface;
|
||||
networkConfig.LinkLocalAddressing = "no";
|
||||
vlan = lib.optional (settings.wan.vlanId != null) "wan-wlan";
|
||||
linkConfig = {
|
||||
RequiredForOnline = "carrier";
|
||||
}
|
||||
// lib.optionalAttrs (settings.wan.macAddress != null) {
|
||||
MACAddress = settings.wan.macAddress;
|
||||
};
|
||||
};
|
||||
}
|
||||
// lib.optionalAttrs (settings.wan.vlanId != null) {
|
||||
"15-wan-lan" = {
|
||||
matchConfig.Name = "wan-vlan";
|
||||
networkConfig.LinkLocalAddressing = "no";
|
||||
linkConfig.RequiredForOnline = "no";
|
||||
};
|
||||
}
|
||||
// {
|
||||
"20-br0" = {
|
||||
matchConfig.Name = "br0";
|
||||
networkConfig.LinkLocalAddressing = "no";
|
||||
vlan = lib.mapAttrsToList (name: _: vlanIf name) settings.vlans;
|
||||
bridgeVLANs = allVLANs;
|
||||
linkConfig.RequiredForOnline = "no";
|
||||
};
|
||||
}
|
||||
// lib.listToAttrs (
|
||||
map (port: {
|
||||
name = "25-trunk-${port}";
|
||||
value = {
|
||||
matchConfig.Name = port;
|
||||
networkConfig.Bridge = "br0";
|
||||
bridgeVLANs = allVLANs;
|
||||
linkConfig.RequiredForOnline = "no";
|
||||
};
|
||||
}) settings.trunkPorts
|
||||
)
|
||||
// lib.mapAttrs' (
|
||||
iface: port:
|
||||
lib.nameValuePair "20-access-${iface}" {
|
||||
matchConfig.Name = port;
|
||||
networkConfig.Bridge = "br0";
|
||||
bridgeVLANs = [
|
||||
{
|
||||
VLAN = port.vlanId;
|
||||
PVID = port.vlanId;
|
||||
EgressUntagged = port.vlanId;
|
||||
}
|
||||
];
|
||||
linkConfig.RequiredForOnline = "no";
|
||||
}
|
||||
) settings.accessPorts
|
||||
// lib.mapAttrs' (
|
||||
name: vlan:
|
||||
lib.nameValuePair "40-${vlanIf name}" {
|
||||
matchConfig.Name = vlanIf name;
|
||||
address = [ "${vlan.address}/${toString vlan.prefixLength}" ];
|
||||
networkConfig = {
|
||||
IPv6AcceptRA = false;
|
||||
IPv6SendRA = true;
|
||||
DHCPPrefixDelegation = true;
|
||||
};
|
||||
dhcpPrefixDelegationConfig.SubnetId = "auto";
|
||||
linkConfig.RequiredForOnline = "no";
|
||||
}
|
||||
) settings.vlans;
|
||||
|
||||
environment.systemPackages = with pkgs; [
|
||||
tcpdump
|
||||
mtr
|
||||
ethtool
|
||||
conntrack-tools
|
||||
knot-dns
|
||||
iftop
|
||||
librespeed-cli
|
||||
];
|
||||
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
imports = [ ./pppoe.nix ];
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
{ self, inputs, ... }:
|
||||
let
|
||||
module = ./default.nix;
|
||||
in
|
||||
{
|
||||
flake.clan.modules = {
|
||||
gw-router = module;
|
||||
};
|
||||
# perSystem =
|
||||
# { ... }:
|
||||
# {
|
||||
# clan.nixosTests.service-headplane = {
|
||||
# imports = [ ./tests/vm/default.nix ];
|
||||
# _module.args = { inherit self inputs; };
|
||||
#
|
||||
# clan.modules."@clan/headplane" = module;
|
||||
# };
|
||||
# };
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
{
|
||||
roles.default.perInstance = { settings, ... }: {
|
||||
nixosModule =
|
||||
{ config, ... }:
|
||||
let
|
||||
creds = config.clan.core.vars.generators.gw-router;
|
||||
in
|
||||
{
|
||||
clan.core.vars.generators.gw-router = {
|
||||
prompts.pppoe-username = {
|
||||
description = "PPPoE username (from the ISP)";
|
||||
type = "hidden";
|
||||
};
|
||||
prompts.pppoe-password = {
|
||||
description = "PPPoE password (from the ISP)";
|
||||
type = "hidden";
|
||||
};
|
||||
files."pppoe-username".secret = true;
|
||||
files."pppoe-password".secret = true;
|
||||
script = ''
|
||||
user="$(cat "$prompts"/pppoe-username)"
|
||||
pass="$(cat "$prompts"/pppoe-password)"
|
||||
printf 'user "%s"\n' "$user" > "$out"/pppoe-username
|
||||
printf '"%s" * "%s"\n' "$user" "$pass" > "$out"/pppoe-password
|
||||
'';
|
||||
};
|
||||
|
||||
services.pppd = {
|
||||
enable = true;
|
||||
peers.wan = {
|
||||
autostart = true;
|
||||
config = ''
|
||||
plugin pppoe.so ${settings.wan.pppInterface}
|
||||
ifname ppp0
|
||||
file ${creds.files."pppoe-username".path}
|
||||
noipdefault
|
||||
defaultroute
|
||||
noauth
|
||||
hide-password
|
||||
persist
|
||||
maxfail 0
|
||||
holdoff 5
|
||||
lcp-echo-interval 15
|
||||
lcp-echo-failure 3
|
||||
+ipv6
|
||||
mtu 1492
|
||||
mru 1492
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
environment.etc."ppp/chap-secrets".source = creds.files."pppoe-password".path;
|
||||
environment.etc."ppp/pap-secrets".source = creds.files."pppoe-password".path;
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -4,4 +4,5 @@
|
||||
"cnx.network"
|
||||
"buildfor.life"
|
||||
"cnx.email"
|
||||
"newedge.house"
|
||||
]
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
$ORIGIN newedge.house.
|
||||
$TTL 3600
|
||||
|
||||
@ IN SOA ns1.cnx.network. hostmaster.cnx.network. (
|
||||
2026061401 ; serial (ignored: Knot auto-assigns a dateserial on signing)
|
||||
3600 ; refresh
|
||||
900 ; retry
|
||||
604800 ; expire
|
||||
300 ) ; negative-cache TTL
|
||||
|
||||
; Served by the same nameservers (out-of-bailiwick, no glue needed here).
|
||||
@ IN NS ns1.cnx.network.
|
||||
@ IN NS ns2.cnx.network.
|
||||
@@ -59,6 +59,31 @@ in
|
||||
}
|
||||
];
|
||||
}
|
||||
{
|
||||
# WAN quality on the site gateways, fed by the hourly speedtest timer
|
||||
# (modules/router/speedtest.nix) via the node_exporter textfile
|
||||
# collector. Degradation is judged against the link's own 7-day median,
|
||||
# so no per-site/per-ISP-plan threshold needs maintaining.
|
||||
name = "wan";
|
||||
rules = [
|
||||
{
|
||||
alert = "WANSpeedDegraded";
|
||||
expr = "speedtest_download_mbps < 0.5 * quantile_over_time(0.5, speedtest_download_mbps[7d])";
|
||||
for = "3h";
|
||||
labels.severity = "warning";
|
||||
annotations.summary = "WAN download on {{ $labels.instance }} is below half its 7-day median";
|
||||
annotations.description = "Sustained for 3h (several test runs) — likely ISP degradation, line fault, or saturation. Compare speedtest_upload_mbps/speedtest_ping_ms and run `librespeed-cli` manually on the gateway.";
|
||||
}
|
||||
{
|
||||
alert = "WANSpeedTestFailing";
|
||||
expr = "speedtest_success == 0";
|
||||
for = "3h";
|
||||
labels.severity = "warning";
|
||||
annotations.summary = "Speed tests on {{ $labels.instance }} have been failing for 3h";
|
||||
annotations.description = "librespeed-cli cannot complete a test while other traffic may still flow; check `journalctl -u speedtest` on the gateway.";
|
||||
}
|
||||
];
|
||||
}
|
||||
{
|
||||
name = "backup";
|
||||
rules = [
|
||||
|
||||
@@ -0,0 +1,267 @@
|
||||
{
|
||||
"uid": "cnx-wan",
|
||||
"title": "CNX WAN Speed",
|
||||
"tags": ["wan", "speedtest", "gateway", "cnx"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"refresh": "5m",
|
||||
"time": { "from": "now-7d", "to": "now" },
|
||||
"templating": { "list": [] },
|
||||
"annotations": { "list": [] },
|
||||
"panels": [
|
||||
{
|
||||
"type": "row",
|
||||
"title": "WAN speed (site gateways)",
|
||||
"id": 1,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Download (latest)",
|
||||
"description": "Most recent hourly librespeed result per gateway.",
|
||||
"id": 2,
|
||||
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
||||
"gridPos": { "h": 5, "w": 6, "x": 0, "y": 1 },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bps",
|
||||
"color": { "mode": "fixed", "fixedColor": "green" },
|
||||
"noValue": "no data"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"textMode": "value_and_name",
|
||||
"orientation": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
||||
"expr": "speedtest_download_mbps * 1e6",
|
||||
"legendFormat": "{{instance}}",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Upload (latest)",
|
||||
"description": "Most recent hourly librespeed result per gateway.",
|
||||
"id": 3,
|
||||
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
||||
"gridPos": { "h": 5, "w": 6, "x": 6, "y": 1 },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bps",
|
||||
"color": { "mode": "fixed", "fixedColor": "blue" },
|
||||
"noValue": "no data"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"textMode": "value_and_name",
|
||||
"orientation": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
||||
"expr": "speedtest_upload_mbps * 1e6",
|
||||
"legendFormat": "{{instance}}",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Ping (latest)",
|
||||
"description": "Latency to the librespeed test server. Yellow over 50 ms, red over 100 ms.",
|
||||
"id": 4,
|
||||
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
||||
"gridPos": { "h": 5, "w": 6, "x": 12, "y": 1 },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ms",
|
||||
"color": { "mode": "thresholds" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 50 },
|
||||
{ "color": "red", "value": 100 }
|
||||
]
|
||||
},
|
||||
"noValue": "no data"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"textMode": "value_and_name",
|
||||
"orientation": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
||||
"expr": "speedtest_ping_ms",
|
||||
"legendFormat": "{{instance}}",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Test status",
|
||||
"description": "Whether the last speed test on each gateway completed. FAILING also fires the WANSpeedTestFailing alert after 3h.",
|
||||
"id": 5,
|
||||
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
||||
"gridPos": { "h": 5, "w": 6, "x": 18, "y": 1 },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "mode": "thresholds" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
},
|
||||
"noValue": "no data",
|
||||
"mappings": [
|
||||
{
|
||||
"type": "value",
|
||||
"options": {
|
||||
"0": { "text": "FAILING", "color": "red", "index": 0 },
|
||||
"1": { "text": "OK", "color": "green", "index": 1 }
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"textMode": "value_and_name",
|
||||
"orientation": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
||||
"expr": "speedtest_success",
|
||||
"legendFormat": "{{instance}}",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Download",
|
||||
"description": "Hourly download rate per gateway, with each link's own 7-day median as reference — the WANSpeedDegraded alert fires when a link stays below half its median for 3h.",
|
||||
"id": 6,
|
||||
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
||||
"gridPos": { "h": 9, "w": 12, "x": 0, "y": 6 },
|
||||
"fieldConfig": {
|
||||
"defaults": { "unit": "bps", "min": 0 },
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": { "id": "byRegexp", "options": ".*7d median.*" },
|
||||
"properties": [
|
||||
{
|
||||
"id": "custom.lineStyle",
|
||||
"value": { "fill": "dash", "dash": [10, 10] }
|
||||
},
|
||||
{ "id": "custom.fillOpacity", "value": 0 }
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
||||
"expr": "speedtest_download_mbps * 1e6",
|
||||
"legendFormat": "{{instance}}"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
||||
"expr": "quantile_over_time(0.5, speedtest_download_mbps[7d]) * 1e6",
|
||||
"legendFormat": "{{instance}} 7d median"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Upload",
|
||||
"description": "Hourly upload rate per gateway.",
|
||||
"id": 7,
|
||||
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
||||
"gridPos": { "h": 9, "w": 12, "x": 12, "y": 6 },
|
||||
"fieldConfig": {
|
||||
"defaults": { "unit": "bps", "min": 0 },
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
||||
"expr": "speedtest_upload_mbps * 1e6",
|
||||
"legendFormat": "{{instance}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Latency & jitter",
|
||||
"description": "Ping and jitter towards the librespeed test server. A latency climb with stable throughput usually means bufferbloat or an ISP routing change.",
|
||||
"id": 8,
|
||||
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 15 },
|
||||
"fieldConfig": {
|
||||
"defaults": { "unit": "ms", "min": 0 },
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
||||
"expr": "speedtest_ping_ms",
|
||||
"legendFormat": "{{instance}} ping"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
||||
"expr": "speedtest_jitter_ms",
|
||||
"legendFormat": "{{instance}} jitter"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -11,6 +11,7 @@
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
let
|
||||
@@ -67,6 +68,25 @@ let
|
||||
e.g. public-WiFi guest VLANs (3600-7200), so the pool recycles.
|
||||
'';
|
||||
};
|
||||
reservations = lib.mkOption {
|
||||
type = lib.types.attrsOf (
|
||||
lib.types.submodule {
|
||||
options = {
|
||||
hwAddress = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
example = "aa:bb:cc:dd:ee:ff";
|
||||
description = "Client MAC address.";
|
||||
};
|
||||
ipAddress = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "Fixed address handed to this client (inside the VLAN's subnet, outside the pool).";
|
||||
};
|
||||
};
|
||||
}
|
||||
);
|
||||
default = { };
|
||||
description = "Static DHCP leases; the attribute name becomes the client's hostname.";
|
||||
};
|
||||
};
|
||||
allowWan = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
@@ -85,6 +105,8 @@ in
|
||||
./crowdsec.nix
|
||||
./omada.nix
|
||||
./proxy.nix
|
||||
./iperf.nix
|
||||
./speedtest.nix
|
||||
];
|
||||
|
||||
options.cnx.router = {
|
||||
@@ -138,6 +160,19 @@ in
|
||||
description = "LAN ports carrying all VLANs tagged (incl. any 10G SFP+ ports).";
|
||||
};
|
||||
|
||||
accessPorts = lib.mkOption {
|
||||
type = lib.types.attrsOf lib.types.str;
|
||||
default = { };
|
||||
example = {
|
||||
enp4s0 = "mgmt";
|
||||
};
|
||||
description = ''
|
||||
Ports acting as untagged access ports on a single VLAN (port name ->
|
||||
VLAN name). Frames are untagged on the wire; the bridge tags them with
|
||||
the VLAN's PVID. Use for an always-available on-site mgmt port.
|
||||
'';
|
||||
};
|
||||
|
||||
vlans = lib.mkOption {
|
||||
type = lib.types.attrsOf (lib.types.submodule vlanModule);
|
||||
description = "VLANs served at this site; `mgmt` and `lan` are mandatory.";
|
||||
@@ -150,6 +185,27 @@ in
|
||||
assertion = cfg.vlans ? mgmt && cfg.vlans ? lan;
|
||||
message = "cnx.router: every site must define the `mgmt` and `lan` VLANs.";
|
||||
}
|
||||
{
|
||||
assertion = lib.all (v: cfg.vlans ? ${v}) (lib.attrValues cfg.accessPorts);
|
||||
message = "cnx.router: every accessPorts value must name a defined VLAN.";
|
||||
}
|
||||
{
|
||||
assertion = lib.all (p: !(cfg.accessPorts ? ${p})) cfg.trunkPorts;
|
||||
message = "cnx.router: a port cannot be both a trunk and an access port.";
|
||||
}
|
||||
];
|
||||
|
||||
# Router diagnostics toolkit: packets (tcpdump), path (mtr), link
|
||||
# negotiation (ethtool), NAT state (conntrack), DNS (kdig), per-flow
|
||||
# bandwidth (iftop), WAN throughput (librespeed-cli; iperf3 covers LAN).
|
||||
environment.systemPackages = with pkgs; [
|
||||
tcpdump
|
||||
mtr
|
||||
ethtool
|
||||
conntrack-tools
|
||||
knot-dns
|
||||
iftop
|
||||
librespeed-cli
|
||||
];
|
||||
|
||||
networking.useNetworkd = true;
|
||||
@@ -235,6 +291,21 @@ in
|
||||
};
|
||||
}) cfg.trunkPorts
|
||||
)
|
||||
// lib.mapAttrs' (
|
||||
port: vlanName:
|
||||
lib.nameValuePair "25-access-${port}" {
|
||||
matchConfig.Name = port;
|
||||
networkConfig.Bridge = "br0";
|
||||
bridgeVLANs = [
|
||||
{
|
||||
VLAN = cfg.vlans.${vlanName}.id;
|
||||
PVID = cfg.vlans.${vlanName}.id;
|
||||
EgressUntagged = cfg.vlans.${vlanName}.id;
|
||||
}
|
||||
];
|
||||
linkConfig.RequiredForOnline = "no";
|
||||
}
|
||||
) cfg.accessPorts
|
||||
// lib.mapAttrs' (
|
||||
name: vlan:
|
||||
lib.nameValuePair "40-${vlanIf name}" {
|
||||
|
||||
@@ -29,6 +29,11 @@ in
|
||||
interface = "vlan-${name}";
|
||||
valid-lifetime = vlan.dhcp.leaseTime;
|
||||
pools = [ { pool = "${vlan.dhcp.pool.from} - ${vlan.dhcp.pool.to}"; } ];
|
||||
reservations = lib.mapAttrsToList (host: res: {
|
||||
hostname = host;
|
||||
hw-address = res.hwAddress;
|
||||
ip-address = res.ipAddress;
|
||||
}) vlan.dhcp.reservations;
|
||||
option-data = [
|
||||
{
|
||||
name = "routers";
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
# iperf3 server on every gateway, for throughput testing from any LAN segment
|
||||
# (e.g. validating AP/switch links: `iperf3 -c 10.<siteId>.<vlan>.1`) and from
|
||||
# admin machines over the mesh. Never reachable from the WAN (default-deny).
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
...
|
||||
}:
|
||||
let
|
||||
cfg = config.cnx.router;
|
||||
mesh = import ../mesh-hosts.nix { inherit config lib; };
|
||||
vlanIfs = lib.mapAttrsToList (name: _: "vlan-${name}") cfg.vlans;
|
||||
in
|
||||
{
|
||||
config = lib.mkIf cfg.enable {
|
||||
services.iperf3.enable = true;
|
||||
|
||||
networking.firewall.interfaces = lib.genAttrs vlanIfs (_: {
|
||||
allowedTCPPorts = [ 5201 ];
|
||||
allowedUDPPorts = [ 5201 ];
|
||||
});
|
||||
|
||||
networking.firewall.extraInputRules = ''
|
||||
ip6 saddr ${mesh.subnet} tcp dport 5201 accept comment "iperf3 over the mesh"
|
||||
ip6 saddr ${mesh.subnet} udp dport 5201 accept comment "iperf3 over the mesh"
|
||||
'';
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
# Periodic WAN speed test so ISP degradation shows up as a trend instead of a
|
||||
# complaint. A timer runs librespeed-cli and writes the results as Prometheus
|
||||
# metrics into node_exporter's textfile collector — they ride the existing
|
||||
# 9100 scrape to VictoriaMetrics, where alerts.nix compares each run against
|
||||
# the link's own 7-day median (no per-site threshold to maintain).
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
let
|
||||
cfg = config.cnx.router;
|
||||
textfileDir = "/var/lib/speedtest";
|
||||
in
|
||||
{
|
||||
options.cnx.router.speedtest.interval = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "hourly";
|
||||
description = "systemd OnCalendar spec for the WAN speed test.";
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
services.prometheus.exporters.node.extraFlags = [
|
||||
"--collector.textfile.directory=${textfileDir}"
|
||||
];
|
||||
|
||||
systemd.services.speedtest = {
|
||||
description = "WAN speed test to Prometheus textfile metrics";
|
||||
after = [ "network-online.target" ];
|
||||
wants = [ "network-online.target" ];
|
||||
path = [
|
||||
pkgs.librespeed-cli
|
||||
pkgs.jq
|
||||
];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
StateDirectory = "speedtest";
|
||||
# One test at boot would race PPPoE and log a spurious failure.
|
||||
ExecCondition = "${pkgs.iproute2}/bin/ip link show ppp0";
|
||||
};
|
||||
script = ''
|
||||
tmp="${textfileDir}/.speedtest.prom.tmp"
|
||||
if result=$(librespeed-cli --json); then
|
||||
jq -r '.[0]
|
||||
| "speedtest_download_mbps \(.download)",
|
||||
"speedtest_upload_mbps \(.upload)",
|
||||
"speedtest_ping_ms \(.ping)",
|
||||
"speedtest_jitter_ms \(.jitter)",
|
||||
"speedtest_success 1"' <<<"$result" > "$tmp"
|
||||
else
|
||||
echo "speedtest_success 0" > "$tmp"
|
||||
fi
|
||||
mv "$tmp" "${textfileDir}/speedtest.prom"
|
||||
'';
|
||||
};
|
||||
|
||||
systemd.timers.speedtest = {
|
||||
wantedBy = [ "timers.target" ];
|
||||
timerConfig = {
|
||||
OnCalendar = cfg.speedtest.interval;
|
||||
RandomizedDelaySec = "10m";
|
||||
Persistent = true;
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user