B4L/cnx-network-clan
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: B4L/cnx-network-clan:mob/clanService-router
Branches Tags
B4L/cnx-network-clan:main B4L/cnx-network-clan:clanService-router-ai B4L/cnx-network-clan:mob/clanService-router B4L/cnx-network-clan:clanService-router
..
compare: B4L/cnx-network-clan:60aac6efb277a12c24fde29eb77422b3c3519df4
Branches Tags
B4L/cnx-network-clan:clanService-router-ai B4L/cnx-network-clan:mob/clanService-router B4L/cnx-network-clan:clanService-router B4L/cnx-network-clan:main
1 Commits
mob/clanService-router .. 60aac6efb2
Author SHA1 Message Date
kurogeek 60aac6efb2 services/router: init 2026-09-17 08:57:32 +07:00
52 changed files with 1926 additions and 1997 deletions
Expand all files Collapse all files
clan.nix
+77 -160
View File
@@ -1,10 +1,4 @@
{ { ... }:
inputs,
self,
libNet,
lib,
...
}:
let let
hosts = import ./modules/hosts.nix; hosts = import ./modules/hosts.nix;
@@ -24,166 +18,89 @@ let
}; };
in in
{ {
clan = { # Site gateways: the router service instance lives in inventory.nix.
# Ensure this is unique among all clans you want to use. imports = [ ./inventory.nix ];
meta.name = "cnx-network-clan";
meta.domain = "cnx-network.internal";
specialArgs = { inherit inputs self; }; # Ensure this is unique among all clans you want to use.
meta.name = "cnx-network-clan";
meta.domain = "cnx-network.internal";
# Customize nixpkgs inventory.machines = fleet;
# pkgsForSystem =
# system:
# import nixpkgs {
# inherit system;
# config = {
# allowUnfree = true;
# };
# overlays = [];
# };
secrets.age.plugins = [
"age-plugin-yubikey"
"age-plugin-fido2-hmac"
];
inventory.machines = fleet; # Services defined in this repo (module.input = "self"). See modules/clan/.
modules.router = ./modules/clan/router;
inventory.instances = { inventory.instances = {
# Admin SSH keys + root password, split per the clan-core migration off # Admin SSH keys + root password, split per the clan-core migration off
# the deprecated `admin` service (sshd handles keys, users the password). # the deprecated `admin` service (sshd handles keys, users the password).
sshd = { sshd = {
roles.server.tags.all = { }; roles.server.tags.all = { };
roles.server.settings.authorizedKeys = { roles.server.settings.authorizedKeys = {
"berwn" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIENAjhGQGraQoAjJzsomKP8GAmQPeGL1rNRNHgRcLqtT"; "berwn" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIENAjhGQGraQoAjJzsomKP8GAmQPeGL1rNRNHgRcLqtT";
"kurogeek" = "kurogeek" =
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEcZ/p1Ofa9liwIzPWzNtONhJ7+FUWd2lCz33r81t8+w kurogeek@kurogeek"; "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEcZ/p1Ofa9liwIzPWzNtONhJ7+FUWd2lCz33r81t8+w kurogeek@kurogeek";
};
};
gw-router = {
module = {
name = "gw-router";
input = "self";
};
roles.default.machines."gw-cnx-1" =
let
genVlan = id: length: rec {
inherit id;
prefixLength = length;
subnet = libNet.net.cidr.make length "10.0.${lib.toString id}.0";
address = libNet.net.cidr.host 1 subnet;
dhcp.pool.from = libNet.net.cidr.host 100 subnet;
dhcp.pool.to = libNet.net.cidr.host 200 subnet;
};
in
{
settings = {
wan = {
interface = "enp1s0";
vlanId = null;
};
vlans = {
mgmt = lib.recursiveUpdate (genVlan 10 24) {
dhcp.fixedIPs.storinator01 = {
hwAddress = "7c:c2:55:e0:d6:40";
ipAddress = "10.1.10.53";
};
};
lan = genVlan 20 24;
iot = genVlan 40 24;
voip = genVlan 50 24;
dmz = genVlan 60 24;
unit1 = genVlan 110 24;
unit2 = genVlan 120 24;
unit3 = genVlan 130 24;
unit4 = genVlan 140 24;
unit5 = lib.recursiveUpdate (genVlan 150 24) {
dhcp.fixedIPs.newt = {
hwAddress = "7c:d3:0a:21:58:0b";
ipAddress = "10.1.150.22";
};
};
};
trunkPorts = [ "enp2s0" ];
accessPorts = {
enp4s0.vlanId = 10;
};
upLinkPorts = [ "enp3s0" ];
enableOmada = true;
speedtest = {
enable = true;
};
};
};
};
root-user = {
module = {
name = "users";
input = "clan-core";
};
roles.default.tags.all = { };
roles.default.settings = {
user = "root";
prompt = false; # auto-generate, like the old admin service
share = false; # per-machine password, not fleet-wide
};
};
zerotier = {
roles.controller.machines."control" = { };
roles.peer.tags.all = { };
# External members admitted by ZeroTier node id (stable per device).
# Inventory machines are auto-accepted; this is only for peers outside the
# clan. Node id comes from `zerotier-cli info` on the joining device.
roles.controller.settings.allowedIds = [
"8802c8d7e0" # alex-nixos
"2bd36db8cc" # kurogeek-thinkpad
];
};
tor = {
roles.server.tags.nixos = { };
};
# Direct SSH to public IPs — clan's priority-1 connection path, with the
# ZeroTier mesh and Tor kept as automatic fallbacks. Raw IPs (from
# modules/hosts.nix, not the ns1/ns2 DNS names) so reaching these hosts never
# depends on their own DNS being up.
internet.roles.default.machines = builtins.mapAttrs (_: h: {
settings.host = h.ipv4;
}) hosts;
# Recovery root password for console access when a machine fails to boot.
emergency-access = {
roles.default.tags.nixos = { };
};
# Encrypted, deduplicating backups. control hosts the repos; ns1 is the
# only client, backing up its declared clan.core.state (the Knot DNSSEC
# keystore) over the mesh. Repo lives at /var/lib/borgbackup/ns1 on control.
# Cross-host so an ns1 loss is recoverable; repokey encryption means control
# never holds plaintext. Run `clan vars generate ns1` (YubiKey) before deploy.
borgbackup = {
roles.server.machines.control = { };
roles.client.machines.ns1 = { };
}; };
}; };
# Fleet-wide baseline applied to every machine. Secrets minted by root-user = {
# `clan vars generate` are encrypted for the admins group from the very module = {
# first run — generating before this took effect is what forced the name = "users";
# re-encryption dance (`clan vars fix`) on gw-cnx-1. input = "clan-core";
machines = builtins.mapAttrs (_: _: { };
clan.core.sops.defaultGroups = [ "admins" ]; roles.default.tags.all = { };
}) fleet; roles.default.settings = {
user = "root";
prompt = false; # auto-generate, like the old admin service
share = false; # per-machine password, not fleet-wide
};
};
zerotier = {
roles.controller.machines."control" = { };
roles.peer.tags.all = { };
# External members admitted by ZeroTier node id (stable per device).
# Inventory machines are auto-accepted; this is only for peers outside the
# clan. Node id comes from `zerotier-cli info` on the joining device.
roles.controller.settings.allowedIds = [
"8802c8d7e0" # alex-nixos
"2bd36db8cc" # kurogeek-thinkpad
];
};
tor = {
roles.server.tags.nixos = { };
};
# Direct SSH to public IPs — clan's priority-1 connection path, with the
# ZeroTier mesh and Tor kept as automatic fallbacks. Raw IPs (from
# modules/hosts.nix, not the ns1/ns2 DNS names) so reaching these hosts never
# depends on their own DNS being up.
internet.roles.default.machines = builtins.mapAttrs (_: h: {
settings.host = h.ipv4;
}) hosts;
# Recovery root password for console access when a machine fails to boot.
emergency-access = {
roles.default.tags.nixos = { };
};
# Encrypted, deduplicating backups. control hosts the repos; ns1 is the
# only client, backing up its declared clan.core.state (the Knot DNSSEC
# keystore) over the mesh. Repo lives at /var/lib/borgbackup/ns1 on control.
# Cross-host so an ns1 loss is recoverable; repokey encryption means control
# never holds plaintext. Run `clan vars generate ns1` (YubiKey) before deploy.
borgbackup = {
roles.server.machines.control = { };
roles.client.machines.ns1 = { };
};
}; };
# Fleet-wide baseline applied to every machine. Secrets minted by
# `clan vars generate` are encrypted for the admins group from the very
# first run — generating before this took effect is what forced the
# re-encryption dance (`clan vars fix`) on gw-cnx-1.
machines = builtins.mapAttrs (_: _: {
clan.core.sops.defaultGroups = [ "admins" ];
}) fleet;
} }
docs/src/gateways.md
+62 -39
View File
@@ -3,28 +3,39 @@
NixOS routers (OPNsense replacements) on Topton 1U fanless boxes — Intel N300, NixOS routers (OPNsense replacements) on Topton 1U fanless boxes — Intel N300,
4x i226-V 2.5G ports (some units add 2x 10G SFP+), 16GB DDR5, 256GB mSATA. 4x i226-V 2.5G ports (some units add 2x 10G SFP+), 16GB DDR5, 256GB mSATA.
Four sites are planned; the first is `gw-cnx-1` (Chiang Mai). Everything lives Four sites are planned; the first is `gw-cnx-1` (Chiang Mai). Everything lives
in the reusable `modules/router/` module (`cnx.router.*` options); a site's in the `router` clan service (`modules/clan/router/`, registered as
machine config only sets port names, VLANs, and feature flags. `modules.router` in `clan.nix`); a site joins by adding its machine to the
`router` inventory instance in `inventory.nix` with settings for port names,
VLANs, and feature flags (`roles.default.machines.<gw>.settings`).
Naming: `gw-<city>-<n>`, e.g. `gw-cnx-1`. Naming: `gw-<city>-<n>`, e.g. `gw-cnx-1`.
The service has a NixOS VM test (`modules/clan/router/tests/vm/`): an ISP
node runs a PPPoE access concentrator, a `client` sits on the tagged lan/iot
VLANs behind the trunk, an `admin` on the untagged mgmt access port, and a
simulated radio (`mac80211_hwsim`) carries two SSIDs with a WPA3 station. It
checks the PPPoE dial-in, leases/reservations, Blocky, NAT, `allowWan`, the
mgmt-only trust model, SSH exposure and the Wi-Fi bridge ports. Run it with
`nix build .#checks.x86_64-linux.router` (also part of `nix flake check`).
## What each gateway runs ## What each gateway runs
| Function | Implementation | | Function | Implementation |
| ------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | ------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| WAN | PPPoE (`pppd`), per-site ISP credentials via clan vars prompts; `wan.vlanId` when the ISP tags the session (AIS: 10); `wan.macAddress` to clone the old router's MAC if the ISP has it pinned | | WAN | PPPoE (`pppd`), per-site ISP credentials via clan vars prompts; `wan.vlanId` when the ISP tags the session (AIS: 10); `wan.macAddress` to clone the old router's MAC if the ISP has it pinned |
| LAN | VLAN-filtering bridge `br0` over the trunk ports (networkd); `accessPorts` pin a port untagged to one VLAN — convention: the last copper port is an untagged `mgmt` recovery port | | LAN | VLAN-filtering bridge `br0` over the trunk ports (networkd); `accessPorts` pin a port untagged to one VLAN — convention: the last copper port is an untagged `mgmt` recovery port; `stagingPort` turns a spare port into a DHCP uplink into the old LAN while the box runs alongside the router it replaces (drop at cutover) |
| Firewall/NAT | nftables: default-deny WAN, no inter-VLAN, MSS clamp, v4 NAT | | Firewall/NAT | nftables: default-deny WAN, no inter-VLAN, MSS clamp, v4 NAT |
| DHCP | Kea, one subnet per VLAN | | DHCP | Kea, one subnet per VLAN |
| DNS | Blocky (blocklist resolver), metrics on :4000 scraped by control | | DNS | Blocky (blocklist resolver), metrics on :4000 scraped by control |
| IPv6 | DHCPv6-PD on ppp0, /64 per VLAN via SLAAC | | IPv6 | DHCPv6-PD on ppp0, /64 per VLAN via SLAAC |
| Bans | CrowdSec + nftables bouncer (sshd log parsing) | | Bans | CrowdSec + nftables bouncer (sshd log parsing) |
| Omada | Optional per site: TP-Link Omada controller as a podman container | | Omada | Optional per site: TP-Link Omada controller as a podman container |
| Proxy | Optional: Caddy reverse proxy for internal services under `*.<site><n>.cnx.network` with a real Let's Encrypt wildcard (DNS-01 against ns1) | | Wi-Fi | Optional: hostapd on the router's radios; each SSID (`wifi.networks`) is an untagged access port of its VLAN, passphrases via vars prompts — see `modules/clan/router/README.md` |
| Diagnostics | iperf3 server on 5201, reachable from every VLAN and the mesh (`iperf3 -c <router-vlan-addr>`); CLI toolkit: tcpdump, mtr, ethtool, conntrack, kdig, iftop, librespeed-cli | | Proxy | Optional: Caddy reverse proxy for internal services under `*.<site><n>.cnx.network` with a real Let's Encrypt wildcard (DNS-01 against ns1) |
| Speed test | Hourly librespeed run (`speedtest.timer`) → `speedtest_*` metrics via node_exporter; vmalert flags download < 50% of the link's own 7-day median (`WANSpeedDegraded`) | | Diagnostics | iperf3 server on 5201, reachable from every VLAN and the mesh (`iperf3 -c <router-vlan-addr>`); CLI toolkit: tcpdump, mtr, ethtool, conntrack, kdig, iftop, librespeed-cli |
| Management | ZeroTier mesh: SSH, node_exporter, journald upload — like the fleet | | Speed test | Hourly librespeed run (`speedtest.timer`) → `speedtest_*` metrics via node_exporter; vmalert flags download < 50% of the link's own 7-day median (`WANSpeedDegraded`) |
| Storage | Single-disk ZFS (zstd, `/var` its own dataset for snapshots) | | Management | ZeroTier mesh: SSH, node_exporter, journald upload — like the fleet |
| Storage | Single-disk ZFS (zstd, `/var` its own dataset for snapshots) |
## Addressing plan ## Addressing plan
@@ -55,22 +66,24 @@ Trust model: mgmt → everything; other VLANs → router DNS/DHCP + internet onl
## Adding a site ## Adding a site
1. Copy `machines/gw-cnx-1/` to `machines/gw-<city>-<n>/`; set `site`, 1. Copy `machines/gw-cnx-1/` to `machines/gw-<city>-<n>/` (disko, facter,
`siteId` (next free number), port names, VLANs, and `omada.enable`. hostId, timezone). Fill the real disk id in `disko.nix` (from the
Fill the real disk id in `disko.nix` (from the installer: installer: `ls -l /dev/disk/by-id/`).
`ls -l /dev/disk/by-id/`). Keep the `mgmt`/`lan` VLANs. 2. Add the machine to `inventory.machines` in `clan.nix`, to the `router`
2. Add the machine to `inventory.machines` in `clan.nix` and to the machine instance in `inventory.nix` (`roles.default.machines.gw-<city>-<n>.settings`: `site`,
list in `modules/mesh-hosts.nix`. Do **not** add it to `modules/hosts.nix` `siteId` (next free number), port names, VLANs, `omada.enable`; keep the
(dynamic PPPoE IP; clan connects over the mesh). `mgmt`/`lan` VLANs), and to the machine list in `modules/mesh-hosts.nix`.
Do **not** add it to `modules/hosts.nix` (dynamic PPPoE IP; clan connects
over the mesh).
3. Add its node_exporter and Blocky scrape targets in 3. Add its node_exporter and Blocky scrape targets in
`modules/monitoring/server.nix`. `modules/monitoring/server.nix`.
4. If the site uses the internal proxy (`cnx.router.proxy.enable`), wire ns1: 4. If the site uses the internal proxy (`proxy.enable` in its settings), wire
import `(import ../../modules/dns/acme-gw-secret.nix "gw-<city>-<n>")` in ns1: import `(import ../../modules/clan/router/acme-secret.nix
**both** the gateway's and ns1's configuration, and on ns1 add the "gw-<city>-<n>")` in ns1's configuration (the gateway declares the same
`dns-acme-gw-<city>-<n>-knot` generator, its `keyFiles` entry, and an shared generator itself), and on ns1 add the `dns-acme-gw-<city>-<n>-knot`
`acl_acme_gw_<city>_<n>` scoped to `_acme-challenge.<city><n>` on the generator, its `keyFiles` entry, and an `acl_acme_gw_<city>_<n>` scoped to
`cnx.network` zone (copy the `gw-cnx-1` blocks). Then `_acme-challenge.<city><n>` on the `cnx.network` zone (copy the `gw-cnx-1`
`clan vars generate ns1` and redeploy ns1. blocks). Then `clan vars generate ns1` and redeploy ns1.
5. `git add` the new machine directory — flake evaluation only sees 5. `git add` the new machine directory — flake evaluation only sees
git-tracked files, so an untracked `machines/gw-…/` is silently ignored. git-tracked files, so an untracked `machines/gw-…/` is silently ignored.
6. `clan vars generate gw-<city>-<n>` — prompts for the site's PPPoE 6. `clan vars generate gw-<city>-<n>` — prompts for the site's PPPoE
@@ -86,9 +99,10 @@ Trust model: mgmt → everything; other VLANs → router DNS/DHCP + internet onl
## Omada controller ## Omada controller
Sites with TP-Link Omada APs/switches set `cnx.router.omada.enable = true`. Sites with TP-Link Omada APs/switches set `omada.enable = true` in their
The controller runs as a podman container (`mbentley/omada-controller` — there router settings. The controller runs as a podman container
is no nixpkgs package) with host networking on the mgmt VLAN, where adoption (`mbentley/omada-controller` — there is no nixpkgs package) with host
networking on the mgmt VLAN, where adoption
broadcasts (UDP 29810) live. UI: `https://<mgmt-addr>:8043` from mgmt, or the broadcasts (UDP 29810) live. UI: `https://<mgmt-addr>:8043` from mgmt, or the
gateway's mesh address from an admin machine. Controller state is under gateway's mesh address from an admin machine. Controller state is under
`/var/lib/omada`, declared as clan state (`clan.core.state.omada`); wiring it `/var/lib/omada`, declared as clan state (`clan.core.state.omada`); wiring it
@@ -96,12 +110,12 @@ into the borgbackup instance is still a follow-up.
## Internal reverse proxy ## Internal reverse proxy
`cnx.router.proxy.enable` puts Caddy on the gateway, terminating TLS for `proxy.enable` in a site's router settings puts Caddy on the gateway,
`<service>.<site><n>.cnx.network` (e.g. `https://omada.cnx1.cnx.network`) and terminating TLS for `<service>.<site><n>.cnx.network` (e.g.
forwarding to internal backends: `https://omada.cnx1.cnx.network`) and forwarding to internal backends:
```nix ```nix
cnx.router.proxy = { proxy = {
enable = true; enable = true;
services.omada = { services.omada = {
backend = "https://127.0.0.1:8043"; backend = "https://127.0.0.1:8043";
@@ -114,13 +128,22 @@ cnx.router.proxy = {
gateway, issued via ACME DNS-01 (lego/RFC2136) against ns1 — works behind gateway, issued via ACME DNS-01 (lego/RFC2136) against ns1 — works behind
PPPoE with zero inbound reachability, and browsers trust it without a CA PPPoE with zero inbound reachability, and browsers trust it without a CA
install. Each gateway has its own TSIG key (`acme_gw_<city>_<n>`) that ns1 install. Each gateway has its own TSIG key (`acme_gw_<city>_<n>`) that ns1
scopes to `_acme-challenge.<site><n>` TXT records only. scopes to `_acme-challenge.<site><n>` TXT records only. The zone, ns1's
address and the ACME contact are fleet-wide role settings in `inventory.nix`
(`proxy.domain`, `proxy.acme.nameserver`, `proxy.acme.email`).
- **Resolution**: the names exist only internally — Blocky answers - **Resolution**: the names exist only internally — Blocky answers
`*.<site><n>.cnx.network` with the router's `lan` address; the public `*.<site><n>.cnx.network` with the router's `lan` address; the public
`cnx.network` zone never carries them. `cnx.network` zone never carries them.
- **Access**: `proxy.allowVlans` (default `mgmt` + `lan`) get 443 (and 80 for - **Access**: `proxy.allowVlans` (default `mgmt` + `lan`) get 443 (and 80 for
the HTTP→HTTPS redirect). Not exposed to WAN, guest VLANs, or the mesh. the HTTP→HTTPS redirect). Not exposed to WAN, guest VLANs, or the mesh.
## Using the service from another clan
The service is exported as `clan.modules.router` of this flake; add this
repo as a flake input and reference it with `module.input = "<input name>"`.
`modules/clan/router/README.md` is the consumer-facing description (it is
also the module's manifest readme, shown by `clan modules`).
## Runbook ## Runbook
- **PPPoE down**: `systemctl status pppd-wan`, `journalctl -u pppd-wan` on the - **PPPoE down**: `systemctl status pppd-wan`, `journalctl -u pppd-wan` on the
flake.lock
Generated
-32
View File
@@ -173,19 +173,6 @@
"type": "github" "type": "github"
} }
}, },
"lib-net": {
"flake": false,
"locked": {
"lastModified": 1596309860,
"narHash": "sha256-izAzepR/6cDvnRfaa2ceSolMLMwqzQB5x9q62aR5J2g=",
"type": "tarball",
"url": "https://gist.github.com/duairc/5c9bb3c922e5d501a1edb9e7b3b845ba/archive/3885f7cd9ed0a746a9d675da6f265d41e9fd6704.tar.gz"
},
"original": {
"type": "tarball",
"url": "https://gist.github.com/duairc/5c9bb3c922e5d501a1edb9e7b3b845ba/archive/3885f7cd9ed0a746a9d675da6f265d41e9fd6704.tar.gz"
}
},
"nix-darwin": { "nix-darwin": {
"inputs": { "inputs": {
"nixpkgs": [ "nixpkgs": [
@@ -207,24 +194,6 @@
"type": "github" "type": "github"
} }
}, },
"nix-lib-net": {
"inputs": {
"lib-net": "lib-net"
},
"locked": {
"lastModified": 1732312010,
"narHash": "sha256-kQuiPSq3Iz/t5RC+QqwHfvj4hKQesm/WGffcQlhhCVE=",
"owner": "reo101",
"repo": "nix-lib-net",
"rev": "0f3412ccb56d783b974341f5f83abfaed7aa8b5b",
"type": "github"
},
"original": {
"owner": "reo101",
"repo": "nix-lib-net",
"type": "github"
}
},
"nix-select": { "nix-select": {
"locked": { "locked": {
"lastModified": 1763303120, "lastModified": 1763303120,
@@ -282,7 +251,6 @@
"clan-core", "clan-core",
"flake-parts" "flake-parts"
], ],
"nix-lib-net": "nix-lib-net",
"nixos-mailserver": "nixos-mailserver", "nixos-mailserver": "nixos-mailserver",
"nixpkgs": [ "nixpkgs": [
"clan-core", "clan-core",
flake.nix
+34 -8
View File
@@ -8,16 +8,37 @@
inputs.nixos-mailserver.inputs.nixpkgs.follows = "nixpkgs"; inputs.nixos-mailserver.inputs.nixpkgs.follows = "nixpkgs";
inputs.flake-parts.follows = "clan-core/flake-parts"; inputs.flake-parts.follows = "clan-core/flake-parts";
inputs.nix-lib-net.url = "github:reo101/nix-lib-net";
outputs = outputs =
inputs@{ inputs@{
self, self,
clan-core, clan-core,
nixpkgs,
flake-parts, flake-parts,
... ...
}: }:
let let
# Usage see: https://docs.clan.lol
clan = clan-core.lib.clan {
inherit self;
imports = [ ./clan.nix ];
specialArgs = { inherit inputs; };
# Customize nixpkgs
# pkgsForSystem =
# system:
# import nixpkgs {
# inherit system;
# config = {
# allowUnfree = true;
# };
# overlays = [];
# };
secrets.age.plugins = [
"age-plugin-yubikey"
"age-plugin-fido2-hmac"
];
};
in in
flake-parts.lib.mkFlake { inherit inputs; } { flake-parts.lib.mkFlake { inherit inputs; } {
systems = [ systems = [
@@ -27,14 +48,13 @@
"x86_64-darwin" "x86_64-darwin"
]; ];
imports = [ # perSystem.clan.nixosTests: NixOS VM tests of our own clan services.
inputs.clan-core.flakeModules.default imports = [ clan-core.flakeModules.testModule ];
./clan.nix flake = {
./modules/clan/flake-module.nix inherit (clan.config) nixosConfigurations nixosModules clanInternals;
]; clan = clan.config;
};
_module.args.libNet = inputs.nixpkgs.lib.extend inputs.nix-lib-net.overlays.raw;
perSystem = perSystem =
{ system, ... }: { system, ... }:
@@ -67,6 +87,12 @@
formatter = treefmtEval.config.build.wrapper; formatter = treefmtEval.config.build.wrapper;
checks.formatting = treefmtEval.config.build.check self; checks.formatting = treefmtEval.config.build.check self;
# `nix build .#checks.x86_64-linux.router` (VM test, Linux only).
clan.nixosTests.router = {
imports = [ ./modules/clan/router/tests/vm ];
clan.modules.router = ./modules/clan/router;
};
# Add the Clan cli tool to the dev shell. # Add the Clan cli tool to the dev shell.
# Use "nix develop" to enter the dev shell. # Use "nix develop" to enter the dev shell.
devShells.default = pkgs.mkShell { devShells.default = pkgs.mkShell {
inventory.nix
+93
View File
@@ -0,0 +1,93 @@
# Site gateways: the `router` service instance (modules/clan/router). One
# instance for the fleet; each site adds its machine with the site's port /
# VLAN layout under roles.default.machines.<gw>.settings. Everything else
# (PPPoE, Kea/Blocky, nftables, IPv6-PD, CrowdSec, iperf3, speedtest) follows
# from the service. Imported by clan.nix.
{ config, lib, ... }:
let
hosts = import ./modules/hosts.nix;
mesh = import ./modules/mesh-hosts.nix {
dir = config.directory;
inherit lib;
};
in
{
inventory.instances.router = {
module = {
name = "router";
input = "self";
};
# Fleet-wide settings: admin paths (SSH, scrapes, iperf3, Omada UI) are
# opened to the mesh; internal proxy names live under
# <site><n>.cnx.network with wildcard certs issued via DNS-01 against
# ns1 (which must declare the gateway's acme-secret.nix generator).
roles.default.settings = {
mesh.subnet = mesh.subnet;
proxy = {
domain = "cnx.network";
acme.nameserver = hosts.ns1.ipv4;
acme.email = "postmaster@cnx.email";
};
};
# Chiang Mai (site 1): Topton 1U, Intel N300, 4x i226-V 2.5G. Port roles
# use the expected igc names — verify against facter.json after the
# first install and adjust if the box enumerates differently.
roles.default.machines.gw-cnx-1.settings = {
site = "cnx";
siteId = 1;
wan.interface = "enp1s0";
wan.vlanId = null; # this ISP runs PPPoE untagged on the port
wan.macAddress = "a8:b8:e0:01:06:87";
trunkPorts = [ "enp2s0" ];
# STAGING (remove at cutover, move enp3s0 back into trunkPorts):
# DHCP-client uplink into the existing OPNsense LAN so the box has
# internet + mesh while it runs alongside the old router.
stagingPort = "enp3s0";
# Dedicated on-site recovery port: untagged mgmt, always available even
# if the switch config is broken.
accessPorts.enp4s0 = "mgmt";
# Replaces the newedge.house OPNsense box; renumbered to the fleet
# convention (10.1.<vlanId>.0/24, router .1, pool .100-.199). The old
# untagged LAN becomes tagged mgmt — infra switch ports get PVID 10.
vlans = {
mgmt = {
id = 10; # 10.1.10.0/24 — servers, APs, switches, Omada, admin
dhcp.reservations.storinator01 = {
hwAddress = "7c:c2:55:e0:d6:40";
ipAddress = "10.1.10.53";
};
};
lan.id = 20; # 10.1.20.0/24 — trusted clients
iot.id = 40; # 10.1.40.0/24
voip.id = 50; # 10.1.50.0/24
dmz.id = 60; # 10.1.60.0/24
unit1.id = 110; # 10.1.110.0/24
unit2.id = 120; # 10.1.120.0/24
unit3.id = 130; # 10.1.130.0/24
unit4.id = 140; # 10.1.140.0/24
unit5 = {
id = 150; # 10.1.150.0/24
dhcp.reservations.newt = {
hwAddress = "7c:d3:0a:21:58:0b";
ipAddress = "10.1.150.22";
};
};
};
# This site runs the Omada controller for its APs/switches.
omada.enable = true;
# Internal reverse proxy: real wildcard cert *.cnx1.cnx.network; Blocky
# resolves the names to the router's LAN address for mgmt+lan clients.
proxy = {
enable = true;
services.omada = {
# Omada's UI is HTTPS with a self-signed cert on the host network.
backend = "https://127.0.0.1:8043";
insecureSkipVerify = true;
};
};
};
};
}
machines/gw-cnx-1/configuration.nix
+3 -73
View File
@@ -1,13 +1,9 @@
# Site gateway Chiang Mai (site 1): Topton 1U, Intel N300, 4x i226-V 2.5G. # Site gateway Chiang Mai (site 1): Topton 1U, Intel N300, 4x i226-V 2.5G.
# Port roles below use the expected igc names — verify against facter.json # The router itself (ports, VLANs, features) is the `router` service instance
# after the first install and adjust if the box enumerates differently. # in clan.nix; only machine-local bits live here.
{ config, lib, ... }: { config, lib, ... }:
{ {
imports = [ imports = [ ../../modules/monitoring/exporters.nix ];
# ../../modules/router
../../modules/monitoring/exporters.nix
(import ../../modules/dns/acme-gw-secret.nix "gw-cnx-1")
];
clan.core.sops.defaultGroups = [ "admins" ]; clan.core.sops.defaultGroups = [ "admins" ];
@@ -20,72 +16,6 @@
builtins.hashString "sha256" config.networking.hostName builtins.hashString "sha256" config.networking.hostName
); );
# cnx.router = {
# enable = true;
# site = "cnx";
# siteId = 1;
# wan.interface = "enp1s0";
# wan.vlanId = null; # this ISP runs PPPoE untagged on the port
# trunkPorts = [
# "enp2s0"
# # "enp3s0" # STAGING: serves as the uplink below until cutover
# ];
# # Dedicated on-site recovery port: untagged mgmt, always available even
# # if the switch config is broken.
# accessPorts.enp4s0 = "mgmt";
# # Replaces the newedge.house OPNsense box; renumbered to the fleet
# # convention (10.1.<vlanId>.0/24, router .1, pool .100-.199). The old
# # untagged LAN becomes tagged mgmt — infra switch ports get PVID 10.
# vlans = {
# mgmt = {
# id = 10; # 10.1.10.0/24 — servers, APs, switches, Omada, admin
# dhcp.reservations.storinator01 = {
# hwAddress = "7c:c2:55:e0:d6:40";
# ipAddress = "10.1.10.53";
# };
# };
# lan.id = 20; # 10.1.20.0/24 — trusted clients
# iot.id = 40; # 10.1.40.0/24
# voip.id = 50; # 10.1.50.0/24
# dmz.id = 60; # 10.1.60.0/24
# unit1.id = 110; # 10.1.110.0/24
# unit2.id = 120; # 10.1.120.0/24
# unit3.id = 130; # 10.1.130.0/24
# unit4.id = 140; # 10.1.140.0/24
# unit5 = {
# id = 150; # 10.1.150.0/24
# dhcp.reservations.newt = {
# hwAddress = "7c:d3:0a:21:58:0b";
# ipAddress = "10.1.150.22";
# };
# };
# };
# # This site runs the Omada controller for its APs/switches.
# omada.enable = true;
#
# # Internal reverse proxy: real wildcard cert *.cnx1.cnx.network; Blocky
# # resolves the names to the router's LAN address for mgmt+lan clients.
# proxy = {
# enable = true;
# services.omada = {
# # Omada's UI is HTTPS with a self-signed cert on the host network.
# backend = "https://127.0.0.1:8043";
# insecureSkipVerify = true;
# };
# };
# };
# STAGING (remove at cutover, and restore enp3s0 to trunkPorts): DHCP-client
# uplink into the existing OPNsense LAN so the box has internet + mesh while
# it runs alongside the old router. Default-deny firewall on this interface
# (it's in no VLAN zone); PPPoE simply retries until the WAN port is cabled.
# Do NOT connect the trunk ports to the production switch while staging —
# Kea on tag 10 would fight the OPNsense LAN DHCP in one broadcast domain.
# systemd.network.networks."05-staging" = {
# matchConfig.Name = "enp3s0";
# networkConfig.DHCP = "ipv4";
# };
time.timeZone = "Etc/GMT-7"; # UTC+7 (Thailand, fixed offset, no DST) time.timeZone = "Etc/GMT-7"; # UTC+7 (Thailand, fixed offset, no DST)
services.chrony.enable = true; services.chrony.enable = true;
} }
machines/ns1/configuration.nix
+6 -2
View File
@@ -6,7 +6,10 @@
}: }:
let let
domains = import ../../modules/dns/domains.nix; domains = import ../../modules/dns/domains.nix;
mesh = import ../../modules/mesh-hosts.nix { inherit config lib; }; mesh = import ../../modules/mesh-hosts.nix {
dir = config.clan.core.settings.directory;
inherit lib;
};
hosts = import ../../modules/hosts.nix; hosts = import ../../modules/hosts.nix;
in in
{ {
@@ -14,7 +17,8 @@ in
../../modules/dns/authoritative.nix ../../modules/dns/authoritative.nix
../../modules/dns/acme-mx1-secret.nix ../../modules/dns/acme-mx1-secret.nix
../../modules/dns/acme-web01-secret.nix ../../modules/dns/acme-web01-secret.nix
(import ../../modules/dns/acme-gw-secret.nix "gw-cnx-1") # Same shared TSIG secret the gateway's router service declares (proxy).
(import ../../modules/clan/router/acme-secret.nix "gw-cnx-1")
../../modules/static-ipv6.nix ../../modules/static-ipv6.nix
../../modules/monitoring/exporters.nix ../../modules/monitoring/exporters.nix
]; ];
modules/clan/flake-module.nix
-26
View File
@@ -1,26 +0,0 @@
{
inputs,
...
}:
{
imports =
let
# Get all subdirectories in the current directory
dirContents = builtins.readDir ./.;
# Filter to include only directories that have a flake-module.nix file
# and exclude special directories like 'result'
validModuleDirs = builtins.filter (
name:
name != "result"
&& dirContents.${name} == "directory"
&& builtins.pathExists (./. + "/${name}/flake-module.nix")
) (builtins.attrNames dirContents);
# Create import paths for each valid directory
imports = (map (name: ./. + "/${name}/flake-module.nix") validModuleDirs) ++ [
inputs.clan-core.flakeModules.testModule
];
in
imports;
}
modules/clan/gw-router/crowdsec.nix
-59
View File
@@ -1,59 +0,0 @@
{
roles.default.perInstance = { ... }: {
nixosModule =
{
config,
...
}:
let
dir = config.clan.core.settings.directory;
instance = "zerotier";
networkId = builtins.readFile "${dir}/vars/shared/zerotier-network-${instance}/network-id/value";
full = "fd" + networkId + "9993";
hextet = i: builtins.substring (i * 4) 4 full;
subnetZtier = "${hextet 0}:${hextet 1}:${hextet 2}:${hextet 3}:${hextet 4}:${builtins.substring 20 2 full}00::/88";
in
{
services.crowdsec = {
enable = true;
autoUpdateService = true;
hub.collections = [
"crowdsecurity/linux"
"crowdsecurity/sshd"
];
localConfig = {
acquisitions = [
{
source = "journalctl";
journalctl_filter = [ "_SYSTEMD_UNIT=sshd.service" ];
labels.type = "syslog";
}
];
# Never ban the ZeroTier mesh — it is the only admin path to these
# boxes (no public SSH), so a false positive would lock us out.
# Parser-stage whitelist: mesh events are dropped before any scenario.
parsers.s02Enrich = [
{
name = "cnx/mesh-whitelist";
description = "Whitelist the ZeroTier management mesh";
whitelist = {
reason = "ZeroTier mesh is the admin path";
cidr = [ subnetZtier ];
};
}
];
};
};
services.crowdsec-firewall-bouncer = {
enable = true;
registerBouncer.enable = true;
settings.mode = "nftables";
};
};
};
}
modules/clan/gw-router/default.nix
-371
View File
@@ -1,371 +0,0 @@
{ ... }:
{
_class = "clan.service";
manifest.name = "gw-router";
manifest.description = "A gateway router service to configure most of a router features";
manifest.readme = "A gateway router service to configure most of a router features";
manifest.categories = [ "System" ];
roles.default = {
description = "Site gateway router role";
interface =
{ lib, config, ... }:
let
vlanModule =
{ ... }:
{
options = {
id = lib.mkOption {
type = lib.types.ints.between 1 4094;
description = "802.1Q VLAN id.";
};
address = lib.mkOption {
type = lib.types.str;
example = "10.0.10.1";
description = "Router address on this VLAN.";
};
prefixLength = lib.mkOption {
type = lib.types.ints.between 8 30;
default = 24;
};
subnet = lib.mkOption {
type = lib.types.str;
example = "10.0.10.0/24";
description = "The VLAN's network in CIDR form (must contain `address`).";
};
dhcp = {
enable = lib.mkOption {
type = lib.types.bool;
default = true;
};
pool = {
from = lib.mkOption {
type = lib.types.str;
example = "10.0.10.100";
};
to = lib.mkOption {
type = lib.types.str;
example = "10.0.10.199";
};
};
leaseTime = lib.mkOption {
type = lib.types.ints.positive;
default = 86400;
description = ''
Lease validity in seconds. Lower it for high-churn networks,
e.g. public-WiFi guest VLANs (3600-7200), so the pool recycles.
'';
};
fixedIPs = lib.mkOption {
type = lib.types.attrsOf (
lib.types.submodule {
options = {
hwAddress = lib.mkOption {
type = lib.types.str;
example = "aa:bb:cc:dd:ee:ff";
description = "Client MAC address.";
};
ipAddress = lib.mkOption {
type = lib.types.str;
description = "Fixed address handed to this client (inside the VLAN's subnet, outside the pool).";
};
};
}
);
default = { };
description = "Static DHCP leases; the attribute name becomes the client's hostname.";
};
};
allowedWAN = lib.mkOption {
type = lib.types.bool;
default = true;
description = "Whether clients on this VLAN may reach the internet.";
};
};
};
in
{
options = {
wan = {
interface = lib.mkOption {
type = lib.types.str;
description = "Physical WAN port the PPPoE session runs on.";
};
vlanId = lib.mkOption {
type = lib.types.nullOr (lib.types.ints.between 1 4094);
default = null;
description = ''
802.1Q tag the ISP requires for the PPPoE session (AIS Thailand: 10);
null for untagged PPPoE directly on the port. Unrelated to the LAN
VLANs — this tag exists only on the WAN port.
'';
};
macAddress = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "aa:bb:cc:dd:ee:ff";
description = ''
Spoofed MAC for the WAN port, e.g. to keep the MAC the ISP has
pinned (cloned from the old router). null keeps the hardware MAC.
'';
};
};
enableOmada = lib.mkOption {
type = lib.types.bool;
default = false;
description = "TP-Link Omada SDN controller (podman container)";
};
speedtest = {
enable = {
type = lib.types.bool;
default = false;
description = "Periodic WAN speed test so ISP degradation shows up as a trend. A timer runs librespeed-cli and writes the results as Prometheus metrics into node_exporter's textfile collector";
};
interval = lib.mkOption {
type = lib.types.str;
default = "hourly";
description = "systemd OnCalendar spec for the WAN speed test.";
};
};
proxy = {
enable = lib.mkEnableOption "internal reverse proxy (Caddy, wildcard cert via DNS-01)";
services = lib.mkOption {
type = lib.types.attrsOf (
lib.types.submodule {
options = {
backend = lib.mkOption {
type = lib.types.str;
example = "https://127.0.0.1:8043";
description = "URL Caddy forwards to (internal/mesh address).";
};
insecureSkipVerify = lib.mkOption {
type = lib.types.bool;
default = false;
description = "Skip TLS verification towards the backend.";
};
};
}
);
default = { };
description = "Proxied services";
};
allowVlans = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [
"mgmt"
"lan"
];
description = "VLANs whose clients may reach the proxy (443, plus 80 for the redirect).";
};
};
vlans = lib.mkOption {
type = lib.types.attrsOf (lib.types.submodule vlanModule);
description = "VLANs setup for this router";
};
trunkPorts = lib.mkOption {
type = lib.types.listOf lib.types.str;
description = "LAN ports carrying all VLANs tagged (incl. any 10G SFP+ ports).";
};
accessPorts = lib.mkOption {
type = lib.types.attrsOf (
lib.types.submodule {
options = {
vlanId = lib.mkOption {
type = lib.types.int;
description = "Untagged traffic in from the device gets tagged VLANs inside the bridge, and VLANs traffic going back out to the device gets untagged, so the device itself never has to know VLANs exist.";
};
};
}
);
description = ''
Ports acting as untagged access ports on a single VLAN (port name ->
VLAN id). Frames are untagged on the wire; the bridge tags them with
the VLAN's PVID. Use for an always-available on-site mgmt port.
'';
};
upLinkPorts = lib.mkOption {
type = lib.types.listOf lib.types.str;
description = "DHCP-client uplink into the existing router LAN so this machine has internet";
};
};
};
perInstance =
{ settings, ... }:
{
nixosModule =
{
pkgs,
lib,
...
}:
let
vlanIf = name: "vlan-${name}";
in
{
networking.useNetworkd = true;
networking.useDHCP = false;
systemd.network.enable = true;
systemd.network.netdevs = {
"20-br0" = {
netdevConfig = {
Name = "br0";
Kind = "bridge";
};
bridgeConfig = {
VLANFiltering = true;
};
};
}
// lib.optionalAttrs (settings.wan.vlanId != null) {
"15-wan-lan" = {
netdevConfig = {
Name = "wan-vlan";
Kind = "vlan";
};
vlanConfig.Id = settings.wan.vlanId;
};
}
// lib.mapAttrs' (
name: vlan:
lib.nameValuePair "30-${vlanIf name}" {
netdevConfig = {
Name = vlanIf name;
Kind = "vlan";
};
vlanConfig.Id = vlan.id;
}
) settings.vlans;
systemd.network.networks =
let
allVLANs = lib.mapAttrsToList (_: vlan: { VLAN = vlan.id; }) settings.vlans;
in
{
"10-wan" = {
matchConfig.Name = settings.wan.interface;
networkConfig.LinkLocalAddressing = "no";
vlan = lib.optional (settings.wan.vlanId != null) "wan-wlan";
linkConfig = {
RequiredForOnline = "carrier";
}
// lib.optionalAttrs (settings.wan.macAddress != null) {
MACAddress = settings.wan.macAddress;
};
};
}
// lib.optionalAttrs (settings.wan.vlanId != null) {
"15-wan-lan" = {
matchConfig.Name = "wan-vlan";
networkConfig.LinkLocalAddressing = "no";
linkConfig.RequiredForOnline = "no";
};
}
// {
"20-br0" = {
matchConfig.Name = "br0";
networkConfig.LinkLocalAddressing = "no";
vlan = lib.mapAttrsToList (name: _: vlanIf name) settings.vlans;
bridgeVLANs = allVLANs;
linkConfig.RequiredForOnline = "no";
};
}
// lib.listToAttrs (
map (port: {
name = "25-trunk-${port}";
value = {
matchConfig.Name = port;
networkConfig.Bridge = "br0";
bridgeVLANs = allVLANs;
linkConfig.RequiredForOnline = "no";
};
}) settings.trunkPorts
)
// lib.mapAttrs' (
iface: port:
lib.nameValuePair "25-access-${iface}" {
matchConfig.Name = iface;
networkConfig.Bridge = "br0";
bridgeVLANs = [
{
VLAN = port.vlanId;
PVID = port.vlanId;
EgressUntagged = port.vlanId;
}
];
linkConfig.RequiredForOnline = "no";
}
) settings.accessPorts
// lib.mapAttrs' (
name: vlan:
lib.nameValuePair "40-${vlanIf name}" {
matchConfig.Name = vlanIf name;
address = [ "${vlan.address}/${toString vlan.prefixLength}" ];
networkConfig = {
IPv6AcceptRA = false;
IPv6SendRA = true;
DHCPPrefixDelegation = true;
};
dhcpPrefixDelegationConfig.SubnetId = "auto";
linkConfig.RequiredForOnline = "no";
}
) settings.vlans
// lib.listToAttrs (
map (port: {
name = "15-uplink-${port}";
value = {
matchConfig.Name = "${port}";
networkConfig.DHCP = "ipv4";
};
}) settings.upLinkPorts
);
environment.systemPackages = with pkgs; [
tcpdump
mtr
ethtool
conntrack-tools
knot-dns
iftop
librespeed-cli
];
};
};
};
imports = [
./crowdsec.nix
./dns-dhcp.nix
./firewall.nix
./iperf.nix
./ipv6.nix
./omada.nix
./pppoe.nix
./proxy.nix
./speedtest.nix
];
}
modules/clan/gw-router/dns-dhcp.nix
-81
View File
@@ -1,81 +0,0 @@
{
roles.default.perInstance = { settings, ... }: {
nixosModule =
{
lib,
...
}:
let
dhcpVlans = lib.filterAttrs (_: vlan: vlan.dhcp.enable) settings.vlans;
in
{
services.kea.dhcp4 = {
enable = true;
settings = {
interfaces-config.interfaces = lib.mapAttrsToList (name: _: "vlan-${name}") dhcpVlans;
lease-database = {
type = "memfile";
persist = true;
name = "/var/lib/kea/dhcp4.leases";
};
valid-lifetime = 86400;
subnet4 = lib.mapAttrsToList (name: vlan: {
id = vlan.id;
subnet = vlan.subnet;
interface = "vlan-${name}";
valid-lifetime = vlan.dhcp.leaseTime;
pools = [ { pool = "${vlan.dhcp.pool.from} - ${vlan.dhcp.pool.to}"; } ];
reservations = lib.mapAttrsToList (host: res: {
hostname = host;
hw-address = res.hwAddress;
ip-address = res.ipAddress;
}) vlan.dhcp.fixedIPs;
option-data = [
{
name = "routers";
data = vlan.address;
}
{
name = "domain-name-servers";
data = vlan.address;
}
];
}) dhcpVlans;
};
};
services.blocky = {
enable = true;
settings = {
ports = {
dns = 53;
http = 4000;
};
upstreams.groups.default = [
"9.9.9.9"
"149.112.112.112"
"2620:fe::fe"
];
blocking = {
denylists.ads = [
"https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts"
];
clientGroupsBlock.default = [ "ads" ];
};
caching = {
minTime = "5m";
prefetching = true;
};
prometheus.enable = true;
};
};
networking.nameservers = [
"9.9.9.9"
"1.1.1.1"
];
};
};
}
modules/clan/gw-router/firewall.nix
-64
View File
@@ -1,64 +0,0 @@
{
roles.default.perInstance = { settings, ... }: {
nixosModule =
{
config,
lib,
...
}:
let
dir = config.clan.core.settings.directory;
instance = "zerotier";
networkId = builtins.readFile "${dir}/vars/shared/zerotier-network-${instance}/network-id/value";
full = "fd" + networkId + "9993";
hextet = i: builtins.substring (i * 4) 4 full;
subnetZtier = "${hextet 0}:${hextet 1}:${hextet 2}:${hextet 3}:${hextet 4}:${builtins.substring 20 2 full}00::/88";
vlanIfs = lib.mapAttrsToList (name: _: "vlan-${name}") settings.vlans;
wanVlanIfs = lib.mapAttrsToList (name: _: "vlan-${name}") (
lib.filterAttrs (_: vlan: vlan.allowedWAN) settings.vlans
);
nonMgmtIfs = lib.filter (i: i != "vlan-mgmt") vlanIfs;
ifSet = ifs: "{ ${lib.concatStringsSep ", " (map (i: "\"${i}\"") ifs)} }";
in
{
networking.nftables.enable = true;
services.openssh.openFirewall = false;
networking.firewall = {
enable = true;
filterForward = true;
trustedInterfaces = [ "vlan-mgmt" ];
interfaces = lib.genAttrs nonMgmtIfs (_: {
allowedTCPPorts = [ 53 ];
allowedUDPPorts = [
53
67
];
});
extraInputRules = ''
ip6 saddr ${subnetZtier} tcp dport 22 accept comment "admin ssh over the mesh"
ip6 saddr ${subnetZtier} tcp dport 4000 accept comment "blocky metrics scrape from control"
'';
extraForwardRules = ''
tcp flags syn tcp option maxseg size set rt mtu comment "MSS clamp for PPPoE mtu 1492"
iifname "vlan-mgmt" accept comment "mgmt reaches all VLANs and the WAN"
iifname ${ifSet wanVlanIfs} oifname "ppp0" accept comment "LAN to internet"
'';
};
networking.nat = {
enable = true;
externalInterface = "ppp0";
internalInterfaces = vlanIfs;
};
};
};
}
modules/clan/gw-router/flake-module.nix
-19
View File
@@ -1,19 +0,0 @@
{ self, inputs, ... }:
let
module = ./default.nix;
in
{
clan.modules = {
gw-router = module;
};
# perSystem =
# { ... }:
# {
# clan.nixosTests.service-headplane = {
# imports = [ ./tests/vm/default.nix ];
# _module.args = { inherit self inputs; };
#
# clan.modules."@clan/headplane" = module;
# };
# };
}
modules/clan/gw-router/iperf.nix
-34
View File
@@ -1,34 +0,0 @@
{
roles.default.perInstance = { settings, ... }: {
nixosModule =
{
config,
lib,
...
}:
let
vlanIfs = lib.mapAttrsToList (name: _: "vlan-${name}") settings.vlans;
dir = config.clan.core.settings.directory;
instance = "zerotier";
networkId = builtins.readFile "${dir}/vars/shared/zerotier-network-${instance}/network-id/value";
full = "fd" + networkId + "9993";
hextet = i: builtins.substring (i * 4) 4 full;
subnetZtier = "${hextet 0}:${hextet 1}:${hextet 2}:${hextet 3}:${hextet 4}:${builtins.substring 20 2 full}00::/88";
in
{
services.iperf3.enable = true;
networking.firewall.interfaces = lib.genAttrs vlanIfs (_: {
allowedTCPPorts = [ 5201 ];
allowedUDPPorts = [ 5201 ];
});
networking.firewall.extraInputRules = ''
ip6 saddr ${subnetZtier} tcp dport 5201 accept comment "iperf3 over the mesh"
ip6 saddr ${subnetZtier} udp dport 5201 accept comment "iperf3 over the mesh"
'';
};
};
}
modules/clan/gw-router/ipv6.nix
-29
View File
@@ -1,29 +0,0 @@
{
roles.default.perInstance = { ... }: {
nixosModule =
{
lib,
...
}:
{
systemd.network.networks."45-ppp0" = {
matchConfig.Name = "ppp0";
networkConfig = {
DHCP = "ipv6";
# pppd owns the v4 address/route on this link; don't let networkd
# tear them down.
KeepConfiguration = "static";
# Default v6 route comes from the ISP's RA when they send one.
IPv6AcceptRA = true;
};
# Many PPPoE ISPs never send an RA with the M flag; solicit regardless.
dhcpV6Config.WithoutRA = "solicit";
linkConfig.RequiredForOnline = "no";
};
boot.kernel.sysctl."net.ipv6.conf.all.forwarding" = lib.mkDefault 1;
};
};
}
modules/clan/gw-router/omada.nix
-43
View File
@@ -1,43 +0,0 @@
{
roles.default.perInstance = { settings, ... }: {
nixosModule =
{
config,
lib,
...
}:
let
dir = config.clan.core.settings.directory;
instance = "zerotier";
networkId = builtins.readFile "${dir}/vars/shared/zerotier-network-${instance}/network-id/value";
full = "fd" + networkId + "9993";
hextet = i: builtins.substring (i * 4) 4 full;
subnetZtier = "${hextet 0}:${hextet 1}:${hextet 2}:${hextet 3}:${hextet 4}:${builtins.substring 20 2 full}00::/88";
in
{
config = lib.mkIf (settings.enableOmada) {
virtualisation.podman.enable = true;
virtualisation.oci-containers = {
backend = "podman";
containers.omada = {
image = "docker.io/mbentley/omada-controller:5.15";
extraOptions = [ "--network=host" ];
environment.TZ = config.time.timeZone;
volumes = [
"/var/lib/omada/data:/opt/tplink/EAPController/data"
"/var/lib/omada/logs:/opt/tplink/EAPController/logs"
];
};
};
networking.firewall.extraInputRules = ''
ip6 saddr ${subnetZtier} tcp dport 8043 accept comment "omada ui over the mesh"
'';
clan.core.state.omada.folders = [ "/var/lib/omada" ];
};
};
};
}
modules/clan/gw-router/pppoe.nix
-58
View File
@@ -1,58 +0,0 @@
{
roles.default.perInstance = { settings, ... }: {
nixosModule =
{ config, ... }:
let
creds = config.clan.core.vars.generators.gw-router;
pppInterface = if settings.wan.vlanId == null then settings.wan.interface else "wan-vlan";
in
{
clan.core.vars.generators.gw-router = {
prompts.pppoe-username = {
description = "PPPoE username (from the ISP)";
type = "hidden";
};
prompts.pppoe-password = {
description = "PPPoE password (from the ISP)";
type = "hidden";
};
files."pppoe-username".secret = true;
files."pppoe-password".secret = true;
script = ''
user="$(cat "$prompts"/pppoe-username)"
pass="$(cat "$prompts"/pppoe-password)"
printf 'user "%s"\n' "$user" > "$out"/pppoe-username
printf '"%s" * "%s"\n' "$user" "$pass" > "$out"/pppoe-password
'';
};
services.pppd = {
enable = true;
peers.wan = {
autostart = true;
config = ''
plugin pppoe.so ${pppInterface}
ifname ppp0
file ${creds.files."pppoe-username".path}
noipdefault
defaultroute
noauth
hide-password
persist
maxfail 0
holdoff 5
lcp-echo-interval 15
lcp-echo-failure 3
+ipv6
mtu 1492
mru 1492
'';
};
};
environment.etc."ppp/chap-secrets".source = creds.files."pppoe-password".path;
environment.etc."ppp/pap-secrets".source = creds.files."pppoe-password".path;
};
};
}
modules/clan/gw-router/proxy.nix
-25
View File
@@ -1,25 +0,0 @@
{
roles.default.perInstance = { settings, ... }: {
nixosModule =
{
config,
lib,
...
}:
let
in
{
config = lib.mkIf (settings.proxy.enable) {
assertions = [
{
assertion = lib.all (v: settings.vlans ? ${v}) settings.proxy.allowVlans;
message = "cnx.router.proxy.allowVlans must name VLANs defined in cnx.router.vlans.";
}
];
};
};
};
}
modules/clan/gw-router/speedtest.nix
-60
View File
@@ -1,60 +0,0 @@
{
roles.default.perInstance = { settings, ... }: {
nixosModule =
{
pkgs,
lib,
...
}:
let
textfileDir = "/var/lib/speedtest";
in
{
config = lib.mkIf (settings.speedtest.enable) {
services.prometheus.exporters.node.extraFlags = [
"--collector.textfile.directory=${textfileDir}"
];
systemd.services.speedtest = {
description = "WAN speed test to Prometheus textfile metrics";
after = [ "network-online.target" ];
wants = [ "network-online.target" ];
path = [
pkgs.librespeed-cli
pkgs.jq
];
serviceConfig = {
Type = "oneshot";
StateDirectory = "speedtest";
# One test at boot would race PPPoE and log a spurious failure.
ExecCondition = "${pkgs.iproute2}/bin/ip link show ppp0";
};
script = ''
tmp="${textfileDir}/.speedtest.prom.tmp"
if result=$(librespeed-cli --json); then
jq -r '.[0]
| "speedtest_download_mbps \(.download)",
"speedtest_upload_mbps \(.upload)",
"speedtest_ping_ms \(.ping)",
"speedtest_jitter_ms \(.jitter)",
"speedtest_success 1"' <<<"$result" > "$tmp"
else
echo "speedtest_success 0" > "$tmp"
fi
mv "$tmp" "${textfileDir}/speedtest.prom"
'';
};
systemd.timers.speedtest = {
wantedBy = [ "timers.target" ];
timerConfig = {
OnCalendar = settings.speedtest.interval;
RandomizedDelaySec = "10m";
Persistent = true;
};
};
};
};
};
}
modules/clan/router/README.md
+91
View File
@@ -0,0 +1,91 @@
# router
Turns a machine with several NICs into a site gateway: PPPoE WAN (ISP
credentials via vars prompts), a VLAN-filtering bridge over the LAN ports with
one L3 interface per VLAN, Kea DHCP and Blocky DNS per VLAN, nftables
firewall/NAT, DHCPv6-PD, CrowdSec, an iperf3 server and a WAN speed-test
timer. Optional: a Wi-Fi access point on the router's own radios (hostapd),
the TP-Link Omada controller (podman) and an internal Caddy reverse proxy
with a real wildcard certificate (ACME DNS-01).
Addressing convention: a site owns `10.<siteId>.0.0/16`; VLAN `<id>` defaults
to `10.<siteId>.<id>.0/24`, router at `.1`, DHCP pool `.100-.199`. The `mgmt`
and `lan` VLANs are mandatory. Trust model: mgmt reaches everything; other
VLANs get router DNS/DHCP and (with `allowWan`) the internet, no inter-VLAN;
WAN nothing inbound; the admin mesh (`mesh.subnet`) gets SSH, metrics, iperf3
and the Omada UI.
## Usage from another clan
```nix
# flake.nix
inputs.cnx-network.url = "git+https://<host>/B4L/cnx-network-clan";
# clan.nix
inventory.instances.router = {
module = { name = "router"; input = "cnx-network"; };
roles.default.settings.mesh.subnet = "fd..::/88"; # your admin overlay
roles.default.machines.gw-1.settings = {
site = "ams";
siteId = 1;
wan.interface = "enp1s0";
wan.vlanId = 10; # or null for untagged PPPoE
trunkPorts = [ "enp2s0" ];
accessPorts.enp4s0 = "mgmt"; # untagged on-site recovery port
# stagingPort = "enp3s0"; # DHCP uplink into the old LAN until cutover
vlans = {
mgmt.id = 10;
lan.id = 20;
iot = { id = 40; allowWan = false; };
};
};
};
```
Then `clan vars generate gw-1` prompts for the PPPoE username/password.
### Wi-Fi access point
If the box has wireless cards, the router can be the site's AP. An SSID is
defined once and behaves like an untagged access port of its VLAN; radios
choose what to broadcast, so a dual-band card serves the same SSID twice:
```nix
wifi = {
enable = true;
countryCode = "TH";
networks = {
home.vlan = "lan"; # WPA3 with WPA2 fallback
things = { vlan = "iot"; security = "wpa2"; }; # legacy IoT
guest = { vlan = "guest"; isolateClients = true; };
};
radios = {
wlp5s0 = { band = "2g"; channel = 6; macAddress = "…"; networks = [ "home" "things" ]; };
wlp6s0 = { band = "5g"; channel = 36; networks = [ "home" ]; };
};
};
```
Passphrases are vars prompts (`wifi-<name>-passphrase`), asked once at `clan
vars generate`. A radio broadcasting more than one SSID needs its hardware
`macAddress`: hostapd wants a fixed BSSID per extra SSID, derived from it.
`security = "wpa3-transition"` (the default) offers SAE and WPA2-PSK-SHA256;
devices that only speak classic WPA2-PSK need `security = "wpa2"`.
### Internal proxy
`proxy.enable` serves `<name>.<site><siteId>.<proxy.domain>` under a wildcard
certificate obtained via RFC 2136 DNS-01 against `proxy.acme.nameserver`. The
gateway signs updates with TSIG key `acme_<hostname with _>`, whose secret is
the shared `dns-acme-<hostname>-secret` generator declared by this service.
The nameserver machine must declare the same generator so both sides hold one
secret — import `acme-secret.nix` from this directory with the gateway's name:
```nix
imports = [ (import "${inputs.cnx-network}/modules/clan/router/acme-secret.nix" "gw-1") ];
```
and load the key with an acl scoped to `_acme-challenge.<site><siteId>`.
The service does not open the WAN to anything; reach gateways over your mesh.
One instance per machine.
modules/clan/router/acme-secret.nix
+22
View File
@@ -0,0 +1,22 @@
# Shared TSIG secret for a gateway's dedicated ACME key (function: machine
# name -> NixOS module). The acme_<machine> key lets that gateway — and only
# it — write _acme-challenge.<site><siteId> TXT records on the authoritative
# nameserver to obtain its internal wildcard cert via DNS-01 (proxy.nix).
#
# The router service declares it on the gateway automatically when
# proxy.enable is set. The nameserver machine must declare the very same
# generator so both sides share one secret:
# imports = [ (import <router-service>/acme-secret.nix "gw-cnx-1") ];
# and then load it into its DNS server as key acme_gw_cnx_1 (hmac-sha256)
# with an acl scoped to that gateway's _acme-challenge label.
machine:
{ pkgs, ... }:
{
clan.core.vars.generators."dns-acme-${machine}-secret" = {
share = true;
files."secret".secret = true;
runtimeInputs = [ pkgs.openssl ];
# 32 random bytes, base64 — a valid hmac-sha256 TSIG secret.
script = ''openssl rand -base64 32 | tr -d '\n' > "$out"/secret'';
};
}
modules/clan/router/crowdsec.nix
+46
View File
@@ -0,0 +1,46 @@
# CrowdSec security engine + nftables bouncer: parses sshd auth attempts from
# the journal and bans offending source IPs at the firewall. Log-based (no
# inline DPI) so it costs the N300 next to nothing.
{ settings }:
{ ... }:
let
cfg = settings;
in
{
services.crowdsec = {
enable = true;
autoUpdateService = true;
hub.collections = [
"crowdsecurity/linux"
"crowdsecurity/sshd"
];
localConfig = {
acquisitions = [
{
source = "journalctl";
journalctl_filter = [ "_SYSTEMD_UNIT=sshd.service" ];
labels.type = "syslog";
}
];
# Never ban the ZeroTier mesh — it is the only admin path to these
# boxes (no public SSH), so a false positive would lock us out.
# Parser-stage whitelist: mesh events are dropped before any scenario.
parsers.s02Enrich = [
{
name = "cnx/mesh-whitelist";
description = "Whitelist the ZeroTier management mesh";
whitelist = {
reason = "ZeroTier mesh is the admin path";
cidr = [ cfg.mesh.subnet ];
};
}
];
};
};
services.crowdsec-firewall-bouncer = {
enable = true;
registerBouncer.enable = true;
settings.mode = "nftables";
};
}
modules/clan/router/default.nix
+67
View File
@@ -0,0 +1,67 @@
# Site gateway (OPNsense replacement) for the Topton 1U boxes, as a clan
# service: PPPoE WAN, VLAN-filtering bridge over the LAN ports, per-VLAN
# DHCP/DNS, firewall/NAT, and the optional Omada controller / internal proxy.
#
# Exported as `clan.modules.router` of this flake; used here with
# `module.input = "self"` and from other clans with `module.input =
# "<this flake's input name>"`. README.md has the consumer view. A site joins
# through the inventory:
#
# inventory.instances.router = {
# module = { name = "router"; input = "self"; };
# roles.default.settings.mesh.subnet = ...; # fleet-wide
# roles.default.machines.gw-<city>-<n>.settings = { site = ...; ... };
# };
#
# The settings schema lives in interface.nix. Implementation files that need
# the settings are functions `{ settings }: <NixOS module>`; the evaluated
# settings are handed in with importApply so nothing goes through
# machine-level options. The rest are plain NixOS modules.
{ lib, ... }:
{
_class = "clan.service";
manifest.name = "router";
manifest.description = "Site gateway: PPPoE WAN, VLAN bridge, DHCP/DNS, firewall/NAT";
manifest.categories = [ "Network" ];
manifest.readme = builtins.readFile ./README.md;
roles.default = {
description = "Turns the machine into the site's router (one instance per machine).";
interface = ./interface.nix;
perInstance =
{ settings, machine, ... }:
{
nixosModule.imports = [
./ipv6.nix
]
# The proxy's TSIG secret is shared with the nameserver (acme-secret.nix).
++ lib.optional settings.proxy.enable (import ./acme-secret.nix machine.name)
++ map (file: lib.modules.importApply file { inherit settings; }) [
./network.nix
./pppoe.nix
./firewall.nix
./dns-dhcp.nix
./crowdsec.nix
./omada.nix
./proxy.nix
./iperf.nix
./speedtest.nix
./wifi.nix
];
};
};
# A machine has exactly one WAN port and one VLAN layout; two instances would
# both claim br0/ppp0 and fight over Kea/Blocky/nftables.
perMachine =
{ instances, machine, ... }:
{
nixosModule.assertions = [
{
assertion = lib.length (lib.attrNames instances) == 1;
message = "router: ${machine.name} is a gateway in several instances (${lib.concatStringsSep ", " (lib.attrNames instances)}); a machine can only be one router.";
}
];
};
}
modules/clan/router/dns-dhcp.nix
+82
View File
@@ -0,0 +1,82 @@
# LAN DHCP (Kea) and DNS (Blocky). Fully declarative: one Kea subnet per VLAN
# with dhcp.enable, Blocky as the blocklist resolver every DHCP lease points
# at. Blocky's HTTP listener (:4000) serves Prometheus metrics, scraped by
# control over the mesh (firewall.nix scopes it to the mesh subnet).
{ settings }:
{ lib, ... }:
let
cfg = settings;
dhcpVlans = lib.filterAttrs (_: vlan: vlan.dhcp.enable) cfg.vlans;
in
{
services.kea.dhcp4 = {
enable = true;
settings = {
interfaces-config.interfaces = lib.mapAttrsToList (name: _: "vlan-${name}") dhcpVlans;
lease-database = {
type = "memfile";
persist = true;
name = "/var/lib/kea/dhcp4.leases";
};
valid-lifetime = 86400;
subnet4 = lib.mapAttrsToList (name: vlan: {
id = vlan.id;
subnet = vlan.subnet;
interface = "vlan-${name}";
valid-lifetime = vlan.dhcp.leaseTime;
pools = [ { pool = "${vlan.dhcp.pool.from} - ${vlan.dhcp.pool.to}"; } ];
reservations = lib.mapAttrsToList (host: res: {
hostname = host;
hw-address = res.hwAddress;
ip-address = res.ipAddress;
}) vlan.dhcp.reservations;
option-data = [
{
name = "routers";
data = vlan.address;
}
{
name = "domain-name-servers";
data = vlan.address;
}
];
}) dhcpVlans;
};
};
services.blocky = {
enable = true;
settings = {
ports = {
dns = 53;
http = 4000;
};
upstreams.groups.default = [
"9.9.9.9"
"149.112.112.112"
"2620:fe::fe"
];
blocking = {
denylists.ads = [
"https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts"
];
clientGroupsBlock.default = [ "ads" ];
};
caching = {
minTime = "5m";
prefetching = true;
};
prometheus.enable = true;
};
};
# The router itself resolves via public resolvers, not via Blocky, so DNS
# for deploys/updates survives a broken local resolver. networkd would
# enable systemd-resolved by default, whose stub listener on 127.0.0.53:53
# makes Blocky's wildcard :53 bind fail — plain resolv.conf instead.
services.resolved.enable = false;
networking.nameservers = [
"9.9.9.9"
"1.1.1.1"
];
}
modules/clan/router/firewall.nix
+65
View File
@@ -0,0 +1,65 @@
# Router firewall/NAT policy (nftables). Trust model:
# mgmt VLAN -> trusted: router services, all VLANs, WAN
# other VLANs -> DNS/DHCP on the router + WAN (if allowWan); no inter-VLAN
# WAN (ppp0) -> nothing inbound beyond established/related
# mesh -> admin SSH + metrics scrapes (same trust boundary as the fleet)
# staging -> admin SSH only (pre-cutover uplink into the old LAN)
{ settings }:
{ lib, ... }:
let
cfg = settings;
vlanIfs = lib.mapAttrsToList (name: _: "vlan-${name}") cfg.vlans;
wanVlanIfs = lib.mapAttrsToList (name: _: "vlan-${name}") (
lib.filterAttrs (_: vlan: vlan.allowWan) cfg.vlans
);
nonMgmtIfs = lib.filter (i: i != "vlan-mgmt") vlanIfs;
in
{
networking.nftables.enable = true;
# SSH reachable only from the mgmt VLAN (trusted) and the mesh — never
# from the WAN or the other VLANs.
services.openssh.openFirewall = false;
networking.firewall = {
enable = true;
filterForward = true;
trustedInterfaces = [ "vlan-mgmt" ];
# Non-mgmt VLANs may only talk to the router's DNS and DHCP; the staging
# uplink (old LAN, pre-cutover) gets admin SSH so the box can be reached
# before the mgmt VLAN or the mesh are up.
interfaces =
lib.genAttrs nonMgmtIfs (_: {
allowedTCPPorts = [ 53 ];
allowedUDPPorts = [
53
67
];
})
// lib.optionalAttrs (cfg.stagingPort != null) {
${cfg.stagingPort}.allowedTCPPorts = [ 22 ];
};
extraInputRules = ''
ip6 saddr ${cfg.mesh.subnet} tcp dport 22 accept comment "admin ssh over the mesh"
ip6 saddr ${cfg.mesh.subnet} tcp dport 4000 accept comment "blocky metrics scrape from control"
'';
extraForwardRules = ''
tcp flags syn tcp option maxseg size set rt mtu comment "MSS clamp for PPPoE mtu 1492"
iifname "vlan-mgmt" accept comment "mgmt reaches all VLANs and the WAN"
'';
};
# networking.nat both masquerades and opens forward-to-WAN for exactly its
# internalInterfaces — so this list, not a rule of our own, is where
# `allowWan` is enforced. Listing every VLAN here would silently let
# allowWan = false VLANs out.
networking.nat = {
enable = true;
externalInterface = "ppp0";
internalInterfaces = wanVlanIfs;
};
}
modules/clan/router/interface.nix
+378
View File
@@ -0,0 +1,378 @@
# Settings of the `router` service (inventory `roles.default.settings`).
# Pure schema: no machine config is reachable here; the implementation files
# get the evaluated result as `settings`.
#
# Fleet addressing convention: each site owns 10.<siteId>.0.0/16. A VLAN's
# subnet defaults to 10.<siteId>.<vlanId>.0/24 with the router at .1 and the
# DHCP pool at .100-.199. VLANs that need more space (e.g. public-wifi guest)
# override `subnet`/`address`/`dhcp.pool` and take a wider block from the
# upper half (10.<siteId>.128.0/17), e.g. guest -> 10.<siteId>.128.0/22.
# VLAN ids: 10 = mgmt, 20 = lan (mandatory); 30 = guest, 40 = iot (reserved).
{ config, lib, ... }:
let
site = toString config.siteId;
vlanModule =
{ config, ... }:
let
octet = toString config.id;
in
{
options = {
id = lib.mkOption {
type = lib.types.ints.between 1 4094;
description = "802.1Q VLAN id (fleet convention: 10 mgmt, 20 lan, 30 guest, 40 iot).";
};
address = lib.mkOption {
type = lib.types.str;
default = "10.${site}.${octet}.1";
defaultText = lib.literalExpression ''"10.<siteId>.<id>.1"'';
description = "Router address on this VLAN.";
};
prefixLength = lib.mkOption {
type = lib.types.ints.between 8 30;
default = 24;
};
subnet = lib.mkOption {
type = lib.types.str;
default = "10.${site}.${octet}.0/24";
defaultText = lib.literalExpression ''"10.<siteId>.<id>.0/24"'';
description = "The VLAN's network in CIDR form (must contain `address`).";
};
dhcp = {
enable = lib.mkOption {
type = lib.types.bool;
default = true;
};
pool = {
from = lib.mkOption {
type = lib.types.str;
default = "10.${site}.${octet}.100";
defaultText = lib.literalExpression ''"10.<siteId>.<id>.100"'';
};
to = lib.mkOption {
type = lib.types.str;
default = "10.${site}.${octet}.199";
defaultText = lib.literalExpression ''"10.<siteId>.<id>.199"'';
};
};
leaseTime = lib.mkOption {
type = lib.types.ints.positive;
default = 86400;
description = ''
Lease validity in seconds. Lower it for high-churn networks,
e.g. public-WiFi guest VLANs (3600-7200), so the pool recycles.
'';
};
reservations = lib.mkOption {
type = lib.types.attrsOf (
lib.types.submodule {
options = {
hwAddress = lib.mkOption {
type = lib.types.str;
example = "aa:bb:cc:dd:ee:ff";
description = "Client MAC address.";
};
ipAddress = lib.mkOption {
type = lib.types.str;
description = "Fixed address handed to this client (inside the VLAN's subnet, outside the pool).";
};
};
}
);
default = { };
description = "Static DHCP leases; the attribute name becomes the client's hostname.";
};
};
allowWan = lib.mkOption {
type = lib.types.bool;
default = true;
description = "Whether clients on this VLAN may reach the internet.";
};
};
};
proxyServiceModule = {
options = {
backend = lib.mkOption {
type = lib.types.str;
example = "https://127.0.0.1:8043";
description = "URL Caddy forwards to (internal/mesh address).";
};
insecureSkipVerify = lib.mkOption {
type = lib.types.bool;
default = false;
description = "Skip TLS verification towards the backend (self-signed upstreams like Omada).";
};
};
};
wifiNetworkModule =
{ name, ... }:
{
options = {
ssid = lib.mkOption {
type = lib.types.str;
default = name;
defaultText = lib.literalExpression "<attribute name>";
description = "SSID as advertised; defaults to the attribute name.";
};
vlan = lib.mkOption {
type = lib.types.str;
example = "lan";
description = "VLAN (by name) the clients of this SSID land in, like an untagged access port.";
};
security = lib.mkOption {
type = lib.types.enum [
"wpa3"
"wpa3-transition"
"wpa2"
"open"
];
default = "wpa3-transition";
description = ''
- `wpa3`: WPA3-Personal (SAE) only.
- `wpa3-transition`: WPA3 with WPA2 fallback for older clients.
- `wpa2`: WPA2-PSK only, for legacy IoT devices.
- `open`: no encryption (captive/guest use; pair with an isolated VLAN).
Except for `open`, the passphrase is a vars prompt
(`wifi-<name>-passphrase`, entered at `clan vars generate`).
'';
};
hidden = lib.mkOption {
type = lib.types.bool;
default = false;
description = "Do not advertise the SSID in beacons (clients must know it).";
};
isolateClients = lib.mkOption {
type = lib.types.bool;
default = false;
description = "Keep wireless clients of this SSID from talking to each other (guest networks).";
};
};
};
wifiRadioModule = {
options = {
band = lib.mkOption {
type = lib.types.enum [
"2g"
"5g"
"6g"
];
default = "2g";
description = "Frequency band of this radio; a dual-band card exposes one radio interface per band.";
};
channel = lib.mkOption {
type = lib.types.ints.unsigned;
default = 0;
description = "Channel; 0 lets hostapd pick one (ACS) — not every driver supports that.";
};
wifi6 = lib.mkOption {
type = lib.types.bool;
default = false;
description = "Enable 802.11ax (WiFi 6) on this radio; WiFi 4/5 are always on.";
};
macAddress = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "aa:bb:cc:dd:ee:ff";
description = ''
Hardware address of the radio (facter.json / `ip link`). Needed when
the radio serves more than one network: hostapd wants a fixed BSSID
per extra network, derived from this address (locally administered
variants of its first octet).
'';
};
networks = lib.mkOption {
type = lib.types.listOf lib.types.str;
example = [
"home"
"iot"
];
description = "Networks (from `wifi.networks`) this radio broadcasts; at most four per radio.";
};
};
};
in
{
options = {
site = lib.mkOption {
type = lib.types.str;
description = "City code of the site, e.g. \"cnx\".";
};
siteId = lib.mkOption {
type = lib.types.ints.between 1 254;
description = "Site number; drives the 10.<siteId>.<vlan>.0/24 addressing.";
};
wan.interface = lib.mkOption {
type = lib.types.str;
description = "Physical WAN port the PPPoE session runs on.";
};
wan.vlanId = lib.mkOption {
type = lib.types.nullOr (lib.types.ints.between 1 4094);
default = null;
description = ''
802.1Q tag the ISP requires for the PPPoE session (AIS Thailand: 10);
null for untagged PPPoE directly on the port. Unrelated to the LAN
VLANs — this tag exists only on the WAN port.
'';
};
wan.macAddress = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "aa:bb:cc:dd:ee:ff";
description = ''
Spoofed MAC for the WAN port, e.g. to keep the MAC the ISP has
pinned (cloned from the old router). null keeps the hardware MAC.
'';
};
trunkPorts = lib.mkOption {
type = lib.types.listOf lib.types.str;
description = "LAN ports carrying all VLANs tagged (incl. any 10G SFP+ ports).";
};
accessPorts = lib.mkOption {
type = lib.types.attrsOf lib.types.str;
default = { };
example = {
enp4s0 = "mgmt";
};
description = ''
Ports acting as untagged access ports on a single VLAN (port name ->
VLAN name). Frames are untagged on the wire; the bridge tags them with
the VLAN's PVID. Use for an always-available on-site mgmt port.
'';
};
stagingPort = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "enp3s0";
description = ''
Temporary DHCPv4-client uplink into the existing LAN while the box
runs alongside the router it replaces: gives it internet + mesh
before the WAN port is cabled (PPPoE simply retries until then). The
port is in no VLAN zone; the firewall admits only SSH on it. Do NOT
connect the trunk ports to the production switch while staging —
Kea on the mgmt tag would fight the old router's DHCP in one
broadcast domain. Set to null at cutover (and usually hand the port
back to `trunkPorts`).
'';
};
vlans = lib.mkOption {
type = lib.types.attrsOf (lib.types.submodule vlanModule);
description = "VLANs served at this site; `mgmt` and `lan` are mandatory.";
};
mesh.subnet = lib.mkOption {
type = lib.types.str;
example = "fd12:3456:789a:bcde:f000::/88";
description = ''
IPv6 prefix of the admin mesh (the ZeroTier overlay; see
modules/mesh-hosts.nix). Admin SSH, metrics scrapes, iperf3 and the
Omada UI accept connections from it, and CrowdSec never bans it.
'';
};
omada.enable = lib.mkEnableOption "TP-Link Omada SDN controller (podman container)";
proxy = {
enable = lib.mkEnableOption "internal reverse proxy (Caddy, wildcard cert via DNS-01)";
domain = lib.mkOption {
type = lib.types.str;
example = "example.net";
description = ''
Parent zone of the proxy names: services are served as
<name>.<site><siteId>.<domain> under a wildcard certificate.
'';
};
acme = {
nameserver = lib.mkOption {
type = lib.types.str;
example = "203.0.113.53";
description = ''
Authoritative nameserver of `domain` that accepts RFC 2136
updates for _acme-challenge.<site><siteId> with this gateway's
TSIG key (acme_<hostname with _>, secret from the shared
dns-acme-<hostname>-secret generator, see acme-secret.nix).
'';
};
email = lib.mkOption {
type = lib.types.str;
example = "postmaster@example.net";
description = "ACME account contact.";
};
};
services = lib.mkOption {
type = lib.types.attrsOf (lib.types.submodule proxyServiceModule);
default = { };
description = "Proxied services; attr name becomes <name>.<site><siteId>.<domain>.";
};
allowVlans = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [
"mgmt"
"lan"
];
description = "VLANs whose clients may reach the proxy (443, plus 80 for the redirect).";
};
};
wifi = {
enable = lib.mkEnableOption "a Wi-Fi access point on the router's own radios (hostapd)";
countryCode = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "TH";
description = "ISO 3166-1 country code for the regulatory domain; required when enabled.";
};
networks = lib.mkOption {
type = lib.types.attrsOf (lib.types.submodule wifiNetworkModule);
default = { };
example = {
home.vlan = "lan";
things = {
vlan = "iot";
security = "wpa2";
};
};
description = ''
Wireless networks (SSIDs). Each one behaves like an untagged access
port on its VLAN; the radios below choose which to broadcast.
'';
};
radios = lib.mkOption {
type = lib.types.attrsOf (lib.types.submodule wifiRadioModule);
default = { };
example = {
wlp5s0 = {
band = "5g";
channel = 36;
networks = [ "home" ];
};
};
description = "Wireless radios of the router (interface name -> config); at least one when enabled.";
};
};
speedtest.interval = lib.mkOption {
type = lib.types.str;
default = "hourly";
description = "systemd OnCalendar spec for the WAN speed test.";
};
};
}
modules/clan/router/iperf.nix
+22
View File
@@ -0,0 +1,22 @@
# iperf3 server on every gateway, for throughput testing from any LAN segment
# (e.g. validating AP/switch links: `iperf3 -c 10.<siteId>.<vlan>.1`) and from
# admin machines over the mesh. Never reachable from the WAN (default-deny).
{ settings }:
{ lib, ... }:
let
cfg = settings;
vlanIfs = lib.mapAttrsToList (name: _: "vlan-${name}") cfg.vlans;
in
{
services.iperf3.enable = true;
networking.firewall.interfaces = lib.genAttrs vlanIfs (_: {
allowedTCPPorts = [ 5201 ];
allowedUDPPorts = [ 5201 ];
});
networking.firewall.extraInputRules = ''
ip6 saddr ${cfg.mesh.subnet} tcp dport 5201 accept comment "iperf3 over the mesh"
ip6 saddr ${cfg.mesh.subnet} udp dport 5201 accept comment "iperf3 over the mesh"
'';
}
modules/clan/router/ipv6.nix
+22
View File
@@ -0,0 +1,22 @@
# IPv6 on the PPPoE uplink: run networkd's DHCPv6 client on ppp0 to obtain a
# delegated prefix; each vlan-* interface (network.nix) carves a /64 out of it
# via DHCPPrefixDelegation and announces it to clients with SLAAC.
{ lib, ... }:
{
systemd.network.networks."45-ppp0" = {
matchConfig.Name = "ppp0";
networkConfig = {
DHCP = "ipv6";
# pppd owns the v4 address/route on this link; don't let networkd
# tear them down.
KeepConfiguration = "static";
# Default v6 route comes from the ISP's RA when they send one.
IPv6AcceptRA = true;
};
# Many PPPoE ISPs never send an RA with the M flag; solicit regardless.
dhcpV6Config.WithoutRA = "solicit";
linkConfig.RequiredForOnline = "no";
};
boot.kernel.sysctl."net.ipv6.conf.all.forwarding" = lib.mkDefault 1;
}
modules/clan/router/network.nix
+170
View File
@@ -0,0 +1,170 @@
# L2/L3 of the gateway: PPPoE WAN port (optionally on an ISP VLAN), a
# VLAN-filtering bridge over the LAN ports, and one L3 interface per VLAN.
{ settings }:
{ lib, pkgs, ... }:
let
cfg = settings;
vlanIf = name: "vlan-${name}";
in
{
assertions = [
{
assertion = cfg.vlans ? mgmt && cfg.vlans ? lan;
message = "router: every site must define the `mgmt` and `lan` VLANs.";
}
{
assertion = lib.all (v: cfg.vlans ? ${v}) (lib.attrValues cfg.accessPorts);
message = "router: every accessPorts value must name a defined VLAN.";
}
{
assertion = lib.all (p: !(cfg.accessPorts ? ${p})) cfg.trunkPorts;
message = "router: a port cannot be both a trunk and an access port.";
}
{
assertion =
cfg.stagingPort == null
|| !(
cfg.stagingPort == cfg.wan.interface
|| lib.elem cfg.stagingPort cfg.trunkPorts
|| cfg.accessPorts ? ${cfg.stagingPort}
);
message = "router: stagingPort ${toString cfg.stagingPort} is also the WAN, a trunk or an access port.";
}
];
# Router diagnostics toolkit: packets (tcpdump), path (mtr), link
# negotiation (ethtool), NAT state (conntrack), DNS (kdig), per-flow
# bandwidth (iftop), WAN throughput (librespeed-cli; iperf3 covers LAN).
environment.systemPackages = with pkgs; [
tcpdump
mtr
ethtool
conntrack-tools
knot-dns
iftop
librespeed-cli
];
networking.useNetworkd = true;
networking.useDHCP = false;
systemd.network.enable = true;
systemd.network.netdevs = {
"20-br0" = {
netdevConfig = {
Name = "br0";
Kind = "bridge";
};
bridgeConfig.VLANFiltering = true;
};
}
// lib.optionalAttrs (cfg.wan.vlanId != null) {
"15-wan-vlan" = {
netdevConfig = {
Name = "wan-vlan";
Kind = "vlan";
};
vlanConfig.Id = cfg.wan.vlanId;
};
}
// lib.mapAttrs' (
name: vlan:
lib.nameValuePair "30-${vlanIf name}" {
netdevConfig = {
Name = vlanIf name;
Kind = "vlan";
};
vlanConfig.Id = vlan.id;
}
) cfg.vlans;
systemd.network.networks =
let
taggedAll = lib.mapAttrsToList (_: vlan: { VLAN = vlan.id; }) cfg.vlans;
in
lib.optionalAttrs (cfg.stagingPort != null) {
# Staging uplink (see interface.nix): plain DHCPv4 client on a spare
# port, no bridge/VLAN membership, so the firewall treats it as untrusted.
"05-staging" = {
matchConfig.Name = cfg.stagingPort;
networkConfig.DHCP = "ipv4";
};
}
// {
# WAN port carries only the PPPoE session; no IP config of its own.
"10-wan" = {
matchConfig.Name = cfg.wan.interface;
networkConfig.LinkLocalAddressing = "no";
vlan = lib.optional (cfg.wan.vlanId != null) "wan-vlan";
linkConfig = {
RequiredForOnline = "carrier";
}
# The wan-vlan subinterface (and thus the PPPoE session) inherits
# the parent port's MAC, so spoofing here covers both cases.
// lib.optionalAttrs (cfg.wan.macAddress != null) {
MACAddress = cfg.wan.macAddress;
};
};
}
// lib.optionalAttrs (cfg.wan.vlanId != null) {
# The ISP-side VLAN subinterface pppd dials on (e.g. AIS tags PPPoE).
"15-wan-vlan" = {
matchConfig.Name = "wan-vlan";
networkConfig.LinkLocalAddressing = "no";
linkConfig.RequiredForOnline = "no";
};
}
// {
# The bridge itself is L2-only; L3 lives on the vlan-* interfaces,
# which hang off the bridge (tagged on the bridge "self" port).
"20-br0" = {
matchConfig.Name = "br0";
networkConfig.LinkLocalAddressing = "no";
vlan = lib.mapAttrsToList (name: _: vlanIf name) cfg.vlans;
bridgeVLANs = taggedAll;
linkConfig.RequiredForOnline = "no";
};
}
// lib.listToAttrs (
map (port: {
name = "25-trunk-${port}";
value = {
matchConfig.Name = port;
networkConfig.Bridge = "br0";
bridgeVLANs = taggedAll;
linkConfig.RequiredForOnline = "no";
};
}) cfg.trunkPorts
)
// lib.mapAttrs' (
port: vlanName:
lib.nameValuePair "25-access-${port}" {
matchConfig.Name = port;
networkConfig.Bridge = "br0";
bridgeVLANs = [
{
VLAN = cfg.vlans.${vlanName}.id;
PVID = cfg.vlans.${vlanName}.id;
EgressUntagged = cfg.vlans.${vlanName}.id;
}
];
linkConfig.RequiredForOnline = "no";
}
) cfg.accessPorts
// lib.mapAttrs' (
name: vlan:
lib.nameValuePair "40-${vlanIf name}" {
matchConfig.Name = vlanIf name;
address = [ "${vlan.address}/${toString vlan.prefixLength}" ];
networkConfig = {
IPv6AcceptRA = false;
# Announce a /64 carved from the DHCPv6-PD prefix on ppp0 (SLAAC).
IPv6SendRA = true;
DHCPPrefixDelegation = true;
};
dhcpPrefixDelegationConfig.SubnetId = "auto";
linkConfig.RequiredForOnline = "no";
}
) cfg.vlans;
}
modules/router/omada.nix → modules/clan/router/omada.nix
+5 -12
View File
@@ -3,20 +3,13 @@
# the de-facto standard image). Host networking because device adoption relies # the de-facto standard image). Host networking because device adoption relies
# on L2 broadcast discovery (UDP 29810) on the mgmt VLAN; the default-deny # on L2 broadcast discovery (UDP 29810) on the mgmt VLAN; the default-deny
# input firewall keeps its ports unreachable from WAN and non-mgmt VLANs. # input firewall keeps its ports unreachable from WAN and non-mgmt VLANs.
{ { settings }:
config, { config, lib, ... }:
lib,
...
}:
let let
cfg = config.cnx.router; cfg = settings;
mesh = import ../mesh-hosts.nix { inherit config lib; };
in in
{ {
options.cnx.router.omada.enable = config = lib.mkIf cfg.omada.enable {
lib.mkEnableOption "TP-Link Omada SDN controller (podman container)";
config = lib.mkIf (cfg.enable && cfg.omada.enable) {
virtualisation.podman.enable = true; virtualisation.podman.enable = true;
virtualisation.oci-containers = { virtualisation.oci-containers = {
backend = "podman"; backend = "podman";
@@ -33,7 +26,7 @@ in
# Admin UI (8043) also reachable over the mesh, like Grafana on control. # Admin UI (8043) also reachable over the mesh, like Grafana on control.
networking.firewall.extraInputRules = '' networking.firewall.extraInputRules = ''
ip6 saddr ${mesh.subnet} tcp dport 8043 accept comment "omada ui over the mesh" ip6 saddr ${cfg.mesh.subnet} tcp dport 8043 accept comment "omada ui over the mesh"
''; '';
# Controller state (adopted devices, site config, cert) — declared as clan # Controller state (adopted devices, site config, cert) — declared as clan
modules/clan/router/pppoe.nix
+61
View File
@@ -0,0 +1,61 @@
# PPPoE WAN session. ISP credentials are entered once at `clan vars generate`
# (prompts). Both are secret — AIS often uses the same string for username and
# password — so neither may land in the Nix store: pppd reads the username from
# an included secret options file and the password from chap/pap-secrets.
{ settings }:
{ config, ... }:
let
cfg = settings;
creds = config.clan.core.vars.generators.pppoe-credentials;
# Interface pppd dials on: the WAN port itself, or its ISP VLAN (network.nix).
pppInterface = if cfg.wan.vlanId == null then cfg.wan.interface else "wan-vlan";
in
{
clan.core.vars.generators.pppoe-credentials = {
prompts.username = {
description = "PPPoE username (from the ISP)";
type = "hidden";
};
prompts.password = {
description = "PPPoE password (from the ISP)";
type = "hidden";
};
files."user-opts".secret = true;
files."chap-secrets".secret = true;
script = ''
user="$(cat "$prompts"/username)"
pass="$(cat "$prompts"/password)"
printf 'user "%s"\n' "$user" > "$out"/user-opts
printf '"%s" * "%s"\n' "$user" "$pass" > "$out"/chap-secrets
'';
};
services.pppd = {
enable = true;
peers.wan = {
autostart = true;
config = ''
plugin pppoe.so ${pppInterface}
ifname ppp0
file ${creds.files."user-opts".path}
noipdefault
defaultroute
noauth
hide-password
persist
maxfail 0
holdoff 5
lcp-echo-interval 15
lcp-echo-failure 3
+ipv6
mtu 1492
mru 1492
'';
};
};
# pppd looks up the password for `user` in these files at dial time; both
# point at the same generated `"<user>" * "<pass>"` line (PAP and CHAP).
environment.etc."ppp/chap-secrets".source = creds.files."chap-secrets".path;
environment.etc."ppp/pap-secrets".source = creds.files."chap-secrets".path;
}
modules/router/proxy.nix → modules/clan/router/proxy.nix
+25 -62
View File
@@ -1,94 +1,57 @@
# Internal reverse proxy for the gateway: Caddy terminates TLS for # Internal reverse proxy for the gateway: Caddy terminates TLS for
# <service>.<site><siteId>.cnx.network (e.g. omada.cnx1.cnx.network) and # <service>.<site><siteId>.<proxy.domain> (e.g. omada.cnx1.cnx.network) and
# forwards to backends by their internal address. The cert is a real Let's # forwards to backends by their internal address. The cert is a real Let's
# Encrypt wildcard (*.<site><siteId>.cnx.network) obtained via ACME DNS-01 # Encrypt wildcard (*.<site><siteId>.<domain>) obtained via ACME DNS-01
# against ns1 with a gateway-scoped TSIG key, so browsers trust it without # against proxy.acme.nameserver with a gateway-scoped TSIG key, so browsers
# any CA install; the names only *resolve* internally — Blocky answers them # trust it without any CA install; the names only *resolve* internally —
# with the router's LAN address, the public zone never carries them. # Blocky answers them with the router's LAN address, the public zone never
# carries them.
# #
# Requires the machine to also import the shared secret generator: # The TSIG secret is the shared dns-acme-<hostname>-secret generator
# (import ../../modules/dns/acme-gw-secret.nix "<hostname>") # (acme-secret.nix, declared here via default.nix); the nameserver machine
# and ns1 to hold the matching key + acl (see machines/ns1/configuration.nix). # must declare the same generator and load the key (this fleet: ns1).
{ { settings }:
config, { config, lib, ... }:
lib,
...
}:
let let
cfg = config.cnx.router; cfg = settings;
hosts = import ../hosts.nix;
hostname = config.networking.hostName; hostname = config.networking.hostName;
tsigKey = "acme_${lib.replaceStrings [ "-" ] [ "_" ] hostname}"; tsigKey = "acme_${lib.replaceStrings [ "-" ] [ "_" ] hostname}";
certName = "${cfg.site}${toString cfg.siteId}.cnx.network"; certName = "${cfg.site}${toString cfg.siteId}.${cfg.proxy.domain}";
nameserver = "${cfg.proxy.acme.nameserver}:53";
serviceModule = {
options = {
backend = lib.mkOption {
type = lib.types.str;
example = "https://127.0.0.1:8043";
description = "URL Caddy forwards to (internal/mesh address).";
};
insecureSkipVerify = lib.mkOption {
type = lib.types.bool;
default = false;
description = "Skip TLS verification towards the backend (self-signed upstreams like Omada).";
};
};
};
in in
{ {
options.cnx.router.proxy = { config = lib.mkIf cfg.proxy.enable {
enable = lib.mkEnableOption "internal reverse proxy (Caddy, wildcard cert via DNS-01)";
services = lib.mkOption {
type = lib.types.attrsOf (lib.types.submodule serviceModule);
default = { };
description = "Proxied services; attr name becomes <name>.${certName}.";
};
allowVlans = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [
"mgmt"
"lan"
];
description = "VLANs whose clients may reach the proxy (443, plus 80 for the redirect).";
};
};
config = lib.mkIf (cfg.enable && cfg.proxy.enable) {
assertions = [ assertions = [
{ {
assertion = lib.all (v: cfg.vlans ? ${v}) cfg.proxy.allowVlans; assertion = lib.all (v: cfg.vlans ? ${v}) cfg.proxy.allowVlans;
message = "cnx.router.proxy.allowVlans must name VLANs defined in cnx.router.vlans."; message = "router: proxy.allowVlans must name VLANs defined in vlans.";
} }
]; ];
# Render the shared per-gateway TSIG secret into a lego rfc2136 env file; # Render the shared per-gateway TSIG secret into a lego rfc2136 env file,
# same pattern as web01 (modules/web-proxy.nix), scoped on ns1 to # scoped on the nameserver to _acme-challenge.<site><siteId> TXT records.
# _acme-challenge.<site><siteId> TXT records only.
clan.core.vars.generators."dns-acme-${hostname}-rfc2136" = { clan.core.vars.generators."dns-acme-${hostname}-rfc2136" = {
files."rfc2136.env".secret = true; # root-owned; systemd reads it as root files."rfc2136.env".secret = true; # root-owned; systemd reads it as root
dependencies = [ "dns-acme-${hostname}-secret" ]; dependencies = [ "dns-acme-${hostname}-secret" ];
script = '' script = ''
printf 'RFC2136_NAMESERVER=${hosts.ns1.ipv4}:53\nRFC2136_TSIG_ALGORITHM=hmac-sha256.\nRFC2136_TSIG_KEY=${tsigKey}\nRFC2136_TSIG_SECRET=%s\n' \ printf 'RFC2136_NAMESERVER=${nameserver}\nRFC2136_TSIG_ALGORITHM=hmac-sha256.\nRFC2136_TSIG_KEY=${tsigKey}\nRFC2136_TSIG_SECRET=%s\n' \
"$(cat "$in"/dns-acme-${hostname}-secret/secret)" > "$out"/rfc2136.env "$(cat "$in"/dns-acme-${hostname}-secret/secret)" > "$out"/rfc2136.env
''; '';
}; };
security.acme = { security.acme = {
acceptTerms = true; acceptTerms = true;
defaults.email = "postmaster@cnx.email"; defaults.email = cfg.proxy.acme.email;
# One wildcard for every proxied service; DNS-01 against ns1, so issuance # One wildcard for every proxied service; DNS-01, so issuance works
# works behind PPPoE with no inbound reachability at all. # behind PPPoE with no inbound reachability at all.
certs.${certName} = { certs.${certName} = {
domain = "*.${certName}"; domain = "*.${certName}";
dnsProvider = "rfc2136"; dnsProvider = "rfc2136";
environmentFile = environmentFile =
config.clan.core.vars.generators."dns-acme-${hostname}-rfc2136".files."rfc2136.env".path; config.clan.core.vars.generators."dns-acme-${hostname}-rfc2136".files."rfc2136.env".path;
# ns1 is the only nameserver that accepts this key's UPDATE; check # Only that nameserver accepts this key's UPDATE; check propagation
# propagation against it directly rather than a public resolver. # against it directly rather than a public resolver.
dnsResolver = "${hosts.ns1.ipv4}:53"; dnsResolver = nameserver;
# Caddy reads the cert from explicit file paths (tls directive below), # Caddy reads the cert from explicit file paths (tls directive below),
# so it won't notice a renewal on its own. # so it won't notice a renewal on its own.
reloadServices = [ "caddy.service" ]; reloadServices = [ "caddy.service" ];
modules/clan/router/speedtest.nix
+55
View File
@@ -0,0 +1,55 @@
# Periodic WAN speed test so ISP degradation shows up as a trend instead of a
# complaint. A timer runs librespeed-cli and writes the results as Prometheus
# metrics into node_exporter's textfile collector — they ride the existing
# 9100 scrape to VictoriaMetrics, where alerts.nix compares each run against
# the link's own 7-day median (no per-site threshold to maintain).
{ settings }:
{ pkgs, ... }:
let
cfg = settings;
textfileDir = "/var/lib/speedtest";
in
{
services.prometheus.exporters.node.extraFlags = [
"--collector.textfile.directory=${textfileDir}"
];
systemd.services.speedtest = {
description = "WAN speed test to Prometheus textfile metrics";
after = [ "network-online.target" ];
wants = [ "network-online.target" ];
path = [
pkgs.librespeed-cli
pkgs.jq
];
serviceConfig = {
Type = "oneshot";
StateDirectory = "speedtest";
# One test at boot would race PPPoE and log a spurious failure.
ExecCondition = "${pkgs.iproute2}/bin/ip link show ppp0";
};
script = ''
tmp="${textfileDir}/.speedtest.prom.tmp"
if result=$(librespeed-cli --json); then
jq -r '.[0]
| "speedtest_download_mbps \(.download)",
"speedtest_upload_mbps \(.upload)",
"speedtest_ping_ms \(.ping)",
"speedtest_jitter_ms \(.jitter)",
"speedtest_success 1"' <<<"$result" > "$tmp"
else
echo "speedtest_success 0" > "$tmp"
fi
mv "$tmp" "${textfileDir}/speedtest.prom"
'';
};
systemd.timers.speedtest = {
wantedBy = [ "timers.target" ];
timerConfig = {
OnCalendar = cfg.speedtest.interval;
RandomizedDelaySec = "10m";
Persistent = true;
};
};
}
modules/clan/router/tests/vm/default.nix
+306
View File
@@ -0,0 +1,306 @@
# End-to-end VM test of the router service: a PPPoE access concentrator plays
# the ISP on the WAN port, a trunk carries tagged lan/iot VLANs to `client`,
# and an untagged access port carries mgmt to `admin`.
#
# isp ---(vlan 1: PPPoE)--- wan [gw] trunk ---(vlan 2: tagged 20/40)--- client
# access --(vlan 3: untagged mgmt)--- admin
#
# What is proven: PPPoE dial-in with the vars-provided credentials, bridge
# VLAN tagging/untagging, Kea leases and reservations per VLAN, Blocky
# answering on the VLAN with the blocklist active, NAT to the WAN, and the
# firewall trust model (allowWan, mgmt-only SSH, no inter-VLAN forwarding).
{ pkgs, lib, ... }:
let
# The vars mock answers every prompt with "mock-prompt-value-<name>"; the
# ISP side must accept exactly those.
chapSecrets = ''"mock-prompt-value-username" * "mock-prompt-value-password" *'';
ispAddress = "192.0.2.1";
clientMac = "02:00:00:00:00:20";
clientAddress = "10.9.20.50";
adminMac = "02:00:00:00:00:10";
adminAddress = "10.9.10.50";
in
{
name = "router";
clan = {
directory = ./.;
# Bridges, VLAN netdevs, PPPoE and nftables need a real kernel.
test.useContainers = false;
inventory = {
# Every node is a clan machine (the test framework's defaults require
# it); only gw gets the router role.
machines = {
gw = { };
isp = { };
client = { };
admin = { };
};
instances.router = {
module.name = "router";
module.input = "self";
roles.default.machines.gw.settings = {
site = "tst";
siteId = 9;
mesh.subnet = "fd00:7e57:c1a1:c0de::/64";
wan.interface = "wan";
trunkPorts = [ "trunk" ];
accessPorts.access = "mgmt";
vlans = {
mgmt = {
id = 10;
dhcp.reservations.admin = {
hwAddress = adminMac;
ipAddress = adminAddress;
};
};
lan = {
id = 20;
dhcp.reservations.client = {
hwAddress = clientMac;
ipAddress = clientAddress;
};
};
iot = {
id = 40;
allowWan = false;
};
};
# Access point on the simulated radio wlan0 (mac80211_hwsim, address
# 02:00:00:00:00:00): two SSIDs, one per VLAN.
wifi = {
enable = true;
countryCode = "US";
networks = {
home.vlan = "lan";
things = {
vlan = "iot";
security = "wpa2";
hidden = true;
isolateClients = true;
};
};
radios.wlan0 = {
channel = 6;
macAddress = "02:00:00:00:00:00";
networks = [
"home"
"things"
];
};
};
};
};
};
};
nodes = {
gw = {
virtualisation.interfaces = {
wan = {
vlan = 1;
assignIP = false;
};
trunk = {
vlan = 2;
assignIP = false;
};
access = {
vlan = 3;
assignIP = false;
};
};
# Something must listen on 22 for the mgmt-only SSH rule to be observable
# (a refused and a dropped connection look the same to the client).
services.openssh.enable = true;
# The sandbox has no internet: serve the blocklist from a local file
# instead of GitHub, and skip CrowdSec, whose hub sync needs the network
# (it is not what this test exercises).
services.blocky.settings.blocking.denylists.ads = lib.mkForce [
(toString (pkgs.writeText "ads.hosts" "0.0.0.0 ads.example.com\n"))
];
services.crowdsec.enable = lib.mkForce false;
services.crowdsec-firewall-bouncer.enable = lib.mkForce false;
# Two simulated radios: wlan0 is the AP (settings above), wlan1 plays a
# wireless client. It lives in its own network namespace, like the
# separate host it stands in for — otherwise its lease would add a
# second 10.9.20.0/24 route to the router's own table. Its DHCP lease
# must come from Kea on the SSID's VLAN: wlan1 -> air -> wlan0 -> br0 ->
# vlan-lan. The mock passphrase is what the vars mock fed into the
# wifi-home-passphrase generator.
boot.kernelModules = [ "mac80211_hwsim" ];
systemd.services.wifi-station = {
wantedBy = [ "multi-user.target" ];
# No BindsTo: the device unit vanishes once wlan1 moves into the netns.
after = [ "sys-subsystem-net-devices-wlan1.device" ];
path = [
pkgs.iproute2
pkgs.iw
];
preStart = ''
ip netns add sta
iw phy phy1 set netns name sta
ip netns exec sta ip link set lo up
mkdir -p /run/wpa_supplicant/client # nixpkgs' wpa_cli keeps its sockets here
'';
serviceConfig.ExecStart =
"${pkgs.iproute2}/bin/ip netns exec sta ${pkgs.wpa_supplicant}/bin/wpa_supplicant -i wlan1 -c "
+ pkgs.writeText "sta.conf" ''
ctrl_interface=/run/wpa_supplicant/control
network={
ssid="home"
key_mgmt=SAE
sae_password="mock-prompt-value-passphrase"
ieee80211w=2
scan_freq=2437
}
'';
};
environment.systemPackages = [
pkgs.wpa_supplicant
(pkgs.writeShellScriptBin "sta-dhcp" ''
# One DHCP round on the station, applying the offered address.
exec ${pkgs.iproute2}/bin/ip netns exec sta ${pkgs.busybox}/bin/udhcpc -i wlan1 -n -q -f \
-s ${pkgs.writeShellScript "udhcpc-apply" ''
case "$1" in
bound|renew) ${pkgs.iproute2}/bin/ip addr replace "$ip/$mask" dev "$interface" ;;
esac
''}
'')
];
};
isp = {
virtualisation.interfaces.wan = {
vlan = 1;
assignIP = false;
};
# PPPoE access concentrator: one session, peer gets 192.0.2.10.
systemd.services.pppoe-server = {
wantedBy = [ "multi-user.target" ];
after = [ "network.target" ];
serviceConfig.ExecStart =
"${pkgs.rp-pppoe}/sbin/pppoe-server -F -O /etc/ppp/pppoe-server-options"
+ " -q ${pkgs.ppp}/sbin/pppd -I wan -L ${ispAddress} -R 192.0.2.10";
};
environment.etc = {
"ppp/pppoe-server-options".text = ''
plugin pppoe.so
require-chap
lcp-echo-interval 10
lcp-echo-failure 2
nobsdcomp
noccp
novj
'';
"ppp/chap-secrets" = {
text = chapSecrets;
mode = "0640";
};
};
};
client = {
virtualisation.interfaces.trunk = {
vlan = 2;
assignIP = false;
};
networking.useDHCP = false;
networking.vlans = {
lan0 = {
id = 20;
interface = "trunk";
};
iot0 = {
id = 40;
interface = "trunk";
};
};
networking.interfaces.lan0 = {
useDHCP = true;
macAddress = clientMac;
};
networking.interfaces.iot0.useDHCP = true;
# Dual-homed on purpose (lan + iot). Both leases bring a default route:
# prefer lan0 so replies to other VLANs and the WAN leave where the
# router expects them (its rp-filter would drop them on vlan-iot), and
# loosen this client's own rp-filter so a WAN reply on iot0 would be
# seen — the negative allowWan check must fail at the router, not here.
systemd.network.networks."40-iot0".dhcpV4Config.RouteMetric = 2048;
networking.firewall.checkReversePath = "loose";
environment.systemPackages = [
pkgs.dnsutils
pkgs.netcat
];
};
admin = {
virtualisation.interfaces.access = {
vlan = 3;
assignIP = false;
};
networking.useDHCP = false;
networking.interfaces.access = {
useDHCP = true;
macAddress = adminMac;
};
environment.systemPackages = [ pkgs.netcat ];
};
};
testScript = ''
start_all()
with subtest("PPPoE session comes up with the vars credentials"):
gw.wait_for_unit("pppd-wan.service")
gw.wait_until_succeeds("ping -c1 -W1 ${ispAddress}")
with subtest("DHCP hands out reserved leases per VLAN"):
gw.wait_for_unit("kea-dhcp4-server.service")
client.wait_until_succeeds("ip -4 addr show lan0 | grep -q 'inet ${clientAddress}/24'")
client.wait_until_succeeds("ip -4 addr show iot0 | grep -q 'inet 10.9.40.1[0-9][0-9]/24'")
admin.wait_until_succeeds("ip -4 addr show access | grep -q 'inet ${adminAddress}/24'")
with subtest("Blocky serves the VLAN and blocks the denylist"):
gw.wait_for_unit("blocky.service")
answer = client.wait_until_succeeds("dig +short +time=2 @10.9.20.1 ads.example.com")
assert answer.strip() == "0.0.0.0", f"expected blocked answer, got: {answer!r}"
with subtest("NAT to the WAN only for VLANs with allowWan"):
client.wait_until_succeeds("ping -c1 -W1 -I lan0 ${ispAddress}")
# iot has a route to the WAN; the router is what refuses to forward.
client.succeed("ip route show dev iot0 | grep -q ^default")
client.fail("ping -c1 -W2 -I iot0 ${ispAddress}")
with subtest("mgmt reaches other VLANs, other VLANs do not"):
admin.succeed("ping -c1 -W2 ${clientAddress}")
client.fail("ping -c1 -W2 -I lan0 ${adminAddress}")
with subtest("SSH on the router only from mgmt"):
gw.wait_for_open_port(22)
admin.succeed("nc -z -w2 10.9.10.1 22")
client.fail("nc -z -w2 10.9.20.1 22")
with subtest("Wireless SSIDs are bridge access ports of their VLAN"):
gw.wait_for_unit("hostapd.service")
# Second SSID: derived BSSID, hidden, bridged into the iot VLAN.
gw.wait_until_succeeds("ip link show wlan0-1 | grep -q '06:00:00:00:00:00'")
gw.wait_until_succeeds("bridge vlan show dev wlan0-1 | grep -q '40 PVID Egress Untagged'")
gw.succeed("hostapd_cli -i wlan0-1 get_config | grep -q '^ssid=things'")
# The second BSS section of the rendered config carries hidden + isolation.
things = gw.succeed("sed -n '/^bss=wlan0-1/,$p' /run/hostapd/wlan0.hostapd.conf")
assert "ignore_broadcast_ssid=1" in things and "ap_isolate=1" in things, things
assert "sae_password=mock-prompt-value-passphrase" not in things, "wpa2 SSID must not carry SAE entries"
# A WPA3 (SAE) station on the first SSID authenticates with the vars
# passphrase and gets its lease from Kea on the lan VLAN, through the bridge.
gw.wait_for_unit("wifi-station.service")
gw.wait_until_succeeds("ip netns exec sta wpa_cli -i wlan1 status | grep -q wpa_state=COMPLETED")
gw.succeed("timeout 60 sta-dhcp")
gw.succeed("ip netns exec sta ip -4 addr show wlan1 | grep -q 'inet 10.9.20.1[0-9][0-9]/24'")
'';
}
modules/clan/router/wifi.nix
+206
View File
@@ -0,0 +1,206 @@
# Wireless access point on the router itself (hostapd). Every SSID is a BSS
# interface that joins the VLAN bridge as an untagged access port of its VLAN,
# so wireless clients get exactly the same DHCP/DNS/firewall treatment as a
# wired port in that VLAN. Passphrases are vars prompts, never in the store.
#
# hostapd names BSS interfaces <radio>, <radio>-1, <radio>-2 ... and wants a
# fixed BSSID for every extra one; they are derived from the radio's hardware
# address by setting the locally-administered bit and flipping bits 2-3 of
# the first octet per index (02 -> 06, 0a, 0e), which never collides with the
# radio's own address.
{ settings }:
{
config,
lib,
pkgs,
...
}:
let
cfg = settings;
wifi = cfg.wifi;
bssIf = radio: i: if i == 0 then radio else "${radio}-${toString i}";
hexDigit =
c:
{
"0" = 0;
"1" = 1;
"2" = 2;
"3" = 3;
"4" = 4;
"5" = 5;
"6" = 6;
"7" = 7;
"8" = 8;
"9" = 9;
a = 10;
b = 11;
c = 12;
d = 13;
e = 14;
f = 15;
}
.${lib.toLower c};
octetToInt = s: 16 * hexDigit (builtins.substring 0 1 s) + hexDigit (builtins.substring 1 1 s);
intToOctet = n: lib.toLower (lib.fixedWidthString 2 "0" (lib.toHexString n));
deriveBssid =
mac: i:
let
octets = lib.splitString ":" mac;
first = builtins.bitXor (builtins.bitOr (octetToInt (builtins.head octets)) 2) (i * 4);
in
lib.concatStringsSep ":" ([ (intToOctet first) ] ++ builtins.tail octets);
# Every BSS of every radio, flattened.
bsses = lib.concatLists (
lib.mapAttrsToList (
radio: r:
lib.imap0 (i: name: {
inherit radio name;
index = i;
iface = bssIf radio i;
net = wifi.networks.${name} or null;
}) r.networks
) wifi.radios
);
referenced = lib.unique (map (b: b.name) bsses);
secured = lib.filter (name: wifi.networks.${name}.security != "open") (
lib.filter (name: wifi.networks ? ${name}) referenced
);
passphraseFile =
name: config.clan.core.vars.generators."wifi-${name}-passphrase".files.passphrase.path;
authentication =
name: net:
{
wpa3 = {
mode = "wpa3-sae";
saePasswordsFile = passphraseFile name;
};
wpa3-transition = {
mode = "wpa3-sae-transition";
saePasswordsFile = passphraseFile name;
wpaPasswordFile = passphraseFile name;
};
wpa2 = {
mode = "wpa2-sha1";
wpaPasswordFile = passphraseFile name;
};
open.mode = "none";
}
.${net.security};
in
{
config = lib.mkIf wifi.enable {
assertions = [
{
assertion = wifi.radios != { };
message = "router: wifi.enable needs at least one radio in wifi.radios.";
}
{
assertion = wifi.countryCode != null;
message = "router: wifi.countryCode is required when wifi.enable is set (regulatory domain).";
}
{
assertion = lib.all (b: b.net != null) bsses;
message = "router: every wifi.radios.<radio>.networks entry must name a network in wifi.networks.";
}
{
assertion = lib.all (b: b.net == null || cfg.vlans ? ${b.net.vlan}) bsses;
message = "router: every wifi.networks.<name>.vlan must name a VLAN in vlans.";
}
{
assertion = lib.all (r: lib.length r.networks <= 4) (lib.attrValues wifi.radios);
message = "router: a radio can broadcast at most four networks.";
}
{
assertion = lib.all (r: lib.length r.networks <= 1 || r.macAddress != null) (
lib.attrValues wifi.radios
);
message = "router: wifi.radios.<radio>.macAddress is required for radios broadcasting more than one network.";
}
];
# Regulatory database for the kernel, so countryCode actually applies.
hardware.wirelessRegulatoryDatabase = true;
# The kernel refuses to bridge a wireless interface in station mode, and
# networkd stops retrying before hostapd switches the radio to AP mode;
# so put it in AP mode the moment it appears (kernel name or the renamed
# one, whichever the user configured). hostapd finds it already there.
services.udev.extraRules = lib.concatMapStrings (
radio:
let
run = ''RUN+="${pkgs.iw}/bin/iw dev ${radio} set type __ap"'';
in
''
ACTION=="add", SUBSYSTEM=="net", KERNEL=="${radio}", ${run}
ACTION=="add", SUBSYSTEM=="net", NAME=="${radio}", ${run}
''
) (lib.attrNames wifi.radios);
clan.core.vars.generators = lib.genAttrs (map (name: "wifi-${name}-passphrase") secured) (
gen:
let
name = lib.removeSuffix "-passphrase" (lib.removePrefix "wifi-" gen);
in
{
prompts.passphrase = {
description = "Wi-Fi passphrase for SSID \"${wifi.networks.${name}.ssid}\" (8-63 ASCII characters)";
type = "hidden";
};
files.passphrase.secret = true;
# No trailing newline: hostapd turns every line of the file into a
# sae_password entry, and an empty one wipes the list.
script = ''printf '%s' "$(cat "$prompts"/passphrase)" > "$out"/passphrase'';
}
);
services.hostapd = {
enable = true;
radios = lib.mapAttrs (radio: r: {
inherit (r) band channel;
inherit (wifi) countryCode;
wifi6.enable = r.wifi6;
networks = lib.listToAttrs (
map (
b:
lib.nameValuePair b.iface (
{
inherit (b.net) ssid;
ignoreBroadcastSsid = if b.net.hidden then "empty" else "disabled";
apIsolate = b.net.isolateClients;
authentication = authentication b.name b.net;
}
// lib.optionalAttrs (lib.length r.networks > 1) {
bssid = if b.index == 0 then r.macAddress else deriveBssid r.macAddress b.index;
}
)
) (lib.filter (b: b.radio == radio) bsses)
);
}) wifi.radios;
};
# Each BSS is an untagged access port of its VLAN on br0 (cf. accessPorts
# in network.nix); networkd enslaves the interface once hostapd creates it.
systemd.network.networks = lib.listToAttrs (
map (
b:
lib.nameValuePair "27-wifi-${b.iface}" {
matchConfig.Name = b.iface;
networkConfig.Bridge = "br0";
bridgeVLANs = [
{
VLAN = cfg.vlans.${b.net.vlan}.id;
PVID = cfg.vlans.${b.net.vlan}.id;
EgressUntagged = cfg.vlans.${b.net.vlan}.id;
}
];
linkConfig.RequiredForOnline = "no";
}
) bsses
);
};
}
modules/dns/acme-gw-secret.nix
-17
View File
@@ -1,17 +0,0 @@
# Shared TSIG secret for a gateway's dedicated ACME key (function: machine
# name -> module). The acme_gw_<x> key lets that gateway — and only it — write
# _acme-challenge.<label> TXT records on ns1 to obtain its internal wildcard
# cert via DNS-01. ns1 scopes it with a matching acl on the cnx.network zone.
# Import on BOTH ns1 and the gateway machine, applied with the machine name:
# (import ../../modules/dns/acme-gw-secret.nix "gw-cnx-1")
machine:
{ pkgs, ... }:
{
clan.core.vars.generators."dns-acme-${machine}-secret" = {
share = true;
files."secret".secret = true;
runtimeInputs = [ pkgs.openssl ];
# 32 random bytes, base64 — a valid hmac-sha256 TSIG secret.
script = ''openssl rand -base64 32 | tr -d '\n' > "$out"/secret'';
};
}
modules/dns/authoritative.nix
+4 -1
View File
@@ -6,7 +6,10 @@
}: }:
let let
# ZeroTier addresses — zone transfers run over the mesh, not the public net. # ZeroTier addresses — zone transfers run over the mesh, not the public net.
mesh = import ../mesh-hosts.nix { inherit config lib; }; mesh = import ../mesh-hosts.nix {
dir = config.clan.core.settings.directory;
inherit lib;
};
ns1zt = mesh.hosts.ns1; ns1zt = mesh.hosts.ns1;
ns2zt = mesh.hosts.ns2; ns2zt = mesh.hosts.ns2;
in in
modules/docs.nix
+4 -1
View File
@@ -8,7 +8,10 @@
... ...
}: }:
let let
mesh = import ./mesh-hosts.nix { inherit config lib; }; mesh = import ./mesh-hosts.nix {
dir = config.clan.core.settings.directory;
inherit lib;
};
port = 8080; port = 8080;
site = pkgs.stdenvNoCC.mkDerivation { site = pkgs.stdenvNoCC.mkDerivation {
modules/mesh-hosts.nix
+3 -3
View File
@@ -8,10 +8,10 @@
# vars/shared/zerotier-network-<instance>/network-id (instance = "zerotier", # vars/shared/zerotier-network-<instance>/network-id (instance = "zerotier",
# the inventory.instances.zerotier name in clan.nix). This keeps the mesh map in # the inventory.instances.zerotier name in clan.nix). This keeps the mesh map in
# lockstep with the actual identities: regenerate or re-key a node and its # lockstep with the actual identities: regenerate or re-key a node and its
# address here follows automatically. Call as: import ../mesh-hosts.nix { inherit config lib; }. # address here follows automatically. `dir` is the clan directory: from NixOS
{ config, lib }: # `config.clan.core.settings.directory`, from clan.nix `config.directory`.
{ dir, lib }:
let let
dir = config.clan.core.settings.directory;
instance = "zerotier"; instance = "zerotier";
readIp = readIp =
modules/monitoring/alerts.nix
+1 -1
View File
@@ -61,7 +61,7 @@ in
} }
{ {
# WAN quality on the site gateways, fed by the hourly speedtest timer # WAN quality on the site gateways, fed by the hourly speedtest timer
# (modules/router/speedtest.nix) via the node_exporter textfile # (modules/clan/router/speedtest.nix) via the node_exporter textfile
# collector. Degradation is judged against the link's own 7-day median, # collector. Degradation is judged against the link's own 7-day median,
# so no per-site/per-ISP-plan threshold needs maintaining. # so no per-site/per-ISP-plan threshold needs maintaining.
name = "wan"; name = "wan";
modules/monitoring/exporters.nix
+4 -1
View File
@@ -10,7 +10,10 @@
... ...
}: }:
let let
mesh = import ../mesh-hosts.nix { inherit config lib; }; mesh = import ../mesh-hosts.nix {
dir = config.clan.core.settings.directory;
inherit lib;
};
knotEnabled = config.services.knot.enable; knotEnabled = config.services.knot.enable;
# node_exporter on every host; knot-exporter only where Knot runs. # node_exporter on every host; knot-exporter only where Knot runs.
ports = [ 9100 ] ++ lib.optional knotEnabled 9433; ports = [ 9100 ] ++ lib.optional knotEnabled 9433;
modules/monitoring/parsedmarc.nix
+4 -1
View File
@@ -16,7 +16,10 @@
... ...
}: }:
let let
mesh = import ../mesh-hosts.nix { inherit config lib; }; mesh = import ../mesh-hosts.nix {
dir = config.clan.core.settings.directory;
inherit lib;
};
# Grafana 13 (since the externalisation of core data sources) no longer bundles # Grafana 13 (since the externalisation of core data sources) no longer bundles
# the Elasticsearch data source, and nixpkgs has no grafanaPlugins entry for it, # the Elasticsearch data source, and nixpkgs has no grafanaPlugins entry for it,
modules/monitoring/server.nix
+4 -1
View File
@@ -9,7 +9,10 @@
... ...
}: }:
let let
mesh = import ../mesh-hosts.nix { inherit config lib; }; mesh = import ../mesh-hosts.nix {
dir = config.clan.core.settings.directory;
inherit lib;
};
probes = import ./blackbox-probes.nix { inherit lib; }; probes = import ./blackbox-probes.nix { inherit lib; };
vmPort = 8428; vmPort = 8428;
logsPort = 9428; logsPort = 9428;
modules/router/crowdsec.nix
-52
View File
@@ -1,52 +0,0 @@
# CrowdSec security engine + nftables bouncer: parses sshd auth attempts from
# the journal and bans offending source IPs at the firewall. Log-based (no
# inline DPI) so it costs the N300 next to nothing.
{
config,
lib,
...
}:
let
cfg = config.cnx.router;
mesh = import ../mesh-hosts.nix { inherit config lib; };
in
{
config = lib.mkIf cfg.enable {
services.crowdsec = {
enable = true;
autoUpdateService = true;
hub.collections = [
"crowdsecurity/linux"
"crowdsecurity/sshd"
];
localConfig = {
acquisitions = [
{
source = "journalctl";
journalctl_filter = [ "_SYSTEMD_UNIT=sshd.service" ];
labels.type = "syslog";
}
];
# Never ban the ZeroTier mesh — it is the only admin path to these
# boxes (no public SSH), so a false positive would lock us out.
# Parser-stage whitelist: mesh events are dropped before any scenario.
parsers.s02Enrich = [
{
name = "cnx/mesh-whitelist";
description = "Whitelist the ZeroTier management mesh";
whitelist = {
reason = "ZeroTier mesh is the admin path";
cidr = [ mesh.subnet ];
};
}
];
};
};
services.crowdsec-firewall-bouncer = {
enable = true;
registerBouncer.enable = true;
settings.mode = "nftables";
};
};
}
modules/router/default.nix
-325
View File
@@ -1,325 +0,0 @@
# Site gateway (OPNsense replacement) for the Topton 1U boxes: PPPoE WAN,
# VLAN-filtering bridge over the LAN ports, and per-VLAN L3 interfaces.
# Imported by machines/gw-<city>-<n>; everything is driven by cnx.router.*.
#
# Fleet addressing convention: each site owns 10.<siteId>.0.0/16. A VLAN's
# subnet defaults to 10.<siteId>.<vlanId>.0/24 with the router at .1 and the
# DHCP pool at .100-.199. VLANs that need more space (e.g. public-wifi guest)
# override `subnet`/`address`/`dhcp.pool` and take a wider block from the
# upper half (10.<siteId>.128.0/17), e.g. guest -> 10.<siteId>.128.0/22.
# VLAN ids: 10 = mgmt, 20 = lan (mandatory); 30 = guest, 40 = iot (reserved).
{
config,
lib,
pkgs,
...
}:
let
cfg = config.cnx.router;
vlanIf = name: "vlan-${name}";
site = toString cfg.siteId;
vlanModule =
{ name, config, ... }:
let
octet = toString config.id;
in
{
options = {
id = lib.mkOption {
type = lib.types.ints.between 1 4094;
description = "802.1Q VLAN id (fleet convention: 10 mgmt, 20 lan, 30 guest, 40 iot).";
};
address = lib.mkOption {
type = lib.types.str;
default = "10.${site}.${octet}.1";
description = "Router address on this VLAN.";
};
prefixLength = lib.mkOption {
type = lib.types.ints.between 8 30;
default = 24;
};
subnet = lib.mkOption {
type = lib.types.str;
default = "10.${site}.${octet}.0/24";
description = "The VLAN's network in CIDR form (must contain `address`).";
};
dhcp = {
enable = lib.mkOption {
type = lib.types.bool;
default = true;
};
pool = {
from = lib.mkOption {
type = lib.types.str;
default = "10.${site}.${octet}.100";
};
to = lib.mkOption {
type = lib.types.str;
default = "10.${site}.${octet}.199";
};
};
leaseTime = lib.mkOption {
type = lib.types.ints.positive;
default = 86400;
description = ''
Lease validity in seconds. Lower it for high-churn networks,
e.g. public-WiFi guest VLANs (3600-7200), so the pool recycles.
'';
};
reservations = lib.mkOption {
type = lib.types.attrsOf (
lib.types.submodule {
options = {
hwAddress = lib.mkOption {
type = lib.types.str;
example = "aa:bb:cc:dd:ee:ff";
description = "Client MAC address.";
};
ipAddress = lib.mkOption {
type = lib.types.str;
description = "Fixed address handed to this client (inside the VLAN's subnet, outside the pool).";
};
};
}
);
default = { };
description = "Static DHCP leases; the attribute name becomes the client's hostname.";
};
};
allowWan = lib.mkOption {
type = lib.types.bool;
default = true;
description = "Whether clients on this VLAN may reach the internet.";
};
};
};
in
{
imports = [
./pppoe.nix
./ipv6.nix
./firewall.nix
./dns-dhcp.nix
./crowdsec.nix
./omada.nix
./proxy.nix
./iperf.nix
./speedtest.nix
];
options.cnx.router = {
enable = lib.mkEnableOption "site gateway (router) role";
site = lib.mkOption {
type = lib.types.str;
description = "City code of the site, e.g. \"cnx\".";
};
siteId = lib.mkOption {
type = lib.types.ints.between 1 254;
description = "Site number; drives the 10.<siteId>.<vlan>.0/24 addressing.";
};
wan.interface = lib.mkOption {
type = lib.types.str;
description = "Physical WAN port the PPPoE session runs on.";
};
wan.vlanId = lib.mkOption {
type = lib.types.nullOr (lib.types.ints.between 1 4094);
default = null;
description = ''
802.1Q tag the ISP requires for the PPPoE session (AIS Thailand: 10);
null for untagged PPPoE directly on the port. Unrelated to the LAN
VLANs — this tag exists only on the WAN port.
'';
};
wan.macAddress = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "aa:bb:cc:dd:ee:ff";
description = ''
Spoofed MAC for the WAN port, e.g. to keep the MAC the ISP has
pinned (cloned from the old router). null keeps the hardware MAC.
'';
};
wan.pppInterface = lib.mkOption {
type = lib.types.str;
internal = true;
readOnly = true;
default = if cfg.wan.vlanId == null then cfg.wan.interface else "wan-vlan";
description = "Interface pppd dials on (the WAN port or its ISP VLAN).";
};
trunkPorts = lib.mkOption {
type = lib.types.listOf lib.types.str;
description = "LAN ports carrying all VLANs tagged (incl. any 10G SFP+ ports).";
};
accessPorts = lib.mkOption {
type = lib.types.attrsOf lib.types.str;
default = { };
example = {
enp4s0 = "mgmt";
};
description = ''
Ports acting as untagged access ports on a single VLAN (port name ->
VLAN name). Frames are untagged on the wire; the bridge tags them with
the VLAN's PVID. Use for an always-available on-site mgmt port.
'';
};
vlans = lib.mkOption {
type = lib.types.attrsOf (lib.types.submodule vlanModule);
description = "VLANs served at this site; `mgmt` and `lan` are mandatory.";
};
};
config = lib.mkIf cfg.enable {
assertions = [
{
assertion = cfg.vlans ? mgmt && cfg.vlans ? lan;
message = "cnx.router: every site must define the `mgmt` and `lan` VLANs.";
}
{
assertion = lib.all (v: cfg.vlans ? ${v}) (lib.attrValues cfg.accessPorts);
message = "cnx.router: every accessPorts value must name a defined VLAN.";
}
{
assertion = lib.all (p: !(cfg.accessPorts ? ${p})) cfg.trunkPorts;
message = "cnx.router: a port cannot be both a trunk and an access port.";
}
];
# Router diagnostics toolkit: packets (tcpdump), path (mtr), link
# negotiation (ethtool), NAT state (conntrack), DNS (kdig), per-flow
# bandwidth (iftop), WAN throughput (librespeed-cli; iperf3 covers LAN).
environment.systemPackages = with pkgs; [
tcpdump
mtr
ethtool
conntrack-tools
knot-dns
iftop
librespeed-cli
];
networking.useNetworkd = true;
networking.useDHCP = false;
systemd.network.enable = true;
systemd.network.netdevs = {
"20-br0" = {
netdevConfig = {
Name = "br0";
Kind = "bridge";
};
bridgeConfig.VLANFiltering = true;
};
}
// lib.optionalAttrs (cfg.wan.vlanId != null) {
"15-wan-vlan" = {
netdevConfig = {
Name = "wan-vlan";
Kind = "vlan";
};
vlanConfig.Id = cfg.wan.vlanId;
};
}
// lib.mapAttrs' (
name: vlan:
lib.nameValuePair "30-${vlanIf name}" {
netdevConfig = {
Name = vlanIf name;
Kind = "vlan";
};
vlanConfig.Id = vlan.id;
}
) cfg.vlans;
systemd.network.networks =
let
taggedAll = lib.mapAttrsToList (_: vlan: { VLAN = vlan.id; }) cfg.vlans;
in
{
# WAN port carries only the PPPoE session; no IP config of its own.
"10-wan" = {
matchConfig.Name = cfg.wan.interface;
networkConfig.LinkLocalAddressing = "no";
vlan = lib.optional (cfg.wan.vlanId != null) "wan-vlan";
linkConfig = {
RequiredForOnline = "carrier";
}
# The wan-vlan subinterface (and thus the PPPoE session) inherits
# the parent port's MAC, so spoofing here covers both cases.
// lib.optionalAttrs (cfg.wan.macAddress != null) {
MACAddress = cfg.wan.macAddress;
};
};
}
// lib.optionalAttrs (cfg.wan.vlanId != null) {
# The ISP-side VLAN subinterface pppd dials on (e.g. AIS tags PPPoE).
"15-wan-vlan" = {
matchConfig.Name = "wan-vlan";
networkConfig.LinkLocalAddressing = "no";
linkConfig.RequiredForOnline = "no";
};
}
// {
# The bridge itself is L2-only; L3 lives on the vlan-* interfaces,
# which hang off the bridge (tagged on the bridge "self" port).
"20-br0" = {
matchConfig.Name = "br0";
networkConfig.LinkLocalAddressing = "no";
vlan = lib.mapAttrsToList (name: _: vlanIf name) cfg.vlans;
bridgeVLANs = taggedAll;
linkConfig.RequiredForOnline = "no";
};
}
// lib.listToAttrs (
map (port: {
name = "25-trunk-${port}";
value = {
matchConfig.Name = port;
networkConfig.Bridge = "br0";
bridgeVLANs = taggedAll;
linkConfig.RequiredForOnline = "no";
};
}) cfg.trunkPorts
)
// lib.mapAttrs' (
port: vlanName:
lib.nameValuePair "25-access-${port}" {
matchConfig.Name = port;
networkConfig.Bridge = "br0";
bridgeVLANs = [
{
VLAN = cfg.vlans.${vlanName}.id;
PVID = cfg.vlans.${vlanName}.id;
EgressUntagged = cfg.vlans.${vlanName}.id;
}
];
linkConfig.RequiredForOnline = "no";
}
) cfg.accessPorts
// lib.mapAttrs' (
name: vlan:
lib.nameValuePair "40-${vlanIf name}" {
matchConfig.Name = vlanIf name;
address = [ "${vlan.address}/${toString vlan.prefixLength}" ];
networkConfig = {
IPv6AcceptRA = false;
# Announce a /64 carved from the DHCPv6-PD prefix on ppp0 (SLAAC).
IPv6SendRA = true;
DHCPPrefixDelegation = true;
};
dhcpPrefixDelegationConfig.SubnetId = "auto";
linkConfig.RequiredForOnline = "no";
}
) cfg.vlans;
};
}
modules/router/dns-dhcp.nix
-84
View File
@@ -1,84 +0,0 @@
# LAN DHCP (Kea) and DNS (Blocky). Fully declarative: one Kea subnet per VLAN
# with dhcp.enable, Blocky as the blocklist resolver every DHCP lease points
# at. Blocky's HTTP listener (:4000) serves Prometheus metrics, scraped by
# control over the mesh (firewall.nix scopes it to the mesh subnet).
{
config,
lib,
...
}:
let
cfg = config.cnx.router;
dhcpVlans = lib.filterAttrs (_: vlan: vlan.dhcp.enable) cfg.vlans;
in
{
config = lib.mkIf cfg.enable {
services.kea.dhcp4 = {
enable = true;
settings = {
interfaces-config.interfaces = lib.mapAttrsToList (name: _: "vlan-${name}") dhcpVlans;
lease-database = {
type = "memfile";
persist = true;
name = "/var/lib/kea/dhcp4.leases";
};
valid-lifetime = 86400;
subnet4 = lib.mapAttrsToList (name: vlan: {
id = vlan.id;
subnet = vlan.subnet;
interface = "vlan-${name}";
valid-lifetime = vlan.dhcp.leaseTime;
pools = [ { pool = "${vlan.dhcp.pool.from} - ${vlan.dhcp.pool.to}"; } ];
reservations = lib.mapAttrsToList (host: res: {
hostname = host;
hw-address = res.hwAddress;
ip-address = res.ipAddress;
}) vlan.dhcp.reservations;
option-data = [
{
name = "routers";
data = vlan.address;
}
{
name = "domain-name-servers";
data = vlan.address;
}
];
}) dhcpVlans;
};
};
services.blocky = {
enable = true;
settings = {
ports = {
dns = 53;
http = 4000;
};
upstreams.groups.default = [
"9.9.9.9"
"149.112.112.112"
"2620:fe::fe"
];
blocking = {
denylists.ads = [
"https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts"
];
clientGroupsBlock.default = [ "ads" ];
};
caching = {
minTime = "5m";
prefetching = true;
};
prometheus.enable = true;
};
};
# The router itself resolves via public resolvers, not via Blocky, so DNS
# for deploys/updates survives a broken local resolver.
networking.nameservers = [
"9.9.9.9"
"1.1.1.1"
];
};
}
modules/router/firewall.nix
-62
View File
@@ -1,62 +0,0 @@
# Router firewall/NAT policy (nftables). Trust model:
# mgmt VLAN -> trusted: router services, all VLANs, WAN
# other VLANs -> DNS/DHCP on the router + WAN (if allowWan); no inter-VLAN
# WAN (ppp0) -> nothing inbound beyond established/related
# mesh -> admin SSH + metrics scrapes (same trust boundary as the fleet)
{
config,
lib,
...
}:
let
cfg = config.cnx.router;
mesh = import ../mesh-hosts.nix { inherit config lib; };
vlanIfs = lib.mapAttrsToList (name: _: "vlan-${name}") cfg.vlans;
wanVlanIfs = lib.mapAttrsToList (name: _: "vlan-${name}") (
lib.filterAttrs (_: vlan: vlan.allowWan) cfg.vlans
);
nonMgmtIfs = lib.filter (i: i != "vlan-mgmt") vlanIfs;
ifSet = ifs: "{ ${lib.concatStringsSep ", " (map (i: "\"${i}\"") ifs)} }";
in
{
config = lib.mkIf cfg.enable {
networking.nftables.enable = true;
# SSH reachable only from the mgmt VLAN (trusted) and the mesh — never
# from the WAN or the other VLANs.
services.openssh.openFirewall = false;
networking.firewall = {
enable = true;
filterForward = true;
trustedInterfaces = [ "vlan-mgmt" ];
# Non-mgmt VLANs may only talk to the router's DNS and DHCP.
interfaces = lib.genAttrs nonMgmtIfs (_: {
allowedTCPPorts = [ 53 ];
allowedUDPPorts = [
53
67
];
});
extraInputRules = ''
ip6 saddr ${mesh.subnet} tcp dport 22 accept comment "admin ssh over the mesh"
ip6 saddr ${mesh.subnet} tcp dport 4000 accept comment "blocky metrics scrape from control"
'';
extraForwardRules = ''
tcp flags syn tcp option maxseg size set rt mtu comment "MSS clamp for PPPoE mtu 1492"
iifname "vlan-mgmt" accept comment "mgmt reaches all VLANs and the WAN"
iifname ${ifSet wanVlanIfs} oifname "ppp0" accept comment "LAN to internet"
'';
};
networking.nat = {
enable = true;
externalInterface = "ppp0";
internalInterfaces = vlanIfs;
};
};
}
modules/router/iperf.nix
-28
View File
@@ -1,28 +0,0 @@
# iperf3 server on every gateway, for throughput testing from any LAN segment
# (e.g. validating AP/switch links: `iperf3 -c 10.<siteId>.<vlan>.1`) and from
# admin machines over the mesh. Never reachable from the WAN (default-deny).
{
config,
lib,
...
}:
let
cfg = config.cnx.router;
mesh = import ../mesh-hosts.nix { inherit config lib; };
vlanIfs = lib.mapAttrsToList (name: _: "vlan-${name}") cfg.vlans;
in
{
config = lib.mkIf cfg.enable {
services.iperf3.enable = true;
networking.firewall.interfaces = lib.genAttrs vlanIfs (_: {
allowedTCPPorts = [ 5201 ];
allowedUDPPorts = [ 5201 ];
});
networking.firewall.extraInputRules = ''
ip6 saddr ${mesh.subnet} tcp dport 5201 accept comment "iperf3 over the mesh"
ip6 saddr ${mesh.subnet} udp dport 5201 accept comment "iperf3 over the mesh"
'';
};
}
modules/router/ipv6.nix
-31
View File
@@ -1,31 +0,0 @@
# IPv6 on the PPPoE uplink: run networkd's DHCPv6 client on ppp0 to obtain a
# delegated prefix; each vlan-* interface (default.nix) carves a /64 out of it
# via DHCPPrefixDelegation and announces it to clients with SLAAC.
{
config,
lib,
...
}:
let
cfg = config.cnx.router;
in
{
config = lib.mkIf cfg.enable {
systemd.network.networks."45-ppp0" = {
matchConfig.Name = "ppp0";
networkConfig = {
DHCP = "ipv6";
# pppd owns the v4 address/route on this link; don't let networkd
# tear them down.
KeepConfiguration = "static";
# Default v6 route comes from the ISP's RA when they send one.
IPv6AcceptRA = true;
};
# Many PPPoE ISPs never send an RA with the M flag; solicit regardless.
dhcpV6Config.WithoutRA = "solicit";
linkConfig.RequiredForOnline = "no";
};
boot.kernel.sysctl."net.ipv6.conf.all.forwarding" = lib.mkDefault 1;
};
}
modules/router/pppoe.nix
-64
View File
@@ -1,64 +0,0 @@
# PPPoE WAN session. ISP credentials are entered once at `clan vars generate`
# (prompts). Both are secret — AIS often uses the same string for username and
# password — so neither may land in the Nix store: pppd reads the username from
# an included secret options file and the password from chap/pap-secrets.
{
config,
lib,
...
}:
let
cfg = config.cnx.router;
creds = config.clan.core.vars.generators.pppoe-credentials;
in
{
config = lib.mkIf cfg.enable {
clan.core.vars.generators.pppoe-credentials = {
prompts.username = {
description = "PPPoE username (from the ISP)";
type = "hidden";
};
prompts.password = {
description = "PPPoE password (from the ISP)";
type = "hidden";
};
files."user-opts".secret = true;
files."chap-secrets".secret = true;
script = ''
user="$(cat "$prompts"/username)"
pass="$(cat "$prompts"/password)"
printf 'user "%s"\n' "$user" > "$out"/user-opts
printf '"%s" * "%s"\n' "$user" "$pass" > "$out"/chap-secrets
'';
};
services.pppd = {
enable = true;
peers.wan = {
autostart = true;
config = ''
plugin pppoe.so ${cfg.wan.pppInterface}
ifname ppp0
file ${creds.files."user-opts".path}
noipdefault
defaultroute
noauth
hide-password
persist
maxfail 0
holdoff 5
lcp-echo-interval 15
lcp-echo-failure 3
+ipv6
mtu 1492
mru 1492
'';
};
};
# pppd looks up the password for `user` in these files at dial time; both
# point at the same generated `"<user>" * "<pass>"` line (PAP and CHAP).
environment.etc."ppp/chap-secrets".source = creds.files."chap-secrets".path;
environment.etc."ppp/pap-secrets".source = creds.files."chap-secrets".path;
};
}
modules/router/speedtest.nix
-67
View File
@@ -1,67 +0,0 @@
# Periodic WAN speed test so ISP degradation shows up as a trend instead of a
# complaint. A timer runs librespeed-cli and writes the results as Prometheus
# metrics into node_exporter's textfile collector — they ride the existing
# 9100 scrape to VictoriaMetrics, where alerts.nix compares each run against
# the link's own 7-day median (no per-site threshold to maintain).
{
config,
lib,
pkgs,
...
}:
let
cfg = config.cnx.router;
textfileDir = "/var/lib/speedtest";
in
{
options.cnx.router.speedtest.interval = lib.mkOption {
type = lib.types.str;
default = "hourly";
description = "systemd OnCalendar spec for the WAN speed test.";
};
config = lib.mkIf cfg.enable {
services.prometheus.exporters.node.extraFlags = [
"--collector.textfile.directory=${textfileDir}"
];
systemd.services.speedtest = {
description = "WAN speed test to Prometheus textfile metrics";
after = [ "network-online.target" ];
wants = [ "network-online.target" ];
path = [
pkgs.librespeed-cli
pkgs.jq
];
serviceConfig = {
Type = "oneshot";
StateDirectory = "speedtest";
# One test at boot would race PPPoE and log a spurious failure.
ExecCondition = "${pkgs.iproute2}/bin/ip link show ppp0";
};
script = ''
tmp="${textfileDir}/.speedtest.prom.tmp"
if result=$(librespeed-cli --json); then
jq -r '.[0]
| "speedtest_download_mbps \(.download)",
"speedtest_upload_mbps \(.upload)",
"speedtest_ping_ms \(.ping)",
"speedtest_jitter_ms \(.jitter)",
"speedtest_success 1"' <<<"$result" > "$tmp"
else
echo "speedtest_success 0" > "$tmp"
fi
mv "$tmp" "${textfileDir}/speedtest.prom"
'';
};
systemd.timers.speedtest = {
wantedBy = [ "timers.target" ];
timerConfig = {
OnCalendar = cfg.speedtest.interval;
RandomizedDelaySec = "10m";
Persistent = true;
};
};
};
}
modules/web-proxy.nix
+4 -1
View File
@@ -10,7 +10,10 @@
... ...
}: }:
let let
mesh = import ./mesh-hosts.nix { inherit config lib; }; mesh = import ./mesh-hosts.nix {
dir = config.clan.core.settings.directory;
inherit lib;
};
hosts = import ./hosts.nix; hosts = import ./hosts.nix;
certName = "cnx.network"; certName = "cnx.network";
in in