diff --git a/clan.nix b/clan.nix index 47badb2..58764a5 100644 --- a/clan.nix +++ b/clan.nix @@ -1,3 +1,4 @@ +{ ... }: let hosts = import ./modules/hosts.nix; @@ -17,12 +18,18 @@ let }; in { + # Site gateways: the router service instance lives in inventory.nix. + imports = [ ./inventory.nix ]; + # Ensure this is unique among all clans you want to use. meta.name = "cnx-network-clan"; meta.domain = "cnx-network.internal"; inventory.machines = fleet; + # Services defined in this repo (module.input = "self"). See modules/clan/. + modules.router = ./modules/clan/router; + inventory.instances = { # Admin SSH keys + root password, split per the clan-core migration off diff --git a/docs/src/gateways.md b/docs/src/gateways.md index b435fc1..d3c2e86 100644 --- a/docs/src/gateways.md +++ b/docs/src/gateways.md @@ -3,28 +3,39 @@ NixOS routers (OPNsense replacements) on Topton 1U fanless boxes — Intel N300, 4x i226-V 2.5G ports (some units add 2x 10G SFP+), 16GB DDR5, 256GB mSATA. Four sites are planned; the first is `gw-cnx-1` (Chiang Mai). Everything lives -in the reusable `modules/router/` module (`cnx.router.*` options); a site's -machine config only sets port names, VLANs, and feature flags. +in the `router` clan service (`modules/clan/router/`, registered as +`modules.router` in `clan.nix`); a site joins by adding its machine to the +`router` inventory instance in `inventory.nix` with settings for port names, +VLANs, and feature flags (`roles.default.machines..settings`). Naming: `gw--`, e.g. `gw-cnx-1`. +The service has a NixOS VM test (`modules/clan/router/tests/vm/`): an ISP +node runs a PPPoE access concentrator, a `client` sits on the tagged lan/iot +VLANs behind the trunk, an `admin` on the untagged mgmt access port, and a +simulated radio (`mac80211_hwsim`) carries two SSIDs with a WPA3 station. It +checks the PPPoE dial-in, leases/reservations, Blocky, NAT, `allowWan`, the +mgmt-only trust model, SSH exposure and the Wi-Fi bridge ports. Run it with +`nix build .#checks.x86_64-linux.router` (also part of `nix flake check`). + ## What each gateway runs -| Function | Implementation | -| ------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| WAN | PPPoE (`pppd`), per-site ISP credentials via clan vars prompts; `wan.vlanId` when the ISP tags the session (AIS: 10); `wan.macAddress` to clone the old router's MAC if the ISP has it pinned | -| LAN | VLAN-filtering bridge `br0` over the trunk ports (networkd); `accessPorts` pin a port untagged to one VLAN — convention: the last copper port is an untagged `mgmt` recovery port | -| Firewall/NAT | nftables: default-deny WAN, no inter-VLAN, MSS clamp, v4 NAT | -| DHCP | Kea, one subnet per VLAN | -| DNS | Blocky (blocklist resolver), metrics on :4000 scraped by control | -| IPv6 | DHCPv6-PD on ppp0, /64 per VLAN via SLAAC | -| Bans | CrowdSec + nftables bouncer (sshd log parsing) | -| Omada | Optional per site: TP-Link Omada controller as a podman container | -| Proxy | Optional: Caddy reverse proxy for internal services under `*..cnx.network` with a real Let's Encrypt wildcard (DNS-01 against ns1) | -| Diagnostics | iperf3 server on 5201, reachable from every VLAN and the mesh (`iperf3 -c `); CLI toolkit: tcpdump, mtr, ethtool, conntrack, kdig, iftop, librespeed-cli | -| Speed test | Hourly librespeed run (`speedtest.timer`) → `speedtest_*` metrics via node_exporter; vmalert flags download < 50% of the link's own 7-day median (`WANSpeedDegraded`) | -| Management | ZeroTier mesh: SSH, node_exporter, journald upload — like the fleet | -| Storage | Single-disk ZFS (zstd, `/var` its own dataset for snapshots) | +| Function | Implementation | +| ------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| WAN | PPPoE (`pppd`), per-site ISP credentials via clan vars prompts; `wan.vlanId` when the ISP tags the session (AIS: 10); `wan.macAddress` to clone the old router's MAC if the ISP has it pinned | +| LAN | VLAN-filtering bridge `br0` over the trunk ports (networkd); `accessPorts` pin a port untagged to one VLAN — convention: the last copper port is an untagged `mgmt` recovery port; `stagingPort` turns a spare port into a DHCP uplink into the old LAN while the box runs alongside the router it replaces (drop at cutover) | +| Firewall/NAT | nftables: default-deny WAN, no inter-VLAN, MSS clamp, v4 NAT | +| DHCP | Kea, one subnet per VLAN | +| DNS | Blocky (blocklist resolver), metrics on :4000 scraped by control | +| IPv6 | DHCPv6-PD on ppp0, /64 per VLAN via SLAAC | +| Bans | CrowdSec + nftables bouncer (sshd log parsing) | +| Omada | Optional per site: TP-Link Omada controller as a podman container | +| Wi-Fi | Optional: hostapd on the router's radios; each SSID (`wifi.networks`) is an untagged access port of its VLAN, passphrases via vars prompts — see `modules/clan/router/README.md` | +| Proxy | Optional: Caddy reverse proxy for internal services under `*..cnx.network` with a real Let's Encrypt wildcard (DNS-01 against ns1) | +| Diagnostics | iperf3 server on 5201, reachable from every VLAN and the mesh (`iperf3 -c `); CLI toolkit: tcpdump, mtr, ethtool, conntrack, kdig, iftop, librespeed-cli | +| Speed test | Hourly librespeed run (`speedtest.timer`) → `speedtest_*` metrics via node_exporter; vmalert flags download < 50% of the link's own 7-day median (`WANSpeedDegraded`) | +| Management | ZeroTier mesh: SSH, node_exporter, journald upload — like the fleet | +| Storage | Single-disk ZFS (zstd, `/var` its own dataset for snapshots) | ## Addressing plan @@ -55,22 +66,24 @@ Trust model: mgmt → everything; other VLANs → router DNS/DHCP + internet onl ## Adding a site -1. Copy `machines/gw-cnx-1/` to `machines/gw--/`; set `site`, - `siteId` (next free number), port names, VLANs, and `omada.enable`. - Fill the real disk id in `disko.nix` (from the installer: - `ls -l /dev/disk/by-id/`). Keep the `mgmt`/`lan` VLANs. -2. Add the machine to `inventory.machines` in `clan.nix` and to the machine - list in `modules/mesh-hosts.nix`. Do **not** add it to `modules/hosts.nix` - (dynamic PPPoE IP; clan connects over the mesh). +1. Copy `machines/gw-cnx-1/` to `machines/gw--/` (disko, facter, + hostId, timezone). Fill the real disk id in `disko.nix` (from the + installer: `ls -l /dev/disk/by-id/`). +2. Add the machine to `inventory.machines` in `clan.nix`, to the `router` + instance in `inventory.nix` (`roles.default.machines.gw--.settings`: `site`, + `siteId` (next free number), port names, VLANs, `omada.enable`; keep the + `mgmt`/`lan` VLANs), and to the machine list in `modules/mesh-hosts.nix`. + Do **not** add it to `modules/hosts.nix` (dynamic PPPoE IP; clan connects + over the mesh). 3. Add its node_exporter and Blocky scrape targets in `modules/monitoring/server.nix`. -4. If the site uses the internal proxy (`cnx.router.proxy.enable`), wire ns1: - import `(import ../../modules/dns/acme-gw-secret.nix "gw--")` in - **both** the gateway's and ns1's configuration, and on ns1 add the - `dns-acme-gw---knot` generator, its `keyFiles` entry, and an - `acl_acme_gw__` scoped to `_acme-challenge.` on the - `cnx.network` zone (copy the `gw-cnx-1` blocks). Then - `clan vars generate ns1` and redeploy ns1. +4. If the site uses the internal proxy (`proxy.enable` in its settings), wire + ns1: import `(import ../../modules/clan/router/acme-secret.nix +"gw--")` in ns1's configuration (the gateway declares the same + shared generator itself), and on ns1 add the `dns-acme-gw---knot` + generator, its `keyFiles` entry, and an `acl_acme_gw__` scoped to + `_acme-challenge.` on the `cnx.network` zone (copy the `gw-cnx-1` + blocks). Then `clan vars generate ns1` and redeploy ns1. 5. `git add` the new machine directory — flake evaluation only sees git-tracked files, so an untracked `machines/gw-…/` is silently ignored. 6. `clan vars generate gw--` — prompts for the site's PPPoE @@ -86,9 +99,10 @@ Trust model: mgmt → everything; other VLANs → router DNS/DHCP + internet onl ## Omada controller -Sites with TP-Link Omada APs/switches set `cnx.router.omada.enable = true`. -The controller runs as a podman container (`mbentley/omada-controller` — there -is no nixpkgs package) with host networking on the mgmt VLAN, where adoption +Sites with TP-Link Omada APs/switches set `omada.enable = true` in their +router settings. The controller runs as a podman container +(`mbentley/omada-controller` — there is no nixpkgs package) with host +networking on the mgmt VLAN, where adoption broadcasts (UDP 29810) live. UI: `https://:8043` from mgmt, or the gateway's mesh address from an admin machine. Controller state is under `/var/lib/omada`, declared as clan state (`clan.core.state.omada`); wiring it @@ -96,12 +110,12 @@ into the borgbackup instance is still a follow-up. ## Internal reverse proxy -`cnx.router.proxy.enable` puts Caddy on the gateway, terminating TLS for -`..cnx.network` (e.g. `https://omada.cnx1.cnx.network`) and -forwarding to internal backends: +`proxy.enable` in a site's router settings puts Caddy on the gateway, +terminating TLS for `..cnx.network` (e.g. +`https://omada.cnx1.cnx.network`) and forwarding to internal backends: ```nix -cnx.router.proxy = { +proxy = { enable = true; services.omada = { backend = "https://127.0.0.1:8043"; @@ -114,13 +128,22 @@ cnx.router.proxy = { gateway, issued via ACME DNS-01 (lego/RFC2136) against ns1 — works behind PPPoE with zero inbound reachability, and browsers trust it without a CA install. Each gateway has its own TSIG key (`acme_gw__`) that ns1 - scopes to `_acme-challenge.` TXT records only. + scopes to `_acme-challenge.` TXT records only. The zone, ns1's + address and the ACME contact are fleet-wide role settings in `inventory.nix` + (`proxy.domain`, `proxy.acme.nameserver`, `proxy.acme.email`). - **Resolution**: the names exist only internally — Blocky answers `*..cnx.network` with the router's `lan` address; the public `cnx.network` zone never carries them. - **Access**: `proxy.allowVlans` (default `mgmt` + `lan`) get 443 (and 80 for the HTTP→HTTPS redirect). Not exposed to WAN, guest VLANs, or the mesh. +## Using the service from another clan + +The service is exported as `clan.modules.router` of this flake; add this +repo as a flake input and reference it with `module.input = ""`. +`modules/clan/router/README.md` is the consumer-facing description (it is +also the module's manifest readme, shown by `clan modules`). + ## Runbook - **PPPoE down**: `systemctl status pppd-wan`, `journalctl -u pppd-wan` on the diff --git a/flake.nix b/flake.nix index 7e1c717..52d9885 100644 --- a/flake.nix +++ b/flake.nix @@ -48,6 +48,9 @@ "x86_64-darwin" ]; + # perSystem.clan.nixosTests: NixOS VM tests of our own clan services. + imports = [ clan-core.flakeModules.testModule ]; + flake = { inherit (clan.config) nixosConfigurations nixosModules clanInternals; clan = clan.config; @@ -84,6 +87,12 @@ formatter = treefmtEval.config.build.wrapper; checks.formatting = treefmtEval.config.build.check self; + # `nix build .#checks.x86_64-linux.router` (VM test, Linux only). + clan.nixosTests.router = { + imports = [ ./modules/clan/router/tests/vm ]; + clan.modules.router = ./modules/clan/router; + }; + # Add the Clan cli tool to the dev shell. # Use "nix develop" to enter the dev shell. devShells.default = pkgs.mkShell { diff --git a/inventory.nix b/inventory.nix new file mode 100644 index 0000000..2899d27 --- /dev/null +++ b/inventory.nix @@ -0,0 +1,93 @@ +# Site gateways: the `router` service instance (modules/clan/router). One +# instance for the fleet; each site adds its machine with the site's port / +# VLAN layout under roles.default.machines..settings. Everything else +# (PPPoE, Kea/Blocky, nftables, IPv6-PD, CrowdSec, iperf3, speedtest) follows +# from the service. Imported by clan.nix. +{ config, lib, ... }: +let + hosts = import ./modules/hosts.nix; + mesh = import ./modules/mesh-hosts.nix { + dir = config.directory; + inherit lib; + }; +in +{ + inventory.instances.router = { + module = { + name = "router"; + input = "self"; + }; + + # Fleet-wide settings: admin paths (SSH, scrapes, iperf3, Omada UI) are + # opened to the mesh; internal proxy names live under + # .cnx.network with wildcard certs issued via DNS-01 against + # ns1 (which must declare the gateway's acme-secret.nix generator). + roles.default.settings = { + mesh.subnet = mesh.subnet; + proxy = { + domain = "cnx.network"; + acme.nameserver = hosts.ns1.ipv4; + acme.email = "postmaster@cnx.email"; + }; + }; + + # Chiang Mai (site 1): Topton 1U, Intel N300, 4x i226-V 2.5G. Port roles + # use the expected igc names — verify against facter.json after the + # first install and adjust if the box enumerates differently. + roles.default.machines.gw-cnx-1.settings = { + site = "cnx"; + siteId = 1; + wan.interface = "enp1s0"; + wan.vlanId = null; # this ISP runs PPPoE untagged on the port + wan.macAddress = "a8:b8:e0:01:06:87"; + trunkPorts = [ "enp2s0" ]; + # STAGING (remove at cutover, move enp3s0 back into trunkPorts): + # DHCP-client uplink into the existing OPNsense LAN so the box has + # internet + mesh while it runs alongside the old router. + stagingPort = "enp3s0"; + # Dedicated on-site recovery port: untagged mgmt, always available even + # if the switch config is broken. + accessPorts.enp4s0 = "mgmt"; + # Replaces the newedge.house OPNsense box; renumbered to the fleet + # convention (10.1..0/24, router .1, pool .100-.199). The old + # untagged LAN becomes tagged mgmt — infra switch ports get PVID 10. + vlans = { + mgmt = { + id = 10; # 10.1.10.0/24 — servers, APs, switches, Omada, admin + dhcp.reservations.storinator01 = { + hwAddress = "7c:c2:55:e0:d6:40"; + ipAddress = "10.1.10.53"; + }; + }; + lan.id = 20; # 10.1.20.0/24 — trusted clients + iot.id = 40; # 10.1.40.0/24 + voip.id = 50; # 10.1.50.0/24 + dmz.id = 60; # 10.1.60.0/24 + unit1.id = 110; # 10.1.110.0/24 + unit2.id = 120; # 10.1.120.0/24 + unit3.id = 130; # 10.1.130.0/24 + unit4.id = 140; # 10.1.140.0/24 + unit5 = { + id = 150; # 10.1.150.0/24 + dhcp.reservations.newt = { + hwAddress = "7c:d3:0a:21:58:0b"; + ipAddress = "10.1.150.22"; + }; + }; + }; + # This site runs the Omada controller for its APs/switches. + omada.enable = true; + + # Internal reverse proxy: real wildcard cert *.cnx1.cnx.network; Blocky + # resolves the names to the router's LAN address for mgmt+lan clients. + proxy = { + enable = true; + services.omada = { + # Omada's UI is HTTPS with a self-signed cert on the host network. + backend = "https://127.0.0.1:8043"; + insecureSkipVerify = true; + }; + }; + }; + }; +} diff --git a/machines/gw-cnx-1/configuration.nix b/machines/gw-cnx-1/configuration.nix index 24c7be2..55f4af5 100644 --- a/machines/gw-cnx-1/configuration.nix +++ b/machines/gw-cnx-1/configuration.nix @@ -1,13 +1,9 @@ # Site gateway Chiang Mai (site 1): Topton 1U, Intel N300, 4x i226-V 2.5G. -# Port roles below use the expected igc names — verify against facter.json -# after the first install and adjust if the box enumerates differently. +# The router itself (ports, VLANs, features) is the `router` service instance +# in clan.nix; only machine-local bits live here. { config, lib, ... }: { - imports = [ - ../../modules/router - ../../modules/monitoring/exporters.nix - (import ../../modules/dns/acme-gw-secret.nix "gw-cnx-1") - ]; + imports = [ ../../modules/monitoring/exporters.nix ]; clan.core.sops.defaultGroups = [ "admins" ]; @@ -20,73 +16,6 @@ builtins.hashString "sha256" config.networking.hostName ); - cnx.router = { - enable = true; - site = "cnx"; - siteId = 1; - wan.interface = "enp1s0"; - wan.vlanId = null; # this ISP runs PPPoE untagged on the port - wan.macAddress = "a8:b8:e0:01:06:87"; - trunkPorts = [ - "enp2s0" - # "enp3s0" # STAGING: serves as the uplink below until cutover - ]; - # Dedicated on-site recovery port: untagged mgmt, always available even - # if the switch config is broken. - accessPorts.enp4s0 = "mgmt"; - # Replaces the newedge.house OPNsense box; renumbered to the fleet - # convention (10.1..0/24, router .1, pool .100-.199). The old - # untagged LAN becomes tagged mgmt — infra switch ports get PVID 10. - vlans = { - mgmt = { - id = 10; # 10.1.10.0/24 — servers, APs, switches, Omada, admin - dhcp.reservations.storinator01 = { - hwAddress = "7c:c2:55:e0:d6:40"; - ipAddress = "10.1.10.53"; - }; - }; - lan.id = 20; # 10.1.20.0/24 — trusted clients - iot.id = 40; # 10.1.40.0/24 - voip.id = 50; # 10.1.50.0/24 - dmz.id = 60; # 10.1.60.0/24 - unit1.id = 110; # 10.1.110.0/24 - unit2.id = 120; # 10.1.120.0/24 - unit3.id = 130; # 10.1.130.0/24 - unit4.id = 140; # 10.1.140.0/24 - unit5 = { - id = 150; # 10.1.150.0/24 - dhcp.reservations.newt = { - hwAddress = "7c:d3:0a:21:58:0b"; - ipAddress = "10.1.150.22"; - }; - }; - }; - # This site runs the Omada controller for its APs/switches. - omada.enable = true; - - # Internal reverse proxy: real wildcard cert *.cnx1.cnx.network; Blocky - # resolves the names to the router's LAN address for mgmt+lan clients. - proxy = { - enable = true; - services.omada = { - # Omada's UI is HTTPS with a self-signed cert on the host network. - backend = "https://127.0.0.1:8043"; - insecureSkipVerify = true; - }; - }; - }; - - # STAGING (remove at cutover, and restore enp3s0 to trunkPorts): DHCP-client - # uplink into the existing OPNsense LAN so the box has internet + mesh while - # it runs alongside the old router. Default-deny firewall on this interface - # (it's in no VLAN zone); PPPoE simply retries until the WAN port is cabled. - # Do NOT connect the trunk ports to the production switch while staging — - # Kea on tag 10 would fight the OPNsense LAN DHCP in one broadcast domain. - systemd.network.networks."05-staging" = { - matchConfig.Name = "enp3s0"; - networkConfig.DHCP = "ipv4"; - }; - time.timeZone = "Etc/GMT-7"; # UTC+7 (Thailand, fixed offset, no DST) services.chrony.enable = true; } diff --git a/machines/ns1/configuration.nix b/machines/ns1/configuration.nix index d4a9082..41ff060 100644 --- a/machines/ns1/configuration.nix +++ b/machines/ns1/configuration.nix @@ -6,7 +6,10 @@ }: let domains = import ../../modules/dns/domains.nix; - mesh = import ../../modules/mesh-hosts.nix { inherit config lib; }; + mesh = import ../../modules/mesh-hosts.nix { + dir = config.clan.core.settings.directory; + inherit lib; + }; hosts = import ../../modules/hosts.nix; in { @@ -14,7 +17,8 @@ in ../../modules/dns/authoritative.nix ../../modules/dns/acme-mx1-secret.nix ../../modules/dns/acme-web01-secret.nix - (import ../../modules/dns/acme-gw-secret.nix "gw-cnx-1") + # Same shared TSIG secret the gateway's router service declares (proxy). + (import ../../modules/clan/router/acme-secret.nix "gw-cnx-1") ../../modules/static-ipv6.nix ../../modules/monitoring/exporters.nix ]; diff --git a/modules/clan/router/README.md b/modules/clan/router/README.md new file mode 100644 index 0000000..d47c14d --- /dev/null +++ b/modules/clan/router/README.md @@ -0,0 +1,91 @@ +# router + +Turns a machine with several NICs into a site gateway: PPPoE WAN (ISP +credentials via vars prompts), a VLAN-filtering bridge over the LAN ports with +one L3 interface per VLAN, Kea DHCP and Blocky DNS per VLAN, nftables +firewall/NAT, DHCPv6-PD, CrowdSec, an iperf3 server and a WAN speed-test +timer. Optional: a Wi-Fi access point on the router's own radios (hostapd), +the TP-Link Omada controller (podman) and an internal Caddy reverse proxy +with a real wildcard certificate (ACME DNS-01). + +Addressing convention: a site owns `10..0.0/16`; VLAN `` defaults +to `10...0/24`, router at `.1`, DHCP pool `.100-.199`. The `mgmt` +and `lan` VLANs are mandatory. Trust model: mgmt reaches everything; other +VLANs get router DNS/DHCP and (with `allowWan`) the internet, no inter-VLAN; +WAN nothing inbound; the admin mesh (`mesh.subnet`) gets SSH, metrics, iperf3 +and the Omada UI. + +## Usage from another clan + +```nix +# flake.nix +inputs.cnx-network.url = "git+https:///B4L/cnx-network-clan"; + +# clan.nix +inventory.instances.router = { + module = { name = "router"; input = "cnx-network"; }; + roles.default.settings.mesh.subnet = "fd..::/88"; # your admin overlay + roles.default.machines.gw-1.settings = { + site = "ams"; + siteId = 1; + wan.interface = "enp1s0"; + wan.vlanId = 10; # or null for untagged PPPoE + trunkPorts = [ "enp2s0" ]; + accessPorts.enp4s0 = "mgmt"; # untagged on-site recovery port + # stagingPort = "enp3s0"; # DHCP uplink into the old LAN until cutover + vlans = { + mgmt.id = 10; + lan.id = 20; + iot = { id = 40; allowWan = false; }; + }; + }; +}; +``` + +Then `clan vars generate gw-1` prompts for the PPPoE username/password. + +### Wi-Fi access point + +If the box has wireless cards, the router can be the site's AP. An SSID is +defined once and behaves like an untagged access port of its VLAN; radios +choose what to broadcast, so a dual-band card serves the same SSID twice: + +```nix +wifi = { + enable = true; + countryCode = "TH"; + networks = { + home.vlan = "lan"; # WPA3 with WPA2 fallback + things = { vlan = "iot"; security = "wpa2"; }; # legacy IoT + guest = { vlan = "guest"; isolateClients = true; }; + }; + radios = { + wlp5s0 = { band = "2g"; channel = 6; macAddress = "…"; networks = [ "home" "things" ]; }; + wlp6s0 = { band = "5g"; channel = 36; networks = [ "home" ]; }; + }; +}; +``` + +Passphrases are vars prompts (`wifi--passphrase`), asked once at `clan +vars generate`. A radio broadcasting more than one SSID needs its hardware +`macAddress`: hostapd wants a fixed BSSID per extra SSID, derived from it. +`security = "wpa3-transition"` (the default) offers SAE and WPA2-PSK-SHA256; +devices that only speak classic WPA2-PSK need `security = "wpa2"`. + +### Internal proxy + +`proxy.enable` serves `..` under a wildcard +certificate obtained via RFC 2136 DNS-01 against `proxy.acme.nameserver`. The +gateway signs updates with TSIG key `acme_`, whose secret is +the shared `dns-acme--secret` generator declared by this service. +The nameserver machine must declare the same generator so both sides hold one +secret — import `acme-secret.nix` from this directory with the gateway's name: + +```nix +imports = [ (import "${inputs.cnx-network}/modules/clan/router/acme-secret.nix" "gw-1") ]; +``` + +and load the key with an acl scoped to `_acme-challenge.`. + +The service does not open the WAN to anything; reach gateways over your mesh. +One instance per machine. diff --git a/modules/clan/router/acme-secret.nix b/modules/clan/router/acme-secret.nix new file mode 100644 index 0000000..0509790 --- /dev/null +++ b/modules/clan/router/acme-secret.nix @@ -0,0 +1,22 @@ +# Shared TSIG secret for a gateway's dedicated ACME key (function: machine +# name -> NixOS module). The acme_ key lets that gateway — and only +# it — write _acme-challenge. TXT records on the authoritative +# nameserver to obtain its internal wildcard cert via DNS-01 (proxy.nix). +# +# The router service declares it on the gateway automatically when +# proxy.enable is set. The nameserver machine must declare the very same +# generator so both sides share one secret: +# imports = [ (import /acme-secret.nix "gw-cnx-1") ]; +# and then load it into its DNS server as key acme_gw_cnx_1 (hmac-sha256) +# with an acl scoped to that gateway's _acme-challenge label. +machine: +{ pkgs, ... }: +{ + clan.core.vars.generators."dns-acme-${machine}-secret" = { + share = true; + files."secret".secret = true; + runtimeInputs = [ pkgs.openssl ]; + # 32 random bytes, base64 — a valid hmac-sha256 TSIG secret. + script = ''openssl rand -base64 32 | tr -d '\n' > "$out"/secret''; + }; +} diff --git a/modules/clan/router/crowdsec.nix b/modules/clan/router/crowdsec.nix new file mode 100644 index 0000000..b7128e5 --- /dev/null +++ b/modules/clan/router/crowdsec.nix @@ -0,0 +1,46 @@ +# CrowdSec security engine + nftables bouncer: parses sshd auth attempts from +# the journal and bans offending source IPs at the firewall. Log-based (no +# inline DPI) so it costs the N300 next to nothing. +{ settings }: +{ ... }: +let + cfg = settings; +in +{ + services.crowdsec = { + enable = true; + autoUpdateService = true; + hub.collections = [ + "crowdsecurity/linux" + "crowdsecurity/sshd" + ]; + localConfig = { + acquisitions = [ + { + source = "journalctl"; + journalctl_filter = [ "_SYSTEMD_UNIT=sshd.service" ]; + labels.type = "syslog"; + } + ]; + # Never ban the ZeroTier mesh — it is the only admin path to these + # boxes (no public SSH), so a false positive would lock us out. + # Parser-stage whitelist: mesh events are dropped before any scenario. + parsers.s02Enrich = [ + { + name = "cnx/mesh-whitelist"; + description = "Whitelist the ZeroTier management mesh"; + whitelist = { + reason = "ZeroTier mesh is the admin path"; + cidr = [ cfg.mesh.subnet ]; + }; + } + ]; + }; + }; + + services.crowdsec-firewall-bouncer = { + enable = true; + registerBouncer.enable = true; + settings.mode = "nftables"; + }; +} diff --git a/modules/clan/router/default.nix b/modules/clan/router/default.nix new file mode 100644 index 0000000..b720315 --- /dev/null +++ b/modules/clan/router/default.nix @@ -0,0 +1,67 @@ +# Site gateway (OPNsense replacement) for the Topton 1U boxes, as a clan +# service: PPPoE WAN, VLAN-filtering bridge over the LAN ports, per-VLAN +# DHCP/DNS, firewall/NAT, and the optional Omada controller / internal proxy. +# +# Exported as `clan.modules.router` of this flake; used here with +# `module.input = "self"` and from other clans with `module.input = +# ""`. README.md has the consumer view. A site joins +# through the inventory: +# +# inventory.instances.router = { +# module = { name = "router"; input = "self"; }; +# roles.default.settings.mesh.subnet = ...; # fleet-wide +# roles.default.machines.gw--.settings = { site = ...; ... }; +# }; +# +# The settings schema lives in interface.nix. Implementation files that need +# the settings are functions `{ settings }: `; the evaluated +# settings are handed in with importApply so nothing goes through +# machine-level options. The rest are plain NixOS modules. +{ lib, ... }: +{ + _class = "clan.service"; + manifest.name = "router"; + manifest.description = "Site gateway: PPPoE WAN, VLAN bridge, DHCP/DNS, firewall/NAT"; + manifest.categories = [ "Network" ]; + manifest.readme = builtins.readFile ./README.md; + + roles.default = { + description = "Turns the machine into the site's router (one instance per machine)."; + interface = ./interface.nix; + + perInstance = + { settings, machine, ... }: + { + nixosModule.imports = [ + ./ipv6.nix + ] + # The proxy's TSIG secret is shared with the nameserver (acme-secret.nix). + ++ lib.optional settings.proxy.enable (import ./acme-secret.nix machine.name) + ++ map (file: lib.modules.importApply file { inherit settings; }) [ + ./network.nix + ./pppoe.nix + ./firewall.nix + ./dns-dhcp.nix + ./crowdsec.nix + ./omada.nix + ./proxy.nix + ./iperf.nix + ./speedtest.nix + ./wifi.nix + ]; + }; + }; + + # A machine has exactly one WAN port and one VLAN layout; two instances would + # both claim br0/ppp0 and fight over Kea/Blocky/nftables. + perMachine = + { instances, machine, ... }: + { + nixosModule.assertions = [ + { + assertion = lib.length (lib.attrNames instances) == 1; + message = "router: ${machine.name} is a gateway in several instances (${lib.concatStringsSep ", " (lib.attrNames instances)}); a machine can only be one router."; + } + ]; + }; +} diff --git a/modules/clan/router/dns-dhcp.nix b/modules/clan/router/dns-dhcp.nix new file mode 100644 index 0000000..491e066 --- /dev/null +++ b/modules/clan/router/dns-dhcp.nix @@ -0,0 +1,82 @@ +# LAN DHCP (Kea) and DNS (Blocky). Fully declarative: one Kea subnet per VLAN +# with dhcp.enable, Blocky as the blocklist resolver every DHCP lease points +# at. Blocky's HTTP listener (:4000) serves Prometheus metrics, scraped by +# control over the mesh (firewall.nix scopes it to the mesh subnet). +{ settings }: +{ lib, ... }: +let + cfg = settings; + dhcpVlans = lib.filterAttrs (_: vlan: vlan.dhcp.enable) cfg.vlans; +in +{ + services.kea.dhcp4 = { + enable = true; + settings = { + interfaces-config.interfaces = lib.mapAttrsToList (name: _: "vlan-${name}") dhcpVlans; + lease-database = { + type = "memfile"; + persist = true; + name = "/var/lib/kea/dhcp4.leases"; + }; + valid-lifetime = 86400; + subnet4 = lib.mapAttrsToList (name: vlan: { + id = vlan.id; + subnet = vlan.subnet; + interface = "vlan-${name}"; + valid-lifetime = vlan.dhcp.leaseTime; + pools = [ { pool = "${vlan.dhcp.pool.from} - ${vlan.dhcp.pool.to}"; } ]; + reservations = lib.mapAttrsToList (host: res: { + hostname = host; + hw-address = res.hwAddress; + ip-address = res.ipAddress; + }) vlan.dhcp.reservations; + option-data = [ + { + name = "routers"; + data = vlan.address; + } + { + name = "domain-name-servers"; + data = vlan.address; + } + ]; + }) dhcpVlans; + }; + }; + + services.blocky = { + enable = true; + settings = { + ports = { + dns = 53; + http = 4000; + }; + upstreams.groups.default = [ + "9.9.9.9" + "149.112.112.112" + "2620:fe::fe" + ]; + blocking = { + denylists.ads = [ + "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts" + ]; + clientGroupsBlock.default = [ "ads" ]; + }; + caching = { + minTime = "5m"; + prefetching = true; + }; + prometheus.enable = true; + }; + }; + + # The router itself resolves via public resolvers, not via Blocky, so DNS + # for deploys/updates survives a broken local resolver. networkd would + # enable systemd-resolved by default, whose stub listener on 127.0.0.53:53 + # makes Blocky's wildcard :53 bind fail — plain resolv.conf instead. + services.resolved.enable = false; + networking.nameservers = [ + "9.9.9.9" + "1.1.1.1" + ]; +} diff --git a/modules/clan/router/firewall.nix b/modules/clan/router/firewall.nix new file mode 100644 index 0000000..f41b0a8 --- /dev/null +++ b/modules/clan/router/firewall.nix @@ -0,0 +1,65 @@ +# Router firewall/NAT policy (nftables). Trust model: +# mgmt VLAN -> trusted: router services, all VLANs, WAN +# other VLANs -> DNS/DHCP on the router + WAN (if allowWan); no inter-VLAN +# WAN (ppp0) -> nothing inbound beyond established/related +# mesh -> admin SSH + metrics scrapes (same trust boundary as the fleet) +# staging -> admin SSH only (pre-cutover uplink into the old LAN) +{ settings }: +{ lib, ... }: +let + cfg = settings; + + vlanIfs = lib.mapAttrsToList (name: _: "vlan-${name}") cfg.vlans; + wanVlanIfs = lib.mapAttrsToList (name: _: "vlan-${name}") ( + lib.filterAttrs (_: vlan: vlan.allowWan) cfg.vlans + ); + nonMgmtIfs = lib.filter (i: i != "vlan-mgmt") vlanIfs; +in +{ + networking.nftables.enable = true; + + # SSH reachable only from the mgmt VLAN (trusted) and the mesh — never + # from the WAN or the other VLANs. + services.openssh.openFirewall = false; + + networking.firewall = { + enable = true; + filterForward = true; + trustedInterfaces = [ "vlan-mgmt" ]; + + # Non-mgmt VLANs may only talk to the router's DNS and DHCP; the staging + # uplink (old LAN, pre-cutover) gets admin SSH so the box can be reached + # before the mgmt VLAN or the mesh are up. + interfaces = + lib.genAttrs nonMgmtIfs (_: { + allowedTCPPorts = [ 53 ]; + allowedUDPPorts = [ + 53 + 67 + ]; + }) + // lib.optionalAttrs (cfg.stagingPort != null) { + ${cfg.stagingPort}.allowedTCPPorts = [ 22 ]; + }; + + extraInputRules = '' + ip6 saddr ${cfg.mesh.subnet} tcp dport 22 accept comment "admin ssh over the mesh" + ip6 saddr ${cfg.mesh.subnet} tcp dport 4000 accept comment "blocky metrics scrape from control" + ''; + + extraForwardRules = '' + tcp flags syn tcp option maxseg size set rt mtu comment "MSS clamp for PPPoE mtu 1492" + iifname "vlan-mgmt" accept comment "mgmt reaches all VLANs and the WAN" + ''; + }; + + # networking.nat both masquerades and opens forward-to-WAN for exactly its + # internalInterfaces — so this list, not a rule of our own, is where + # `allowWan` is enforced. Listing every VLAN here would silently let + # allowWan = false VLANs out. + networking.nat = { + enable = true; + externalInterface = "ppp0"; + internalInterfaces = wanVlanIfs; + }; +} diff --git a/modules/clan/router/interface.nix b/modules/clan/router/interface.nix new file mode 100644 index 0000000..be1ce50 --- /dev/null +++ b/modules/clan/router/interface.nix @@ -0,0 +1,378 @@ +# Settings of the `router` service (inventory `roles.default.settings`). +# Pure schema: no machine config is reachable here; the implementation files +# get the evaluated result as `settings`. +# +# Fleet addressing convention: each site owns 10..0.0/16. A VLAN's +# subnet defaults to 10...0/24 with the router at .1 and the +# DHCP pool at .100-.199. VLANs that need more space (e.g. public-wifi guest) +# override `subnet`/`address`/`dhcp.pool` and take a wider block from the +# upper half (10..128.0/17), e.g. guest -> 10..128.0/22. +# VLAN ids: 10 = mgmt, 20 = lan (mandatory); 30 = guest, 40 = iot (reserved). +{ config, lib, ... }: +let + site = toString config.siteId; + + vlanModule = + { config, ... }: + let + octet = toString config.id; + in + { + options = { + id = lib.mkOption { + type = lib.types.ints.between 1 4094; + description = "802.1Q VLAN id (fleet convention: 10 mgmt, 20 lan, 30 guest, 40 iot)."; + }; + address = lib.mkOption { + type = lib.types.str; + default = "10.${site}.${octet}.1"; + defaultText = lib.literalExpression ''"10...1"''; + description = "Router address on this VLAN."; + }; + prefixLength = lib.mkOption { + type = lib.types.ints.between 8 30; + default = 24; + }; + subnet = lib.mkOption { + type = lib.types.str; + default = "10.${site}.${octet}.0/24"; + defaultText = lib.literalExpression ''"10...0/24"''; + description = "The VLAN's network in CIDR form (must contain `address`)."; + }; + dhcp = { + enable = lib.mkOption { + type = lib.types.bool; + default = true; + }; + pool = { + from = lib.mkOption { + type = lib.types.str; + default = "10.${site}.${octet}.100"; + defaultText = lib.literalExpression ''"10...100"''; + }; + to = lib.mkOption { + type = lib.types.str; + default = "10.${site}.${octet}.199"; + defaultText = lib.literalExpression ''"10...199"''; + }; + }; + leaseTime = lib.mkOption { + type = lib.types.ints.positive; + default = 86400; + description = '' + Lease validity in seconds. Lower it for high-churn networks, + e.g. public-WiFi guest VLANs (3600-7200), so the pool recycles. + ''; + }; + reservations = lib.mkOption { + type = lib.types.attrsOf ( + lib.types.submodule { + options = { + hwAddress = lib.mkOption { + type = lib.types.str; + example = "aa:bb:cc:dd:ee:ff"; + description = "Client MAC address."; + }; + ipAddress = lib.mkOption { + type = lib.types.str; + description = "Fixed address handed to this client (inside the VLAN's subnet, outside the pool)."; + }; + }; + } + ); + default = { }; + description = "Static DHCP leases; the attribute name becomes the client's hostname."; + }; + }; + allowWan = lib.mkOption { + type = lib.types.bool; + default = true; + description = "Whether clients on this VLAN may reach the internet."; + }; + }; + }; + + proxyServiceModule = { + options = { + backend = lib.mkOption { + type = lib.types.str; + example = "https://127.0.0.1:8043"; + description = "URL Caddy forwards to (internal/mesh address)."; + }; + insecureSkipVerify = lib.mkOption { + type = lib.types.bool; + default = false; + description = "Skip TLS verification towards the backend (self-signed upstreams like Omada)."; + }; + }; + }; + + wifiNetworkModule = + { name, ... }: + { + options = { + ssid = lib.mkOption { + type = lib.types.str; + default = name; + defaultText = lib.literalExpression ""; + description = "SSID as advertised; defaults to the attribute name."; + }; + vlan = lib.mkOption { + type = lib.types.str; + example = "lan"; + description = "VLAN (by name) the clients of this SSID land in, like an untagged access port."; + }; + security = lib.mkOption { + type = lib.types.enum [ + "wpa3" + "wpa3-transition" + "wpa2" + "open" + ]; + default = "wpa3-transition"; + description = '' + - `wpa3`: WPA3-Personal (SAE) only. + - `wpa3-transition`: WPA3 with WPA2 fallback for older clients. + - `wpa2`: WPA2-PSK only, for legacy IoT devices. + - `open`: no encryption (captive/guest use; pair with an isolated VLAN). + Except for `open`, the passphrase is a vars prompt + (`wifi--passphrase`, entered at `clan vars generate`). + ''; + }; + hidden = lib.mkOption { + type = lib.types.bool; + default = false; + description = "Do not advertise the SSID in beacons (clients must know it)."; + }; + isolateClients = lib.mkOption { + type = lib.types.bool; + default = false; + description = "Keep wireless clients of this SSID from talking to each other (guest networks)."; + }; + }; + }; + + wifiRadioModule = { + options = { + band = lib.mkOption { + type = lib.types.enum [ + "2g" + "5g" + "6g" + ]; + default = "2g"; + description = "Frequency band of this radio; a dual-band card exposes one radio interface per band."; + }; + channel = lib.mkOption { + type = lib.types.ints.unsigned; + default = 0; + description = "Channel; 0 lets hostapd pick one (ACS) — not every driver supports that."; + }; + wifi6 = lib.mkOption { + type = lib.types.bool; + default = false; + description = "Enable 802.11ax (WiFi 6) on this radio; WiFi 4/5 are always on."; + }; + macAddress = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "aa:bb:cc:dd:ee:ff"; + description = '' + Hardware address of the radio (facter.json / `ip link`). Needed when + the radio serves more than one network: hostapd wants a fixed BSSID + per extra network, derived from this address (locally administered + variants of its first octet). + ''; + }; + networks = lib.mkOption { + type = lib.types.listOf lib.types.str; + example = [ + "home" + "iot" + ]; + description = "Networks (from `wifi.networks`) this radio broadcasts; at most four per radio."; + }; + }; + }; +in +{ + options = { + site = lib.mkOption { + type = lib.types.str; + description = "City code of the site, e.g. \"cnx\"."; + }; + + siteId = lib.mkOption { + type = lib.types.ints.between 1 254; + description = "Site number; drives the 10...0/24 addressing."; + }; + + wan.interface = lib.mkOption { + type = lib.types.str; + description = "Physical WAN port the PPPoE session runs on."; + }; + + wan.vlanId = lib.mkOption { + type = lib.types.nullOr (lib.types.ints.between 1 4094); + default = null; + description = '' + 802.1Q tag the ISP requires for the PPPoE session (AIS Thailand: 10); + null for untagged PPPoE directly on the port. Unrelated to the LAN + VLANs — this tag exists only on the WAN port. + ''; + }; + + wan.macAddress = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "aa:bb:cc:dd:ee:ff"; + description = '' + Spoofed MAC for the WAN port, e.g. to keep the MAC the ISP has + pinned (cloned from the old router). null keeps the hardware MAC. + ''; + }; + + trunkPorts = lib.mkOption { + type = lib.types.listOf lib.types.str; + description = "LAN ports carrying all VLANs tagged (incl. any 10G SFP+ ports)."; + }; + + accessPorts = lib.mkOption { + type = lib.types.attrsOf lib.types.str; + default = { }; + example = { + enp4s0 = "mgmt"; + }; + description = '' + Ports acting as untagged access ports on a single VLAN (port name -> + VLAN name). Frames are untagged on the wire; the bridge tags them with + the VLAN's PVID. Use for an always-available on-site mgmt port. + ''; + }; + + stagingPort = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "enp3s0"; + description = '' + Temporary DHCPv4-client uplink into the existing LAN while the box + runs alongside the router it replaces: gives it internet + mesh + before the WAN port is cabled (PPPoE simply retries until then). The + port is in no VLAN zone; the firewall admits only SSH on it. Do NOT + connect the trunk ports to the production switch while staging — + Kea on the mgmt tag would fight the old router's DHCP in one + broadcast domain. Set to null at cutover (and usually hand the port + back to `trunkPorts`). + ''; + }; + + vlans = lib.mkOption { + type = lib.types.attrsOf (lib.types.submodule vlanModule); + description = "VLANs served at this site; `mgmt` and `lan` are mandatory."; + }; + + mesh.subnet = lib.mkOption { + type = lib.types.str; + example = "fd12:3456:789a:bcde:f000::/88"; + description = '' + IPv6 prefix of the admin mesh (the ZeroTier overlay; see + modules/mesh-hosts.nix). Admin SSH, metrics scrapes, iperf3 and the + Omada UI accept connections from it, and CrowdSec never bans it. + ''; + }; + + omada.enable = lib.mkEnableOption "TP-Link Omada SDN controller (podman container)"; + + proxy = { + enable = lib.mkEnableOption "internal reverse proxy (Caddy, wildcard cert via DNS-01)"; + + domain = lib.mkOption { + type = lib.types.str; + example = "example.net"; + description = '' + Parent zone of the proxy names: services are served as + .. under a wildcard certificate. + ''; + }; + + acme = { + nameserver = lib.mkOption { + type = lib.types.str; + example = "203.0.113.53"; + description = '' + Authoritative nameserver of `domain` that accepts RFC 2136 + updates for _acme-challenge. with this gateway's + TSIG key (acme_, secret from the shared + dns-acme--secret generator, see acme-secret.nix). + ''; + }; + email = lib.mkOption { + type = lib.types.str; + example = "postmaster@example.net"; + description = "ACME account contact."; + }; + }; + + services = lib.mkOption { + type = lib.types.attrsOf (lib.types.submodule proxyServiceModule); + default = { }; + description = "Proxied services; attr name becomes ..."; + }; + + allowVlans = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ + "mgmt" + "lan" + ]; + description = "VLANs whose clients may reach the proxy (443, plus 80 for the redirect)."; + }; + }; + + wifi = { + enable = lib.mkEnableOption "a Wi-Fi access point on the router's own radios (hostapd)"; + + countryCode = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "TH"; + description = "ISO 3166-1 country code for the regulatory domain; required when enabled."; + }; + + networks = lib.mkOption { + type = lib.types.attrsOf (lib.types.submodule wifiNetworkModule); + default = { }; + example = { + home.vlan = "lan"; + things = { + vlan = "iot"; + security = "wpa2"; + }; + }; + description = '' + Wireless networks (SSIDs). Each one behaves like an untagged access + port on its VLAN; the radios below choose which to broadcast. + ''; + }; + + radios = lib.mkOption { + type = lib.types.attrsOf (lib.types.submodule wifiRadioModule); + default = { }; + example = { + wlp5s0 = { + band = "5g"; + channel = 36; + networks = [ "home" ]; + }; + }; + description = "Wireless radios of the router (interface name -> config); at least one when enabled."; + }; + }; + + speedtest.interval = lib.mkOption { + type = lib.types.str; + default = "hourly"; + description = "systemd OnCalendar spec for the WAN speed test."; + }; + }; +} diff --git a/modules/clan/router/iperf.nix b/modules/clan/router/iperf.nix new file mode 100644 index 0000000..e3a76bf --- /dev/null +++ b/modules/clan/router/iperf.nix @@ -0,0 +1,22 @@ +# iperf3 server on every gateway, for throughput testing from any LAN segment +# (e.g. validating AP/switch links: `iperf3 -c 10...1`) and from +# admin machines over the mesh. Never reachable from the WAN (default-deny). +{ settings }: +{ lib, ... }: +let + cfg = settings; + vlanIfs = lib.mapAttrsToList (name: _: "vlan-${name}") cfg.vlans; +in +{ + services.iperf3.enable = true; + + networking.firewall.interfaces = lib.genAttrs vlanIfs (_: { + allowedTCPPorts = [ 5201 ]; + allowedUDPPorts = [ 5201 ]; + }); + + networking.firewall.extraInputRules = '' + ip6 saddr ${cfg.mesh.subnet} tcp dport 5201 accept comment "iperf3 over the mesh" + ip6 saddr ${cfg.mesh.subnet} udp dport 5201 accept comment "iperf3 over the mesh" + ''; +} diff --git a/modules/clan/router/ipv6.nix b/modules/clan/router/ipv6.nix new file mode 100644 index 0000000..1248afe --- /dev/null +++ b/modules/clan/router/ipv6.nix @@ -0,0 +1,22 @@ +# IPv6 on the PPPoE uplink: run networkd's DHCPv6 client on ppp0 to obtain a +# delegated prefix; each vlan-* interface (network.nix) carves a /64 out of it +# via DHCPPrefixDelegation and announces it to clients with SLAAC. +{ lib, ... }: +{ + systemd.network.networks."45-ppp0" = { + matchConfig.Name = "ppp0"; + networkConfig = { + DHCP = "ipv6"; + # pppd owns the v4 address/route on this link; don't let networkd + # tear them down. + KeepConfiguration = "static"; + # Default v6 route comes from the ISP's RA when they send one. + IPv6AcceptRA = true; + }; + # Many PPPoE ISPs never send an RA with the M flag; solicit regardless. + dhcpV6Config.WithoutRA = "solicit"; + linkConfig.RequiredForOnline = "no"; + }; + + boot.kernel.sysctl."net.ipv6.conf.all.forwarding" = lib.mkDefault 1; +} diff --git a/modules/clan/router/network.nix b/modules/clan/router/network.nix new file mode 100644 index 0000000..dd43e06 --- /dev/null +++ b/modules/clan/router/network.nix @@ -0,0 +1,170 @@ +# L2/L3 of the gateway: PPPoE WAN port (optionally on an ISP VLAN), a +# VLAN-filtering bridge over the LAN ports, and one L3 interface per VLAN. +{ settings }: +{ lib, pkgs, ... }: +let + cfg = settings; + + vlanIf = name: "vlan-${name}"; +in +{ + assertions = [ + { + assertion = cfg.vlans ? mgmt && cfg.vlans ? lan; + message = "router: every site must define the `mgmt` and `lan` VLANs."; + } + { + assertion = lib.all (v: cfg.vlans ? ${v}) (lib.attrValues cfg.accessPorts); + message = "router: every accessPorts value must name a defined VLAN."; + } + { + assertion = lib.all (p: !(cfg.accessPorts ? ${p})) cfg.trunkPorts; + message = "router: a port cannot be both a trunk and an access port."; + } + { + assertion = + cfg.stagingPort == null + || !( + cfg.stagingPort == cfg.wan.interface + || lib.elem cfg.stagingPort cfg.trunkPorts + || cfg.accessPorts ? ${cfg.stagingPort} + ); + message = "router: stagingPort ${toString cfg.stagingPort} is also the WAN, a trunk or an access port."; + } + ]; + + # Router diagnostics toolkit: packets (tcpdump), path (mtr), link + # negotiation (ethtool), NAT state (conntrack), DNS (kdig), per-flow + # bandwidth (iftop), WAN throughput (librespeed-cli; iperf3 covers LAN). + environment.systemPackages = with pkgs; [ + tcpdump + mtr + ethtool + conntrack-tools + knot-dns + iftop + librespeed-cli + ]; + + networking.useNetworkd = true; + networking.useDHCP = false; + systemd.network.enable = true; + + systemd.network.netdevs = { + "20-br0" = { + netdevConfig = { + Name = "br0"; + Kind = "bridge"; + }; + bridgeConfig.VLANFiltering = true; + }; + } + // lib.optionalAttrs (cfg.wan.vlanId != null) { + "15-wan-vlan" = { + netdevConfig = { + Name = "wan-vlan"; + Kind = "vlan"; + }; + vlanConfig.Id = cfg.wan.vlanId; + }; + } + // lib.mapAttrs' ( + name: vlan: + lib.nameValuePair "30-${vlanIf name}" { + netdevConfig = { + Name = vlanIf name; + Kind = "vlan"; + }; + vlanConfig.Id = vlan.id; + } + ) cfg.vlans; + + systemd.network.networks = + let + taggedAll = lib.mapAttrsToList (_: vlan: { VLAN = vlan.id; }) cfg.vlans; + in + lib.optionalAttrs (cfg.stagingPort != null) { + # Staging uplink (see interface.nix): plain DHCPv4 client on a spare + # port, no bridge/VLAN membership, so the firewall treats it as untrusted. + "05-staging" = { + matchConfig.Name = cfg.stagingPort; + networkConfig.DHCP = "ipv4"; + }; + } + // { + # WAN port carries only the PPPoE session; no IP config of its own. + "10-wan" = { + matchConfig.Name = cfg.wan.interface; + networkConfig.LinkLocalAddressing = "no"; + vlan = lib.optional (cfg.wan.vlanId != null) "wan-vlan"; + linkConfig = { + RequiredForOnline = "carrier"; + } + # The wan-vlan subinterface (and thus the PPPoE session) inherits + # the parent port's MAC, so spoofing here covers both cases. + // lib.optionalAttrs (cfg.wan.macAddress != null) { + MACAddress = cfg.wan.macAddress; + }; + }; + } + // lib.optionalAttrs (cfg.wan.vlanId != null) { + # The ISP-side VLAN subinterface pppd dials on (e.g. AIS tags PPPoE). + "15-wan-vlan" = { + matchConfig.Name = "wan-vlan"; + networkConfig.LinkLocalAddressing = "no"; + linkConfig.RequiredForOnline = "no"; + }; + } + // { + # The bridge itself is L2-only; L3 lives on the vlan-* interfaces, + # which hang off the bridge (tagged on the bridge "self" port). + "20-br0" = { + matchConfig.Name = "br0"; + networkConfig.LinkLocalAddressing = "no"; + vlan = lib.mapAttrsToList (name: _: vlanIf name) cfg.vlans; + bridgeVLANs = taggedAll; + linkConfig.RequiredForOnline = "no"; + }; + } + // lib.listToAttrs ( + map (port: { + name = "25-trunk-${port}"; + value = { + matchConfig.Name = port; + networkConfig.Bridge = "br0"; + bridgeVLANs = taggedAll; + linkConfig.RequiredForOnline = "no"; + }; + }) cfg.trunkPorts + ) + // lib.mapAttrs' ( + port: vlanName: + lib.nameValuePair "25-access-${port}" { + matchConfig.Name = port; + networkConfig.Bridge = "br0"; + bridgeVLANs = [ + { + VLAN = cfg.vlans.${vlanName}.id; + PVID = cfg.vlans.${vlanName}.id; + EgressUntagged = cfg.vlans.${vlanName}.id; + } + ]; + linkConfig.RequiredForOnline = "no"; + } + ) cfg.accessPorts + // lib.mapAttrs' ( + name: vlan: + lib.nameValuePair "40-${vlanIf name}" { + matchConfig.Name = vlanIf name; + address = [ "${vlan.address}/${toString vlan.prefixLength}" ]; + networkConfig = { + IPv6AcceptRA = false; + # Announce a /64 carved from the DHCPv6-PD prefix on ppp0 (SLAAC). + IPv6SendRA = true; + DHCPPrefixDelegation = true; + }; + dhcpPrefixDelegationConfig.SubnetId = "auto"; + linkConfig.RequiredForOnline = "no"; + } + ) cfg.vlans; +} diff --git a/modules/router/omada.nix b/modules/clan/router/omada.nix similarity index 76% rename from modules/router/omada.nix rename to modules/clan/router/omada.nix index dec7e17..e71f949 100644 --- a/modules/router/omada.nix +++ b/modules/clan/router/omada.nix @@ -3,20 +3,13 @@ # the de-facto standard image). Host networking because device adoption relies # on L2 broadcast discovery (UDP 29810) on the mgmt VLAN; the default-deny # input firewall keeps its ports unreachable from WAN and non-mgmt VLANs. -{ - config, - lib, - ... -}: +{ settings }: +{ config, lib, ... }: let - cfg = config.cnx.router; - mesh = import ../mesh-hosts.nix { inherit config lib; }; + cfg = settings; in { - options.cnx.router.omada.enable = - lib.mkEnableOption "TP-Link Omada SDN controller (podman container)"; - - config = lib.mkIf (cfg.enable && cfg.omada.enable) { + config = lib.mkIf cfg.omada.enable { virtualisation.podman.enable = true; virtualisation.oci-containers = { backend = "podman"; @@ -33,7 +26,7 @@ in # Admin UI (8043) also reachable over the mesh, like Grafana on control. networking.firewall.extraInputRules = '' - ip6 saddr ${mesh.subnet} tcp dport 8043 accept comment "omada ui over the mesh" + ip6 saddr ${cfg.mesh.subnet} tcp dport 8043 accept comment "omada ui over the mesh" ''; # Controller state (adopted devices, site config, cert) — declared as clan diff --git a/modules/clan/router/pppoe.nix b/modules/clan/router/pppoe.nix new file mode 100644 index 0000000..9848a0f --- /dev/null +++ b/modules/clan/router/pppoe.nix @@ -0,0 +1,61 @@ +# PPPoE WAN session. ISP credentials are entered once at `clan vars generate` +# (prompts). Both are secret — AIS often uses the same string for username and +# password — so neither may land in the Nix store: pppd reads the username from +# an included secret options file and the password from chap/pap-secrets. +{ settings }: +{ config, ... }: +let + cfg = settings; + creds = config.clan.core.vars.generators.pppoe-credentials; + # Interface pppd dials on: the WAN port itself, or its ISP VLAN (network.nix). + pppInterface = if cfg.wan.vlanId == null then cfg.wan.interface else "wan-vlan"; +in +{ + clan.core.vars.generators.pppoe-credentials = { + prompts.username = { + description = "PPPoE username (from the ISP)"; + type = "hidden"; + }; + prompts.password = { + description = "PPPoE password (from the ISP)"; + type = "hidden"; + }; + files."user-opts".secret = true; + files."chap-secrets".secret = true; + script = '' + user="$(cat "$prompts"/username)" + pass="$(cat "$prompts"/password)" + printf 'user "%s"\n' "$user" > "$out"/user-opts + printf '"%s" * "%s"\n' "$user" "$pass" > "$out"/chap-secrets + ''; + }; + + services.pppd = { + enable = true; + peers.wan = { + autostart = true; + config = '' + plugin pppoe.so ${pppInterface} + ifname ppp0 + file ${creds.files."user-opts".path} + noipdefault + defaultroute + noauth + hide-password + persist + maxfail 0 + holdoff 5 + lcp-echo-interval 15 + lcp-echo-failure 3 + +ipv6 + mtu 1492 + mru 1492 + ''; + }; + }; + + # pppd looks up the password for `user` in these files at dial time; both + # point at the same generated `"" * ""` line (PAP and CHAP). + environment.etc."ppp/chap-secrets".source = creds.files."chap-secrets".path; + environment.etc."ppp/pap-secrets".source = creds.files."chap-secrets".path; +} diff --git a/modules/router/proxy.nix b/modules/clan/router/proxy.nix similarity index 52% rename from modules/router/proxy.nix rename to modules/clan/router/proxy.nix index 78ca45e..e36334d 100644 --- a/modules/router/proxy.nix +++ b/modules/clan/router/proxy.nix @@ -1,94 +1,57 @@ # Internal reverse proxy for the gateway: Caddy terminates TLS for -# ..cnx.network (e.g. omada.cnx1.cnx.network) and +# .. (e.g. omada.cnx1.cnx.network) and # forwards to backends by their internal address. The cert is a real Let's -# Encrypt wildcard (*..cnx.network) obtained via ACME DNS-01 -# against ns1 with a gateway-scoped TSIG key, so browsers trust it without -# any CA install; the names only *resolve* internally — Blocky answers them -# with the router's LAN address, the public zone never carries them. +# Encrypt wildcard (*..) obtained via ACME DNS-01 +# against proxy.acme.nameserver with a gateway-scoped TSIG key, so browsers +# trust it without any CA install; the names only *resolve* internally — +# Blocky answers them with the router's LAN address, the public zone never +# carries them. # -# Requires the machine to also import the shared secret generator: -# (import ../../modules/dns/acme-gw-secret.nix "") -# and ns1 to hold the matching key + acl (see machines/ns1/configuration.nix). -{ - config, - lib, - ... -}: +# The TSIG secret is the shared dns-acme--secret generator +# (acme-secret.nix, declared here via default.nix); the nameserver machine +# must declare the same generator and load the key (this fleet: ns1). +{ settings }: +{ config, lib, ... }: let - cfg = config.cnx.router; - hosts = import ../hosts.nix; + cfg = settings; hostname = config.networking.hostName; tsigKey = "acme_${lib.replaceStrings [ "-" ] [ "_" ] hostname}"; - certName = "${cfg.site}${toString cfg.siteId}.cnx.network"; - - serviceModule = { - options = { - backend = lib.mkOption { - type = lib.types.str; - example = "https://127.0.0.1:8043"; - description = "URL Caddy forwards to (internal/mesh address)."; - }; - insecureSkipVerify = lib.mkOption { - type = lib.types.bool; - default = false; - description = "Skip TLS verification towards the backend (self-signed upstreams like Omada)."; - }; - }; - }; + certName = "${cfg.site}${toString cfg.siteId}.${cfg.proxy.domain}"; + nameserver = "${cfg.proxy.acme.nameserver}:53"; in { - options.cnx.router.proxy = { - enable = lib.mkEnableOption "internal reverse proxy (Caddy, wildcard cert via DNS-01)"; - - services = lib.mkOption { - type = lib.types.attrsOf (lib.types.submodule serviceModule); - default = { }; - description = "Proxied services; attr name becomes .${certName}."; - }; - - allowVlans = lib.mkOption { - type = lib.types.listOf lib.types.str; - default = [ - "mgmt" - "lan" - ]; - description = "VLANs whose clients may reach the proxy (443, plus 80 for the redirect)."; - }; - }; - - config = lib.mkIf (cfg.enable && cfg.proxy.enable) { + config = lib.mkIf cfg.proxy.enable { assertions = [ { assertion = lib.all (v: cfg.vlans ? ${v}) cfg.proxy.allowVlans; - message = "cnx.router.proxy.allowVlans must name VLANs defined in cnx.router.vlans."; + message = "router: proxy.allowVlans must name VLANs defined in vlans."; } ]; - # Render the shared per-gateway TSIG secret into a lego rfc2136 env file; - # same pattern as web01 (modules/web-proxy.nix), scoped on ns1 to - # _acme-challenge. TXT records only. + # Render the shared per-gateway TSIG secret into a lego rfc2136 env file, + # scoped on the nameserver to _acme-challenge. TXT records. clan.core.vars.generators."dns-acme-${hostname}-rfc2136" = { files."rfc2136.env".secret = true; # root-owned; systemd reads it as root dependencies = [ "dns-acme-${hostname}-secret" ]; script = '' - printf 'RFC2136_NAMESERVER=${hosts.ns1.ipv4}:53\nRFC2136_TSIG_ALGORITHM=hmac-sha256.\nRFC2136_TSIG_KEY=${tsigKey}\nRFC2136_TSIG_SECRET=%s\n' \ + printf 'RFC2136_NAMESERVER=${nameserver}\nRFC2136_TSIG_ALGORITHM=hmac-sha256.\nRFC2136_TSIG_KEY=${tsigKey}\nRFC2136_TSIG_SECRET=%s\n' \ "$(cat "$in"/dns-acme-${hostname}-secret/secret)" > "$out"/rfc2136.env ''; }; security.acme = { acceptTerms = true; - defaults.email = "postmaster@cnx.email"; - # One wildcard for every proxied service; DNS-01 against ns1, so issuance - # works behind PPPoE with no inbound reachability at all. + defaults.email = cfg.proxy.acme.email; + # One wildcard for every proxied service; DNS-01, so issuance works + # behind PPPoE with no inbound reachability at all. certs.${certName} = { domain = "*.${certName}"; dnsProvider = "rfc2136"; environmentFile = config.clan.core.vars.generators."dns-acme-${hostname}-rfc2136".files."rfc2136.env".path; - # ns1 is the only nameserver that accepts this key's UPDATE; check - # propagation against it directly rather than a public resolver. - dnsResolver = "${hosts.ns1.ipv4}:53"; + # Only that nameserver accepts this key's UPDATE; check propagation + # against it directly rather than a public resolver. + dnsResolver = nameserver; # Caddy reads the cert from explicit file paths (tls directive below), # so it won't notice a renewal on its own. reloadServices = [ "caddy.service" ]; diff --git a/modules/clan/router/speedtest.nix b/modules/clan/router/speedtest.nix new file mode 100644 index 0000000..7682648 --- /dev/null +++ b/modules/clan/router/speedtest.nix @@ -0,0 +1,55 @@ +# Periodic WAN speed test so ISP degradation shows up as a trend instead of a +# complaint. A timer runs librespeed-cli and writes the results as Prometheus +# metrics into node_exporter's textfile collector — they ride the existing +# 9100 scrape to VictoriaMetrics, where alerts.nix compares each run against +# the link's own 7-day median (no per-site threshold to maintain). +{ settings }: +{ pkgs, ... }: +let + cfg = settings; + textfileDir = "/var/lib/speedtest"; +in +{ + services.prometheus.exporters.node.extraFlags = [ + "--collector.textfile.directory=${textfileDir}" + ]; + + systemd.services.speedtest = { + description = "WAN speed test to Prometheus textfile metrics"; + after = [ "network-online.target" ]; + wants = [ "network-online.target" ]; + path = [ + pkgs.librespeed-cli + pkgs.jq + ]; + serviceConfig = { + Type = "oneshot"; + StateDirectory = "speedtest"; + # One test at boot would race PPPoE and log a spurious failure. + ExecCondition = "${pkgs.iproute2}/bin/ip link show ppp0"; + }; + script = '' + tmp="${textfileDir}/.speedtest.prom.tmp" + if result=$(librespeed-cli --json); then + jq -r '.[0] + | "speedtest_download_mbps \(.download)", + "speedtest_upload_mbps \(.upload)", + "speedtest_ping_ms \(.ping)", + "speedtest_jitter_ms \(.jitter)", + "speedtest_success 1"' <<<"$result" > "$tmp" + else + echo "speedtest_success 0" > "$tmp" + fi + mv "$tmp" "${textfileDir}/speedtest.prom" + ''; + }; + + systemd.timers.speedtest = { + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = cfg.speedtest.interval; + RandomizedDelaySec = "10m"; + Persistent = true; + }; + }; +} diff --git a/modules/clan/router/tests/vm/default.nix b/modules/clan/router/tests/vm/default.nix new file mode 100644 index 0000000..682921b --- /dev/null +++ b/modules/clan/router/tests/vm/default.nix @@ -0,0 +1,306 @@ +# End-to-end VM test of the router service: a PPPoE access concentrator plays +# the ISP on the WAN port, a trunk carries tagged lan/iot VLANs to `client`, +# and an untagged access port carries mgmt to `admin`. +# +# isp ---(vlan 1: PPPoE)--- wan [gw] trunk ---(vlan 2: tagged 20/40)--- client +# access --(vlan 3: untagged mgmt)--- admin +# +# What is proven: PPPoE dial-in with the vars-provided credentials, bridge +# VLAN tagging/untagging, Kea leases and reservations per VLAN, Blocky +# answering on the VLAN with the blocklist active, NAT to the WAN, and the +# firewall trust model (allowWan, mgmt-only SSH, no inter-VLAN forwarding). +{ pkgs, lib, ... }: +let + # The vars mock answers every prompt with "mock-prompt-value-"; the + # ISP side must accept exactly those. + chapSecrets = ''"mock-prompt-value-username" * "mock-prompt-value-password" *''; + ispAddress = "192.0.2.1"; + + clientMac = "02:00:00:00:00:20"; + clientAddress = "10.9.20.50"; + adminMac = "02:00:00:00:00:10"; + adminAddress = "10.9.10.50"; +in +{ + name = "router"; + + clan = { + directory = ./.; + # Bridges, VLAN netdevs, PPPoE and nftables need a real kernel. + test.useContainers = false; + inventory = { + # Every node is a clan machine (the test framework's defaults require + # it); only gw gets the router role. + machines = { + gw = { }; + isp = { }; + client = { }; + admin = { }; + }; + + instances.router = { + module.name = "router"; + module.input = "self"; + roles.default.machines.gw.settings = { + site = "tst"; + siteId = 9; + mesh.subnet = "fd00:7e57:c1a1:c0de::/64"; + wan.interface = "wan"; + trunkPorts = [ "trunk" ]; + accessPorts.access = "mgmt"; + vlans = { + mgmt = { + id = 10; + dhcp.reservations.admin = { + hwAddress = adminMac; + ipAddress = adminAddress; + }; + }; + lan = { + id = 20; + dhcp.reservations.client = { + hwAddress = clientMac; + ipAddress = clientAddress; + }; + }; + iot = { + id = 40; + allowWan = false; + }; + }; + # Access point on the simulated radio wlan0 (mac80211_hwsim, address + # 02:00:00:00:00:00): two SSIDs, one per VLAN. + wifi = { + enable = true; + countryCode = "US"; + networks = { + home.vlan = "lan"; + things = { + vlan = "iot"; + security = "wpa2"; + hidden = true; + isolateClients = true; + }; + }; + radios.wlan0 = { + channel = 6; + macAddress = "02:00:00:00:00:00"; + networks = [ + "home" + "things" + ]; + }; + }; + }; + }; + }; + }; + + nodes = { + gw = { + virtualisation.interfaces = { + wan = { + vlan = 1; + assignIP = false; + }; + trunk = { + vlan = 2; + assignIP = false; + }; + access = { + vlan = 3; + assignIP = false; + }; + }; + + # Something must listen on 22 for the mgmt-only SSH rule to be observable + # (a refused and a dropped connection look the same to the client). + services.openssh.enable = true; + + # The sandbox has no internet: serve the blocklist from a local file + # instead of GitHub, and skip CrowdSec, whose hub sync needs the network + # (it is not what this test exercises). + services.blocky.settings.blocking.denylists.ads = lib.mkForce [ + (toString (pkgs.writeText "ads.hosts" "0.0.0.0 ads.example.com\n")) + ]; + services.crowdsec.enable = lib.mkForce false; + services.crowdsec-firewall-bouncer.enable = lib.mkForce false; + + # Two simulated radios: wlan0 is the AP (settings above), wlan1 plays a + # wireless client. It lives in its own network namespace, like the + # separate host it stands in for — otherwise its lease would add a + # second 10.9.20.0/24 route to the router's own table. Its DHCP lease + # must come from Kea on the SSID's VLAN: wlan1 -> air -> wlan0 -> br0 -> + # vlan-lan. The mock passphrase is what the vars mock fed into the + # wifi-home-passphrase generator. + boot.kernelModules = [ "mac80211_hwsim" ]; + systemd.services.wifi-station = { + wantedBy = [ "multi-user.target" ]; + # No BindsTo: the device unit vanishes once wlan1 moves into the netns. + after = [ "sys-subsystem-net-devices-wlan1.device" ]; + path = [ + pkgs.iproute2 + pkgs.iw + ]; + preStart = '' + ip netns add sta + iw phy phy1 set netns name sta + ip netns exec sta ip link set lo up + mkdir -p /run/wpa_supplicant/client # nixpkgs' wpa_cli keeps its sockets here + ''; + serviceConfig.ExecStart = + "${pkgs.iproute2}/bin/ip netns exec sta ${pkgs.wpa_supplicant}/bin/wpa_supplicant -i wlan1 -c " + + pkgs.writeText "sta.conf" '' + ctrl_interface=/run/wpa_supplicant/control + network={ + ssid="home" + key_mgmt=SAE + sae_password="mock-prompt-value-passphrase" + ieee80211w=2 + scan_freq=2437 + } + ''; + }; + environment.systemPackages = [ + pkgs.wpa_supplicant + (pkgs.writeShellScriptBin "sta-dhcp" '' + # One DHCP round on the station, applying the offered address. + exec ${pkgs.iproute2}/bin/ip netns exec sta ${pkgs.busybox}/bin/udhcpc -i wlan1 -n -q -f \ + -s ${pkgs.writeShellScript "udhcpc-apply" '' + case "$1" in + bound|renew) ${pkgs.iproute2}/bin/ip addr replace "$ip/$mask" dev "$interface" ;; + esac + ''} + '') + ]; + }; + + isp = { + virtualisation.interfaces.wan = { + vlan = 1; + assignIP = false; + }; + # PPPoE access concentrator: one session, peer gets 192.0.2.10. + systemd.services.pppoe-server = { + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + serviceConfig.ExecStart = + "${pkgs.rp-pppoe}/sbin/pppoe-server -F -O /etc/ppp/pppoe-server-options" + + " -q ${pkgs.ppp}/sbin/pppd -I wan -L ${ispAddress} -R 192.0.2.10"; + }; + environment.etc = { + "ppp/pppoe-server-options".text = '' + plugin pppoe.so + require-chap + lcp-echo-interval 10 + lcp-echo-failure 2 + nobsdcomp + noccp + novj + ''; + "ppp/chap-secrets" = { + text = chapSecrets; + mode = "0640"; + }; + }; + }; + + client = { + virtualisation.interfaces.trunk = { + vlan = 2; + assignIP = false; + }; + networking.useDHCP = false; + networking.vlans = { + lan0 = { + id = 20; + interface = "trunk"; + }; + iot0 = { + id = 40; + interface = "trunk"; + }; + }; + networking.interfaces.lan0 = { + useDHCP = true; + macAddress = clientMac; + }; + networking.interfaces.iot0.useDHCP = true; + # Dual-homed on purpose (lan + iot). Both leases bring a default route: + # prefer lan0 so replies to other VLANs and the WAN leave where the + # router expects them (its rp-filter would drop them on vlan-iot), and + # loosen this client's own rp-filter so a WAN reply on iot0 would be + # seen — the negative allowWan check must fail at the router, not here. + systemd.network.networks."40-iot0".dhcpV4Config.RouteMetric = 2048; + networking.firewall.checkReversePath = "loose"; + environment.systemPackages = [ + pkgs.dnsutils + pkgs.netcat + ]; + }; + + admin = { + virtualisation.interfaces.access = { + vlan = 3; + assignIP = false; + }; + networking.useDHCP = false; + networking.interfaces.access = { + useDHCP = true; + macAddress = adminMac; + }; + environment.systemPackages = [ pkgs.netcat ]; + }; + }; + + testScript = '' + start_all() + + with subtest("PPPoE session comes up with the vars credentials"): + gw.wait_for_unit("pppd-wan.service") + gw.wait_until_succeeds("ping -c1 -W1 ${ispAddress}") + + with subtest("DHCP hands out reserved leases per VLAN"): + gw.wait_for_unit("kea-dhcp4-server.service") + client.wait_until_succeeds("ip -4 addr show lan0 | grep -q 'inet ${clientAddress}/24'") + client.wait_until_succeeds("ip -4 addr show iot0 | grep -q 'inet 10.9.40.1[0-9][0-9]/24'") + admin.wait_until_succeeds("ip -4 addr show access | grep -q 'inet ${adminAddress}/24'") + + with subtest("Blocky serves the VLAN and blocks the denylist"): + gw.wait_for_unit("blocky.service") + answer = client.wait_until_succeeds("dig +short +time=2 @10.9.20.1 ads.example.com") + assert answer.strip() == "0.0.0.0", f"expected blocked answer, got: {answer!r}" + + with subtest("NAT to the WAN only for VLANs with allowWan"): + client.wait_until_succeeds("ping -c1 -W1 -I lan0 ${ispAddress}") + # iot has a route to the WAN; the router is what refuses to forward. + client.succeed("ip route show dev iot0 | grep -q ^default") + client.fail("ping -c1 -W2 -I iot0 ${ispAddress}") + + with subtest("mgmt reaches other VLANs, other VLANs do not"): + admin.succeed("ping -c1 -W2 ${clientAddress}") + client.fail("ping -c1 -W2 -I lan0 ${adminAddress}") + + with subtest("SSH on the router only from mgmt"): + gw.wait_for_open_port(22) + admin.succeed("nc -z -w2 10.9.10.1 22") + client.fail("nc -z -w2 10.9.20.1 22") + + with subtest("Wireless SSIDs are bridge access ports of their VLAN"): + gw.wait_for_unit("hostapd.service") + # Second SSID: derived BSSID, hidden, bridged into the iot VLAN. + gw.wait_until_succeeds("ip link show wlan0-1 | grep -q '06:00:00:00:00:00'") + gw.wait_until_succeeds("bridge vlan show dev wlan0-1 | grep -q '40 PVID Egress Untagged'") + gw.succeed("hostapd_cli -i wlan0-1 get_config | grep -q '^ssid=things'") + # The second BSS section of the rendered config carries hidden + isolation. + things = gw.succeed("sed -n '/^bss=wlan0-1/,$p' /run/hostapd/wlan0.hostapd.conf") + assert "ignore_broadcast_ssid=1" in things and "ap_isolate=1" in things, things + assert "sae_password=mock-prompt-value-passphrase" not in things, "wpa2 SSID must not carry SAE entries" + # A WPA3 (SAE) station on the first SSID authenticates with the vars + # passphrase and gets its lease from Kea on the lan VLAN, through the bridge. + gw.wait_for_unit("wifi-station.service") + gw.wait_until_succeeds("ip netns exec sta wpa_cli -i wlan1 status | grep -q wpa_state=COMPLETED") + gw.succeed("timeout 60 sta-dhcp") + gw.succeed("ip netns exec sta ip -4 addr show wlan1 | grep -q 'inet 10.9.20.1[0-9][0-9]/24'") + ''; +} diff --git a/modules/clan/router/wifi.nix b/modules/clan/router/wifi.nix new file mode 100644 index 0000000..8b21768 --- /dev/null +++ b/modules/clan/router/wifi.nix @@ -0,0 +1,206 @@ +# Wireless access point on the router itself (hostapd). Every SSID is a BSS +# interface that joins the VLAN bridge as an untagged access port of its VLAN, +# so wireless clients get exactly the same DHCP/DNS/firewall treatment as a +# wired port in that VLAN. Passphrases are vars prompts, never in the store. +# +# hostapd names BSS interfaces , -1, -2 ... and wants a +# fixed BSSID for every extra one; they are derived from the radio's hardware +# address by setting the locally-administered bit and flipping bits 2-3 of +# the first octet per index (02 -> 06, 0a, 0e), which never collides with the +# radio's own address. +{ settings }: +{ + config, + lib, + pkgs, + ... +}: +let + cfg = settings; + wifi = cfg.wifi; + + bssIf = radio: i: if i == 0 then radio else "${radio}-${toString i}"; + + hexDigit = + c: + { + "0" = 0; + "1" = 1; + "2" = 2; + "3" = 3; + "4" = 4; + "5" = 5; + "6" = 6; + "7" = 7; + "8" = 8; + "9" = 9; + a = 10; + b = 11; + c = 12; + d = 13; + e = 14; + f = 15; + } + .${lib.toLower c}; + octetToInt = s: 16 * hexDigit (builtins.substring 0 1 s) + hexDigit (builtins.substring 1 1 s); + intToOctet = n: lib.toLower (lib.fixedWidthString 2 "0" (lib.toHexString n)); + deriveBssid = + mac: i: + let + octets = lib.splitString ":" mac; + first = builtins.bitXor (builtins.bitOr (octetToInt (builtins.head octets)) 2) (i * 4); + in + lib.concatStringsSep ":" ([ (intToOctet first) ] ++ builtins.tail octets); + + # Every BSS of every radio, flattened. + bsses = lib.concatLists ( + lib.mapAttrsToList ( + radio: r: + lib.imap0 (i: name: { + inherit radio name; + index = i; + iface = bssIf radio i; + net = wifi.networks.${name} or null; + }) r.networks + ) wifi.radios + ); + + referenced = lib.unique (map (b: b.name) bsses); + secured = lib.filter (name: wifi.networks.${name}.security != "open") ( + lib.filter (name: wifi.networks ? ${name}) referenced + ); + passphraseFile = + name: config.clan.core.vars.generators."wifi-${name}-passphrase".files.passphrase.path; + + authentication = + name: net: + { + wpa3 = { + mode = "wpa3-sae"; + saePasswordsFile = passphraseFile name; + }; + wpa3-transition = { + mode = "wpa3-sae-transition"; + saePasswordsFile = passphraseFile name; + wpaPasswordFile = passphraseFile name; + }; + wpa2 = { + mode = "wpa2-sha1"; + wpaPasswordFile = passphraseFile name; + }; + open.mode = "none"; + } + .${net.security}; +in +{ + config = lib.mkIf wifi.enable { + assertions = [ + { + assertion = wifi.radios != { }; + message = "router: wifi.enable needs at least one radio in wifi.radios."; + } + { + assertion = wifi.countryCode != null; + message = "router: wifi.countryCode is required when wifi.enable is set (regulatory domain)."; + } + { + assertion = lib.all (b: b.net != null) bsses; + message = "router: every wifi.radios..networks entry must name a network in wifi.networks."; + } + { + assertion = lib.all (b: b.net == null || cfg.vlans ? ${b.net.vlan}) bsses; + message = "router: every wifi.networks..vlan must name a VLAN in vlans."; + } + { + assertion = lib.all (r: lib.length r.networks <= 4) (lib.attrValues wifi.radios); + message = "router: a radio can broadcast at most four networks."; + } + { + assertion = lib.all (r: lib.length r.networks <= 1 || r.macAddress != null) ( + lib.attrValues wifi.radios + ); + message = "router: wifi.radios..macAddress is required for radios broadcasting more than one network."; + } + ]; + + # Regulatory database for the kernel, so countryCode actually applies. + hardware.wirelessRegulatoryDatabase = true; + + # The kernel refuses to bridge a wireless interface in station mode, and + # networkd stops retrying before hostapd switches the radio to AP mode; + # so put it in AP mode the moment it appears (kernel name or the renamed + # one, whichever the user configured). hostapd finds it already there. + services.udev.extraRules = lib.concatMapStrings ( + radio: + let + run = ''RUN+="${pkgs.iw}/bin/iw dev ${radio} set type __ap"''; + in + '' + ACTION=="add", SUBSYSTEM=="net", KERNEL=="${radio}", ${run} + ACTION=="add", SUBSYSTEM=="net", NAME=="${radio}", ${run} + '' + ) (lib.attrNames wifi.radios); + + clan.core.vars.generators = lib.genAttrs (map (name: "wifi-${name}-passphrase") secured) ( + gen: + let + name = lib.removeSuffix "-passphrase" (lib.removePrefix "wifi-" gen); + in + { + prompts.passphrase = { + description = "Wi-Fi passphrase for SSID \"${wifi.networks.${name}.ssid}\" (8-63 ASCII characters)"; + type = "hidden"; + }; + files.passphrase.secret = true; + # No trailing newline: hostapd turns every line of the file into a + # sae_password entry, and an empty one wipes the list. + script = ''printf '%s' "$(cat "$prompts"/passphrase)" > "$out"/passphrase''; + } + ); + + services.hostapd = { + enable = true; + radios = lib.mapAttrs (radio: r: { + inherit (r) band channel; + inherit (wifi) countryCode; + wifi6.enable = r.wifi6; + networks = lib.listToAttrs ( + map ( + b: + lib.nameValuePair b.iface ( + { + inherit (b.net) ssid; + ignoreBroadcastSsid = if b.net.hidden then "empty" else "disabled"; + apIsolate = b.net.isolateClients; + authentication = authentication b.name b.net; + } + // lib.optionalAttrs (lib.length r.networks > 1) { + bssid = if b.index == 0 then r.macAddress else deriveBssid r.macAddress b.index; + } + ) + ) (lib.filter (b: b.radio == radio) bsses) + ); + }) wifi.radios; + }; + + # Each BSS is an untagged access port of its VLAN on br0 (cf. accessPorts + # in network.nix); networkd enslaves the interface once hostapd creates it. + systemd.network.networks = lib.listToAttrs ( + map ( + b: + lib.nameValuePair "27-wifi-${b.iface}" { + matchConfig.Name = b.iface; + networkConfig.Bridge = "br0"; + bridgeVLANs = [ + { + VLAN = cfg.vlans.${b.net.vlan}.id; + PVID = cfg.vlans.${b.net.vlan}.id; + EgressUntagged = cfg.vlans.${b.net.vlan}.id; + } + ]; + linkConfig.RequiredForOnline = "no"; + } + ) bsses + ); + }; +} diff --git a/modules/dns/acme-gw-secret.nix b/modules/dns/acme-gw-secret.nix deleted file mode 100644 index 4dfa778..0000000 --- a/modules/dns/acme-gw-secret.nix +++ /dev/null @@ -1,17 +0,0 @@ -# Shared TSIG secret for a gateway's dedicated ACME key (function: machine -# name -> module). The acme_gw_ key lets that gateway — and only it — write -# _acme-challenge.