From 158252323f4935a98a6c96b372a1ae21449d7a7e Mon Sep 17 00:00:00 2001 From: Berwn Date: Tue, 28 Jul 2026 17:06:07 +0700 Subject: [PATCH] Add site gateway role (modules/router) and gw-cnx-1 Reusable cnx.router.* module for the Topton 1U boxes replacing OPNsense: PPPoE WAN (optionally VLAN-tagged, AIS: 10, secret credentials incl. username), VLAN-filtering bridge, nftables NAT/firewall with MSS clamp, Kea DHCP with per-VLAN lease time, Blocky DNS, DHCPv6-PD, CrowdSec with the ZeroTier mesh whitelisted, optional Omada controller, ZFS disk. Fleet baseline rides along: admins sops group is now derived for every machine in clan.nix (secrets encrypt to it from the first vars generate) and time sync is chrony everywhere instead of systemd-timesyncd. --- clan.nix | 32 +++- docs/src/SUMMARY.md | 2 + docs/src/gateways.md | 100 +++++++++++ docs/src/overview.md | 23 ++- machines/control/configuration.nix | 4 +- machines/gw-cnx-1/configuration.nix | 41 +++++ machines/gw-cnx-1/disko.nix | 78 +++++++++ machines/mx1/configuration.nix | 4 +- machines/ns1/configuration.nix | 4 +- machines/ns2/configuration.nix | 4 +- machines/web01/configuration.nix | 4 +- modules/mesh-hosts.nix | 1 + modules/monitoring/server.nix | 7 + modules/router/crowdsec.nix | 52 ++++++ modules/router/default.nix | 253 ++++++++++++++++++++++++++++ modules/router/dns-dhcp.nix | 79 +++++++++ modules/router/firewall.nix | 62 +++++++ modules/router/ipv6.nix | 31 ++++ modules/router/omada.nix | 43 +++++ modules/router/pppoe.nix | 64 +++++++ 20 files changed, 857 insertions(+), 31 deletions(-) create mode 100644 docs/src/gateways.md create mode 100644 machines/gw-cnx-1/configuration.nix create mode 100644 machines/gw-cnx-1/disko.nix create mode 100644 modules/router/crowdsec.nix create mode 100644 modules/router/default.nix create mode 100644 modules/router/dns-dhcp.nix create mode 100644 modules/router/firewall.nix create mode 100644 modules/router/ipv6.nix create mode 100644 modules/router/omada.nix create mode 100644 modules/router/pppoe.nix diff --git a/clan.nix b/clan.nix index 15cc8ea..5ab0d5e 100644 --- a/clan.nix +++ b/clan.nix @@ -1,18 +1,27 @@ let hosts = import ./modules/hosts.nix; + + # Single source of the machine list: inventory AND the per-machine baseline + # below are both derived from it, so no machine can be added without the + # baseline (e.g. admins group encryption for all its generated secrets). + fleet = { + control = { }; + ns1 = { }; + ns2 = { }; + mx1 = { }; + web01 = { }; + # Site gateways (Topton 1U routers): dynamic PPPoE WAN, so they are NOT in + # modules/hosts.nix / the `internet` instance — clan reaches them over the + # zerotier mesh (or Tor) instead. + gw-cnx-1 = { }; + }; in { # Ensure this is unique among all clans you want to use. meta.name = "cnx-network-clan"; meta.domain = "cnx-network.internal"; - inventory.machines = { - control = { }; - ns1 = { }; - ns2 = { }; - mx1 = { }; - web01 = { }; - }; + inventory.machines = fleet; inventory.instances = { @@ -65,6 +74,11 @@ in }; }; - machines = { - }; + # Fleet-wide baseline applied to every machine. Secrets minted by + # `clan vars generate` are encrypted for the admins group from the very + # first run — generating before this took effect is what forced the + # re-encryption dance (`clan vars fix`) on gw-cnx-1. + machines = builtins.mapAttrs (_: _: { + clan.core.sops.defaultGroups = [ "admins" ]; + }) fleet; } diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index 45be790..22333b6 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -3,5 +3,7 @@ - [Overview](./overview.md) - [ZeroTier mesh](./mesh.md) - [DNS](./dns.md) +- [Mail](./mail.md) +- [Site gateways](./gateways.md) - [Monitoring](./monitoring.md) - [Backups](./backups.md) diff --git a/docs/src/gateways.md b/docs/src/gateways.md new file mode 100644 index 0000000..5d0dc2e --- /dev/null +++ b/docs/src/gateways.md @@ -0,0 +1,100 @@ +# Site gateways + +NixOS routers (OPNsense replacements) on Topton 1U fanless boxes — Intel N300, +4x i226-V 2.5G ports (some units add 2x 10G SFP+), 16GB DDR5, 256GB mSATA. +Four sites are planned; the first is `gw-cnx-1` (Chiang Mai). Everything lives +in the reusable `modules/router/` module (`cnx.router.*` options); a site's +machine config only sets port names, VLANs, and feature flags. + +Naming: `gw--`, e.g. `gw-cnx-1`. + +## What each gateway runs + +| Function | Implementation | +| ------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| WAN | PPPoE (`pppd`), per-site ISP credentials via clan vars prompts; `wan.vlanId` when the ISP tags the session (AIS: 10); `wan.macAddress` to clone the old router's MAC if the ISP has it pinned | +| LAN | VLAN-filtering bridge `br0` over the trunk ports (networkd) | +| Firewall/NAT | nftables: default-deny WAN, no inter-VLAN, MSS clamp, v4 NAT | +| DHCP | Kea, one subnet per VLAN | +| DNS | Blocky (blocklist resolver), metrics on :4000 scraped by control | +| IPv6 | DHCPv6-PD on ppp0, /64 per VLAN via SLAAC | +| Bans | CrowdSec + nftables bouncer (sshd log parsing) | +| Omada | Optional per site: TP-Link Omada controller as a podman container | +| Management | ZeroTier mesh: SSH, node_exporter, journald upload — like the fleet | +| Storage | Single-disk ZFS (zstd, `/var` its own dataset for snapshots) | + +## Addressing plan + +Each site owns `10..0.0/16`. Defaults per VLAN: subnet +`10...0/24`, router at `.1`, DHCP pool `.100–.199` +(`.2–.99` static/infra, `.200–.254` reserved). + +VLAN id convention (fleet-wide): **10 = mgmt**, **20 = lan** (both mandatory at +every site), 30 = guest, 40 = iot (reserved). Sites add their own beyond that. + +Larger subnets (e.g. public-WiFi guest networks that outgrow a /24) are carved +from the **upper half** `10..128.0/17` and set explicitly on the VLAN. +The lower half stays reserved for /24s indexed by VLAN id. High-churn VLANs +should also shorten `dhcp.leaseTime` (default 86400 s) so the pool recycles. +First user: `gw-cnx-2` (site 2) runs the public WiFi — guest VLAN 30 at +`10.2.128.0/22`, pool `10.2.128.100 – 10.2.131.250`, `dhcp.leaseTime = 3600`. + +| Site | siteId | mgmt | lan | +| ---- | ------ | -------------- | -------------- | +| cnx | 1 | `10.1.10.0/24` | `10.1.20.0/24` | + +Trust model: mgmt → everything; other VLANs → router DNS/DHCP + internet only +(no inter-VLAN); WAN → nothing inbound; mesh → SSH, metrics, Omada UI. + +## Adding a site + +1. Copy `machines/gw-cnx-1/` to `machines/gw--/`; set `site`, + `siteId` (next free number), port names, VLANs, and `omada.enable`. + Fill the real disk id in `disko.nix` (from the installer: + `ls -l /dev/disk/by-id/`). Keep the `mgmt`/`lan` VLANs. +2. Add the machine to `inventory.machines` in `clan.nix` and to the machine + list in `modules/mesh-hosts.nix`. Do **not** add it to `modules/hosts.nix` + (dynamic PPPoE IP; clan connects over the mesh). +3. Add its node_exporter and Blocky scrape targets in + `modules/monitoring/server.nix`. +4. `git add` the new machine directory — flake evaluation only sees + git-tracked files, so an untracked `machines/gw-…/` is silently ignored. +5. `clan vars generate gw--` — prompts for the site's PPPoE + credentials, mints the ZeroTier identity etc. (`nix flake check` fails + until this has run, because mesh-hosts reads the ZeroTier IP var.) +6. Boot the box from a NixOS installer USB on the local network, then: + `clan machines install gw-- --target-host root@` +7. Check `facter.json` for the real NIC names, fix `wan.interface` / + `trunkPorts` if the enumeration differs, and + `clan machines update gw--` (rides the mesh from then on). +8. Add a row to the site table above and to the machines table in + [Overview](./overview.md). + +## Omada controller + +Sites with TP-Link Omada APs/switches set `cnx.router.omada.enable = true`. +The controller runs as a podman container (`mbentley/omada-controller` — there +is no nixpkgs package) with host networking on the mgmt VLAN, where adoption +broadcasts (UDP 29810) live. UI: `https://:8043` from mgmt, or the +gateway's mesh address from an admin machine. Controller state is under +`/var/lib/omada`, declared as clan state (`clan.core.state.omada`); wiring it +into the borgbackup instance is still a follow-up. + +## Runbook + +- **PPPoE down**: `systemctl status pppd-wan`, `journalctl -u pppd-wan` on the + gateway (over the mesh). ISP credentials live in the `pppoe-credentials` + vars generator; re-enter with `clan vars generate gw-- --regenerate`. +- **A VLAN gets no leases**: `systemctl status kea-dhcp4-server`; check the + port's `bridgeVLANs` tagging and that the switch trunk carries the VLAN. +- **DNS/blocklist issues**: Blocky metrics are in VictoriaMetrics (job + `blocky`); `journalctl -u blocky` on the gateway. +- **Banned yourself**: `cscli decisions list` / `cscli decisions delete --ip ` + on the gateway. The ZeroTier mesh `/88` is whitelisted at the parser stage + (`cnx/mesh-whitelist`), so admin access over the mesh can never be banned. + +## Follow-ups + +- Borgbackup client for Omada + Kea lease state. +- Suricata (IDS-only) if CPU headroom allows — deliberately skipped for now. +- Remaining three sites. diff --git a/docs/src/overview.md b/docs/src/overview.md index fc4ba22..2abd11a 100644 --- a/docs/src/overview.md +++ b/docs/src/overview.md @@ -6,13 +6,14 @@ this book is built from `docs/` and served on `control` over the ZeroTier mesh. ## Machines -| Machine | Role | Public IPv4 | Public IPv6 | -| --------- | -------------------------------------- | ---------------- | ----------------------- | -| `control` | ZeroTier controller, monitoring, docs | `77.42.68.181` | `2a01:4f9:c013:e6d0::1` | -| `ns1` | Knot DNS **primary** (master) | `46.224.170.206` | `2a01:4f8:c014:b5c5::1` | -| `ns2` | Knot DNS **secondary** (slave) | `157.180.70.82` | `2a01:4f9:c014:6d87::1` | -| `mx1` | Mail server (**MX** for cnx.email) | `5.223.65.38` | `2a01:4ff:2f0:1963::1` | -| `web01` | Public reverse proxy (TLS termination) | `5.223.55.246` | `2a01:4ff:2f0:2d8f::1` | +| Machine | Role | Public IPv4 | Public IPv6 | +| ---------- | -------------------------------------- | ---------------- | ----------------------- | +| `control` | ZeroTier controller, monitoring, docs | `77.42.68.181` | `2a01:4f9:c013:e6d0::1` | +| `ns1` | Knot DNS **primary** (master) | `46.224.170.206` | `2a01:4f8:c014:b5c5::1` | +| `ns2` | Knot DNS **secondary** (slave) | `157.180.70.82` | `2a01:4f9:c014:6d87::1` | +| `mx1` | Mail server (**MX** for cnx.email) | `5.223.65.38` | `2a01:4ff:2f0:1963::1` | +| `web01` | Public reverse proxy (TLS termination) | `5.223.55.246` | `2a01:4ff:2f0:2d8f::1` | +| `gw-cnx-1` | Site gateway Chiang Mai (router) | dynamic (PPPoE) | — | ## Access @@ -21,6 +22,14 @@ this book is built from `docs/` and served on `control` over the ZeroTier mesh. - clan reaches machines by their public IPs first (the `internet` instance), with the mesh and Tor as automatic fallbacks. +## Host baseline + +Every machine sets a fixed-offset `time.timeZone` and runs **chrony** for time +sync (`services.chrony.enable`). chrony is the single NTP implementation across +the fleet — do **not** use `systemd-timesyncd`; the two are mutually exclusive +and we standardise on chrony everywhere. New hosts must set both the timezone and +chrony as part of their baseline config. + ## Editing these docs Commit-to-edit: change the markdown under `docs/src/`, commit, and redeploy diff --git a/machines/control/configuration.nix b/machines/control/configuration.nix index 55d558f..60fdb80 100644 --- a/machines/control/configuration.nix +++ b/machines/control/configuration.nix @@ -14,8 +14,6 @@ in ../../modules/docs.nix ]; - clan.core.sops.defaultGroups = [ "admins" ]; - # Public IPv6 (from modules/hosts.nix); SLAAC doesn't bring it up here. cnx.staticIPv6 = { enable = true; @@ -23,7 +21,7 @@ in }; time.timeZone = "Etc/GMT-3"; # UTC+3 (fixed offset, no DST) - services.timesyncd.enable = true; + services.chrony.enable = true; # Public Hetzner Cloud firewalls, synced from this config on every deploy. # Rules live in their own data file; see that file for the no-public-SSH note. diff --git a/machines/gw-cnx-1/configuration.nix b/machines/gw-cnx-1/configuration.nix new file mode 100644 index 0000000..fc4dbb6 --- /dev/null +++ b/machines/gw-cnx-1/configuration.nix @@ -0,0 +1,41 @@ +# Site gateway Chiang Mai (site 1): Topton 1U, Intel N300, 4x i226-V 2.5G. +# Port roles below use the expected igc names — verify against facter.json +# after the first install and adjust if the box enumerates differently. +{ config, lib, ... }: +{ + imports = [ + ../../modules/router + ../../modules/monitoring/exporters.nix + ]; + + # Until the install generates facter.json (which normally provides this). + nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; + + # ZFS (disko.nix) needs a stable machine-unique hostId; derive it from the + # hostname so every gateway gets one for free when copied for a new site. + networking.hostId = builtins.substring 0 8 ( + builtins.hashString "sha256" config.networking.hostName + ); + + cnx.router = { + enable = true; + site = "cnx"; + siteId = 1; + wan.interface = "enp1s0"; + wan.vlanId = 10; # AIS delivers PPPoE tagged on VLAN 10 + trunkPorts = [ + "enp2s0" + "enp3s0" + "enp4s0" + ]; + vlans = { + mgmt.id = 10; # 10.1.10.0/24 — APs, switches, Omada, admin + lan.id = 20; # 10.1.20.0/24 — trusted clients + }; + # This site runs the Omada controller for its APs/switches. + omada.enable = true; + }; + + time.timeZone = "Etc/GMT-7"; # UTC+7 (Thailand, fixed offset, no DST) + services.chrony.enable = true; +} diff --git a/machines/gw-cnx-1/disko.nix b/machines/gw-cnx-1/disko.nix new file mode 100644 index 0000000..4efaf32 --- /dev/null +++ b/machines/gw-cnx-1/disko.nix @@ -0,0 +1,78 @@ +# Single-disk ZFS layout for the Topton's 256GB mSATA SSD: vfat ESP for boot, +# the rest a zpool (zstd compression, no atime). Single disk = no redundancy; +# ZFS buys us compression, snapshots, and checksumming. Requires +# networking.hostId (set in configuration.nix). The device is a placeholder: +# boot the installer, read the real id from `ls -l /dev/disk/by-id/`, and fill +# it in before `clan machines install`. Changing the layout later requires +# wiping and reinstalling. +{ + + boot.loader.grub.efiSupport = true; + boot.loader.grub.efiInstallAsRemovable = true; + boot.loader.grub.enable = true; + disko.devices = { + disk = { + main = { + name = "main-gw-cnx-1"; + device = "/dev/disk/by-id/CHANGE-ME-msata-ssd"; + type = "disk"; + content = { + type = "gpt"; + partitions = { + "boot" = { + size = "1M"; + type = "EF02"; # for grub MBR + priority = 1; + }; + ESP = { + type = "EF00"; + size = "500M"; + content = { + type = "filesystem"; + format = "vfat"; + mountpoint = "/boot"; + mountOptions = [ "umask=0077" ]; + }; + }; + zfs = { + size = "100%"; + content = { + type = "zfs"; + pool = "rpool"; + }; + }; + }; + }; + }; + }; + zpool = { + rpool = { + type = "zpool"; + options.ashift = "12"; + rootFsOptions = { + compression = "zstd"; + acltype = "posixacl"; + xattr = "sa"; + atime = "off"; + mountpoint = "none"; + }; + datasets = { + root = { + type = "zfs_fs"; + mountpoint = "/"; + }; + nix = { + type = "zfs_fs"; + mountpoint = "/nix"; + }; + # Service state (Omada, Kea leases, CrowdSec db, journald) — its own + # dataset so it can be snapshotted/sent independently of the OS. + var = { + type = "zfs_fs"; + mountpoint = "/var"; + }; + }; + }; + }; + }; +} diff --git a/machines/mx1/configuration.nix b/machines/mx1/configuration.nix index 36e71a0..d8ddb53 100644 --- a/machines/mx1/configuration.nix +++ b/machines/mx1/configuration.nix @@ -10,8 +10,6 @@ in ../../modules/monitoring/exporters.nix ]; - clan.core.sops.defaultGroups = [ "admins" ]; - # Public IPv6 (from modules/hosts.nix); SLAAC doesn't bring it up here. cnx.staticIPv6 = { enable = true; @@ -19,5 +17,5 @@ in }; time.timeZone = "Etc/GMT-8"; # UTC+8 (Singapore, fixed offset, no DST) - services.timesyncd.enable = true; + services.chrony.enable = true; } diff --git a/machines/ns1/configuration.nix b/machines/ns1/configuration.nix index 578ec21..d5606a4 100644 --- a/machines/ns1/configuration.nix +++ b/machines/ns1/configuration.nix @@ -18,8 +18,6 @@ in ../../modules/monitoring/exporters.nix ]; - clan.core.sops.defaultGroups = [ "admins" ]; - # Knot's state dir holds the non-regenerable DNSSEC key material (KSK/ZSK # private keys in the KASP keystore). Declaring it as clan state makes the # borgbackup client back it up; losing it forces an emergency DS rollover at @@ -38,7 +36,7 @@ in }; time.timeZone = "Etc/GMT-1"; # UTC+1 (fixed offset, no DST) - services.timesyncd.enable = true; + services.chrony.enable = true; # ACME DNS-01 (RFC 2136), general key. A dedicated TSIG key scoped by acl_acme # (referenced by every zone below) to TXT updates at or under _acme-challenge. diff --git a/machines/ns2/configuration.nix b/machines/ns2/configuration.nix index 4b613fa..5207310 100644 --- a/machines/ns2/configuration.nix +++ b/machines/ns2/configuration.nix @@ -10,8 +10,6 @@ in ../../modules/monitoring/exporters.nix ]; - clan.core.sops.defaultGroups = [ "admins" ]; - # Public IPv6 (from modules/hosts.nix; matches the ns2 AAAA glue); SLAAC # doesn't bring it up here. cnx.staticIPv6 = { @@ -20,7 +18,7 @@ in }; time.timeZone = "Etc/GMT-3"; # UTC+3 (fixed offset, no DST) - services.timesyncd.enable = true; + services.chrony.enable = true; # ns2 = secondary (slave): pulls every zone from ns1 and accepts its NOTIFY. services.knot.settings.zone = map (d: { diff --git a/machines/web01/configuration.nix b/machines/web01/configuration.nix index 0a8dab3..99f986e 100644 --- a/machines/web01/configuration.nix +++ b/machines/web01/configuration.nix @@ -9,8 +9,6 @@ in ../../modules/web-proxy.nix ]; - clan.core.sops.defaultGroups = [ "admins" ]; - # Public IPv6 (from modules/hosts.nix); SLAAC doesn't bring it up here. cnx.staticIPv6 = { enable = true; @@ -18,5 +16,5 @@ in }; time.timeZone = "Etc/GMT-8"; # UTC+8 (Singapore, fixed offset, no DST) - services.timesyncd.enable = true; + services.chrony.enable = true; } diff --git a/modules/mesh-hosts.nix b/modules/mesh-hosts.nix index bfe738b..c969373 100644 --- a/modules/mesh-hosts.nix +++ b/modules/mesh-hosts.nix @@ -23,6 +23,7 @@ let "ns2" "mx1" "web01" + "gw-cnx-1" ] readIp; # RFC 4193 prefix of this ZeroTier network: fd + the 8-byte network id + the diff --git a/modules/monitoring/server.nix b/modules/monitoring/server.nix index 71f5842..aff0ba2 100644 --- a/modules/monitoring/server.nix +++ b/modules/monitoring/server.nix @@ -48,6 +48,13 @@ in (target "ns2" (v6 mesh.hosts.ns2) 9100) (target "mx1" (v6 mesh.hosts.mx1) 9100) (target "web01" (v6 mesh.hosts.web01) 9100) + (target "gw-cnx-1" (v6 mesh.hosts.gw-cnx-1) 9100) + ]; + } + { + job_name = "blocky"; + static_configs = [ + (target "gw-cnx-1" (v6 mesh.hosts.gw-cnx-1) 4000) ]; } { diff --git a/modules/router/crowdsec.nix b/modules/router/crowdsec.nix new file mode 100644 index 0000000..1189db4 --- /dev/null +++ b/modules/router/crowdsec.nix @@ -0,0 +1,52 @@ +# CrowdSec security engine + nftables bouncer: parses sshd auth attempts from +# the journal and bans offending source IPs at the firewall. Log-based (no +# inline DPI) so it costs the N300 next to nothing. +{ + config, + lib, + ... +}: +let + cfg = config.cnx.router; + mesh = import ../mesh-hosts.nix { inherit config lib; }; +in +{ + config = lib.mkIf cfg.enable { + services.crowdsec = { + enable = true; + autoUpdateService = true; + hub.collections = [ + "crowdsecurity/linux" + "crowdsecurity/sshd" + ]; + localConfig = { + acquisitions = [ + { + source = "journalctl"; + journalctl_filter = [ "_SYSTEMD_UNIT=sshd.service" ]; + labels.type = "syslog"; + } + ]; + # Never ban the ZeroTier mesh — it is the only admin path to these + # boxes (no public SSH), so a false positive would lock us out. + # Parser-stage whitelist: mesh events are dropped before any scenario. + parsers.s02Enrich = [ + { + name = "cnx/mesh-whitelist"; + description = "Whitelist the ZeroTier management mesh"; + whitelist = { + reason = "ZeroTier mesh is the admin path"; + cidr = [ mesh.subnet ]; + }; + } + ]; + }; + }; + + services.crowdsec-firewall-bouncer = { + enable = true; + registerBouncer.enable = true; + settings.mode = "nftables"; + }; + }; +} diff --git a/modules/router/default.nix b/modules/router/default.nix new file mode 100644 index 0000000..1f5ca70 --- /dev/null +++ b/modules/router/default.nix @@ -0,0 +1,253 @@ +# Site gateway (OPNsense replacement) for the Topton 1U boxes: PPPoE WAN, +# VLAN-filtering bridge over the LAN ports, and per-VLAN L3 interfaces. +# Imported by machines/gw--; everything is driven by cnx.router.*. +# +# Fleet addressing convention: each site owns 10..0.0/16. A VLAN's +# subnet defaults to 10...0/24 with the router at .1 and the +# DHCP pool at .100-.199. VLANs that need more space (e.g. public-wifi guest) +# override `subnet`/`address`/`dhcp.pool` and take a wider block from the +# upper half (10..128.0/17), e.g. guest -> 10..128.0/22. +# VLAN ids: 10 = mgmt, 20 = lan (mandatory); 30 = guest, 40 = iot (reserved). +{ + config, + lib, + ... +}: +let + cfg = config.cnx.router; + + vlanIf = name: "vlan-${name}"; + site = toString cfg.siteId; + + vlanModule = + { name, config, ... }: + let + octet = toString config.id; + in + { + options = { + id = lib.mkOption { + type = lib.types.ints.between 1 4094; + description = "802.1Q VLAN id (fleet convention: 10 mgmt, 20 lan, 30 guest, 40 iot)."; + }; + address = lib.mkOption { + type = lib.types.str; + default = "10.${site}.${octet}.1"; + description = "Router address on this VLAN."; + }; + prefixLength = lib.mkOption { + type = lib.types.ints.between 8 30; + default = 24; + }; + subnet = lib.mkOption { + type = lib.types.str; + default = "10.${site}.${octet}.0/24"; + description = "The VLAN's network in CIDR form (must contain `address`)."; + }; + dhcp = { + enable = lib.mkOption { + type = lib.types.bool; + default = true; + }; + pool = { + from = lib.mkOption { + type = lib.types.str; + default = "10.${site}.${octet}.100"; + }; + to = lib.mkOption { + type = lib.types.str; + default = "10.${site}.${octet}.199"; + }; + }; + leaseTime = lib.mkOption { + type = lib.types.ints.positive; + default = 86400; + description = '' + Lease validity in seconds. Lower it for high-churn networks, + e.g. public-WiFi guest VLANs (3600-7200), so the pool recycles. + ''; + }; + }; + allowWan = lib.mkOption { + type = lib.types.bool; + default = true; + description = "Whether clients on this VLAN may reach the internet."; + }; + }; + }; +in +{ + imports = [ + ./pppoe.nix + ./ipv6.nix + ./firewall.nix + ./dns-dhcp.nix + ./crowdsec.nix + ./omada.nix + ]; + + options.cnx.router = { + enable = lib.mkEnableOption "site gateway (router) role"; + + site = lib.mkOption { + type = lib.types.str; + description = "City code of the site, e.g. \"cnx\"."; + }; + + siteId = lib.mkOption { + type = lib.types.ints.between 1 254; + description = "Site number; drives the 10...0/24 addressing."; + }; + + wan.interface = lib.mkOption { + type = lib.types.str; + description = "Physical WAN port the PPPoE session runs on."; + }; + + wan.vlanId = lib.mkOption { + type = lib.types.nullOr (lib.types.ints.between 1 4094); + default = null; + description = '' + 802.1Q tag the ISP requires for the PPPoE session (AIS Thailand: 10); + null for untagged PPPoE directly on the port. Unrelated to the LAN + VLANs — this tag exists only on the WAN port. + ''; + }; + + wan.macAddress = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "aa:bb:cc:dd:ee:ff"; + description = '' + Spoofed MAC for the WAN port, e.g. to keep the MAC the ISP has + pinned (cloned from the old router). null keeps the hardware MAC. + ''; + }; + + wan.pppInterface = lib.mkOption { + type = lib.types.str; + internal = true; + readOnly = true; + default = if cfg.wan.vlanId == null then cfg.wan.interface else "wan-vlan"; + description = "Interface pppd dials on (the WAN port or its ISP VLAN)."; + }; + + trunkPorts = lib.mkOption { + type = lib.types.listOf lib.types.str; + description = "LAN ports carrying all VLANs tagged (incl. any 10G SFP+ ports)."; + }; + + vlans = lib.mkOption { + type = lib.types.attrsOf (lib.types.submodule vlanModule); + description = "VLANs served at this site; `mgmt` and `lan` are mandatory."; + }; + }; + + config = lib.mkIf cfg.enable { + assertions = [ + { + assertion = cfg.vlans ? mgmt && cfg.vlans ? lan; + message = "cnx.router: every site must define the `mgmt` and `lan` VLANs."; + } + ]; + + networking.useNetworkd = true; + networking.useDHCP = false; + systemd.network.enable = true; + + systemd.network.netdevs = { + "20-br0" = { + netdevConfig = { + Name = "br0"; + Kind = "bridge"; + }; + bridgeConfig.VLANFiltering = true; + }; + } + // lib.optionalAttrs (cfg.wan.vlanId != null) { + "15-wan-vlan" = { + netdevConfig = { + Name = "wan-vlan"; + Kind = "vlan"; + }; + vlanConfig.Id = cfg.wan.vlanId; + }; + } + // lib.mapAttrs' ( + name: vlan: + lib.nameValuePair "30-${vlanIf name}" { + netdevConfig = { + Name = vlanIf name; + Kind = "vlan"; + }; + vlanConfig.Id = vlan.id; + } + ) cfg.vlans; + + systemd.network.networks = + let + taggedAll = lib.mapAttrsToList (_: vlan: { VLAN = vlan.id; }) cfg.vlans; + in + { + # WAN port carries only the PPPoE session; no IP config of its own. + "10-wan" = { + matchConfig.Name = cfg.wan.interface; + networkConfig.LinkLocalAddressing = "no"; + vlan = lib.optional (cfg.wan.vlanId != null) "wan-vlan"; + linkConfig = { + RequiredForOnline = "carrier"; + } + # The wan-vlan subinterface (and thus the PPPoE session) inherits + # the parent port's MAC, so spoofing here covers both cases. + // lib.optionalAttrs (cfg.wan.macAddress != null) { + MACAddress = cfg.wan.macAddress; + }; + }; + } + // lib.optionalAttrs (cfg.wan.vlanId != null) { + # The ISP-side VLAN subinterface pppd dials on (e.g. AIS tags PPPoE). + "15-wan-vlan" = { + matchConfig.Name = "wan-vlan"; + networkConfig.LinkLocalAddressing = "no"; + linkConfig.RequiredForOnline = "no"; + }; + } + // { + # The bridge itself is L2-only; L3 lives on the vlan-* interfaces, + # which hang off the bridge (tagged on the bridge "self" port). + "20-br0" = { + matchConfig.Name = "br0"; + networkConfig.LinkLocalAddressing = "no"; + vlan = lib.mapAttrsToList (name: _: vlanIf name) cfg.vlans; + bridgeVLANs = taggedAll; + linkConfig.RequiredForOnline = "no"; + }; + } + // lib.listToAttrs ( + map (port: { + name = "25-trunk-${port}"; + value = { + matchConfig.Name = port; + networkConfig.Bridge = "br0"; + bridgeVLANs = taggedAll; + linkConfig.RequiredForOnline = "no"; + }; + }) cfg.trunkPorts + ) + // lib.mapAttrs' ( + name: vlan: + lib.nameValuePair "40-${vlanIf name}" { + matchConfig.Name = vlanIf name; + address = [ "${vlan.address}/${toString vlan.prefixLength}" ]; + networkConfig = { + IPv6AcceptRA = false; + # Announce a /64 carved from the DHCPv6-PD prefix on ppp0 (SLAAC). + IPv6SendRA = true; + DHCPPrefixDelegation = true; + }; + dhcpPrefixDelegationConfig.SubnetId = "auto"; + linkConfig.RequiredForOnline = "no"; + } + ) cfg.vlans; + }; +} diff --git a/modules/router/dns-dhcp.nix b/modules/router/dns-dhcp.nix new file mode 100644 index 0000000..98cf776 --- /dev/null +++ b/modules/router/dns-dhcp.nix @@ -0,0 +1,79 @@ +# LAN DHCP (Kea) and DNS (Blocky). Fully declarative: one Kea subnet per VLAN +# with dhcp.enable, Blocky as the blocklist resolver every DHCP lease points +# at. Blocky's HTTP listener (:4000) serves Prometheus metrics, scraped by +# control over the mesh (firewall.nix scopes it to the mesh subnet). +{ + config, + lib, + ... +}: +let + cfg = config.cnx.router; + dhcpVlans = lib.filterAttrs (_: vlan: vlan.dhcp.enable) cfg.vlans; +in +{ + config = lib.mkIf cfg.enable { + services.kea.dhcp4 = { + enable = true; + settings = { + interfaces-config.interfaces = lib.mapAttrsToList (name: _: "vlan-${name}") dhcpVlans; + lease-database = { + type = "memfile"; + persist = true; + name = "/var/lib/kea/dhcp4.leases"; + }; + valid-lifetime = 86400; + subnet4 = lib.mapAttrsToList (name: vlan: { + id = vlan.id; + subnet = vlan.subnet; + interface = "vlan-${name}"; + valid-lifetime = vlan.dhcp.leaseTime; + pools = [ { pool = "${vlan.dhcp.pool.from} - ${vlan.dhcp.pool.to}"; } ]; + option-data = [ + { + name = "routers"; + data = vlan.address; + } + { + name = "domain-name-servers"; + data = vlan.address; + } + ]; + }) dhcpVlans; + }; + }; + + services.blocky = { + enable = true; + settings = { + ports = { + dns = 53; + http = 4000; + }; + upstreams.groups.default = [ + "9.9.9.9" + "149.112.112.112" + "2620:fe::fe" + ]; + blocking = { + denylists.ads = [ + "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts" + ]; + clientGroupsBlock.default = [ "ads" ]; + }; + caching = { + minTime = "5m"; + prefetching = true; + }; + prometheus.enable = true; + }; + }; + + # The router itself resolves via public resolvers, not via Blocky, so DNS + # for deploys/updates survives a broken local resolver. + networking.nameservers = [ + "9.9.9.9" + "1.1.1.1" + ]; + }; +} diff --git a/modules/router/firewall.nix b/modules/router/firewall.nix new file mode 100644 index 0000000..3a804aa --- /dev/null +++ b/modules/router/firewall.nix @@ -0,0 +1,62 @@ +# Router firewall/NAT policy (nftables). Trust model: +# mgmt VLAN -> trusted: router services, all VLANs, WAN +# other VLANs -> DNS/DHCP on the router + WAN (if allowWan); no inter-VLAN +# WAN (ppp0) -> nothing inbound beyond established/related +# mesh -> admin SSH + metrics scrapes (same trust boundary as the fleet) +{ + config, + lib, + ... +}: +let + cfg = config.cnx.router; + mesh = import ../mesh-hosts.nix { inherit config lib; }; + + vlanIfs = lib.mapAttrsToList (name: _: "vlan-${name}") cfg.vlans; + wanVlanIfs = lib.mapAttrsToList (name: _: "vlan-${name}") ( + lib.filterAttrs (_: vlan: vlan.allowWan) cfg.vlans + ); + nonMgmtIfs = lib.filter (i: i != "vlan-mgmt") vlanIfs; + ifSet = ifs: "{ ${lib.concatStringsSep ", " (map (i: "\"${i}\"") ifs)} }"; +in +{ + config = lib.mkIf cfg.enable { + networking.nftables.enable = true; + + # SSH reachable only from the mgmt VLAN (trusted) and the mesh — never + # from the WAN or the other VLANs. + services.openssh.openFirewall = false; + + networking.firewall = { + enable = true; + filterForward = true; + trustedInterfaces = [ "vlan-mgmt" ]; + + # Non-mgmt VLANs may only talk to the router's DNS and DHCP. + interfaces = lib.genAttrs nonMgmtIfs (_: { + allowedTCPPorts = [ 53 ]; + allowedUDPPorts = [ + 53 + 67 + ]; + }); + + extraInputRules = '' + ip6 saddr ${mesh.subnet} tcp dport 22 accept comment "admin ssh over the mesh" + ip6 saddr ${mesh.subnet} tcp dport 4000 accept comment "blocky metrics scrape from control" + ''; + + extraForwardRules = '' + tcp flags syn tcp option maxseg size set rt mtu comment "MSS clamp for PPPoE mtu 1492" + iifname "vlan-mgmt" accept comment "mgmt reaches all VLANs and the WAN" + iifname ${ifSet wanVlanIfs} oifname "ppp0" accept comment "LAN to internet" + ''; + }; + + networking.nat = { + enable = true; + externalInterface = "ppp0"; + internalInterfaces = vlanIfs; + }; + }; +} diff --git a/modules/router/ipv6.nix b/modules/router/ipv6.nix new file mode 100644 index 0000000..25e4d7c --- /dev/null +++ b/modules/router/ipv6.nix @@ -0,0 +1,31 @@ +# IPv6 on the PPPoE uplink: run networkd's DHCPv6 client on ppp0 to obtain a +# delegated prefix; each vlan-* interface (default.nix) carves a /64 out of it +# via DHCPPrefixDelegation and announces it to clients with SLAAC. +{ + config, + lib, + ... +}: +let + cfg = config.cnx.router; +in +{ + config = lib.mkIf cfg.enable { + systemd.network.networks."45-ppp0" = { + matchConfig.Name = "ppp0"; + networkConfig = { + DHCP = "ipv6"; + # pppd owns the v4 address/route on this link; don't let networkd + # tear them down. + KeepConfiguration = "static"; + # Default v6 route comes from the ISP's RA when they send one. + IPv6AcceptRA = true; + }; + # Many PPPoE ISPs never send an RA with the M flag; solicit regardless. + dhcpV6Config.WithoutRA = "solicit"; + linkConfig.RequiredForOnline = "no"; + }; + + boot.kernel.sysctl."net.ipv6.conf.all.forwarding" = lib.mkDefault 1; + }; +} diff --git a/modules/router/omada.nix b/modules/router/omada.nix new file mode 100644 index 0000000..dec7e17 --- /dev/null +++ b/modules/router/omada.nix @@ -0,0 +1,43 @@ +# TP-Link Omada SDN controller for sites with Omada APs/switches. There is no +# nixpkgs package, so it runs as a podman container (mbentley/omada-controller, +# the de-facto standard image). Host networking because device adoption relies +# on L2 broadcast discovery (UDP 29810) on the mgmt VLAN; the default-deny +# input firewall keeps its ports unreachable from WAN and non-mgmt VLANs. +{ + config, + lib, + ... +}: +let + cfg = config.cnx.router; + mesh = import ../mesh-hosts.nix { inherit config lib; }; +in +{ + options.cnx.router.omada.enable = + lib.mkEnableOption "TP-Link Omada SDN controller (podman container)"; + + config = lib.mkIf (cfg.enable && cfg.omada.enable) { + virtualisation.podman.enable = true; + virtualisation.oci-containers = { + backend = "podman"; + containers.omada = { + image = "docker.io/mbentley/omada-controller:5.15"; + extraOptions = [ "--network=host" ]; + environment.TZ = config.time.timeZone; + volumes = [ + "/var/lib/omada/data:/opt/tplink/EAPController/data" + "/var/lib/omada/logs:/opt/tplink/EAPController/logs" + ]; + }; + }; + + # Admin UI (8043) also reachable over the mesh, like Grafana on control. + networking.firewall.extraInputRules = '' + ip6 saddr ${mesh.subnet} tcp dport 8043 accept comment "omada ui over the mesh" + ''; + + # Controller state (adopted devices, site config, cert) — declared as clan + # state so a borgbackup client can pick it up; backup wiring is a later step. + clan.core.state.omada.folders = [ "/var/lib/omada" ]; + }; +} diff --git a/modules/router/pppoe.nix b/modules/router/pppoe.nix new file mode 100644 index 0000000..59adf0c --- /dev/null +++ b/modules/router/pppoe.nix @@ -0,0 +1,64 @@ +# PPPoE WAN session. ISP credentials are entered once at `clan vars generate` +# (prompts). Both are secret — AIS often uses the same string for username and +# password — so neither may land in the Nix store: pppd reads the username from +# an included secret options file and the password from chap/pap-secrets. +{ + config, + lib, + ... +}: +let + cfg = config.cnx.router; + creds = config.clan.core.vars.generators.pppoe-credentials; +in +{ + config = lib.mkIf cfg.enable { + clan.core.vars.generators.pppoe-credentials = { + prompts.username = { + description = "PPPoE username (from the ISP)"; + type = "hidden"; + }; + prompts.password = { + description = "PPPoE password (from the ISP)"; + type = "hidden"; + }; + files."user-opts".secret = true; + files."chap-secrets".secret = true; + script = '' + user="$(cat "$prompts"/username)" + pass="$(cat "$prompts"/password)" + printf 'user "%s"\n' "$user" > "$out"/user-opts + printf '"%s" * "%s"\n' "$user" "$pass" > "$out"/chap-secrets + ''; + }; + + services.pppd = { + enable = true; + peers.wan = { + autostart = true; + config = '' + plugin pppoe.so ${cfg.wan.pppInterface} + ifname ppp0 + file ${creds.files."user-opts".path} + noipdefault + defaultroute + noauth + hide-password + persist + maxfail 0 + holdoff 5 + lcp-echo-interval 15 + lcp-echo-failure 3 + +ipv6 + mtu 1492 + mru 1492 + ''; + }; + }; + + # pppd looks up the password for `user` in these files at dial time; both + # point at the same generated `"" * ""` line (PAP and CHAP). + environment.etc."ppp/chap-secrets".source = creds.files."chap-secrets".path; + environment.etc."ppp/pap-secrets".source = creds.files."chap-secrets".path; + }; +}