router: make CrowdSec opt-in (crowdsec.enable, default off)

Not every site wants the ban engine (hub sync needs internet at
activation, and it is one more moving part on a small box). Gate
crowdsec.nix on a new `crowdsec.enable` option like `omada.enable`.

gw-cnx-1 keeps it on; the VM test drops its mkForce overrides.

.gitignore

@@ -1,2 +1,2 @@
-result
-.direnv/
+result*
+.direnv/

TODO.md

@@ -0,0 +1,69 @@
+# Infra roadmap
+
+Prioritized backlog for the cnx-network clan. See `docs/` for how the current
+pieces work.
+
+## 1. Alerting (done — pending deploy)
+
+Rules evaluated by vmalert against VictoriaMetrics on control, declared in
+`modules/monitoring/alerts.nix`:
+
+- [x] SOA serial divergence between ns1 and ns2 (secondary out of sync)
+- [x] Zone-expiry countdown on the secondary approaching zero (transfers failing)
+- [x] Any scrape target down (`up == 0`)
+- [x] Root filesystem nearly full
+
+Delivery stays minimal for now (`notifier.blackhole`): vmalert remote-writes
+alert state back to VM, so firing alerts show up as the `ALERTS` series in
+Grafana. Wiring a real notifier (Matrix) is a later step — drop `blackhole` and
+set `settings."notifier.url"` to an Alertmanager.
+
+## 2. Backups of critical state (DNSSEC done — pending vars + deploy)
+
+clan `borgbackup` instance in `clan.nix`: control is the server (repos under
+`/var/lib/borgbackup/<client>`), ns1 the client. ns1 declares
+`clan.core.state.knot.folders = [ "/var/lib/knot" ]`, so the Knot KASP keystore
+is backed up nightly (01:00) over the mesh with repokey encryption — control
+never holds plaintext. ns1 maps the `control` machine name to its mesh IP via
+`networking.hosts` so the `borg@control` repo resolves.
+
+Before deploy: `clan vars generate ns1` (YubiKey) to mint the borgbackup ssh
+keypair + repokey; control won't evaluate until ns1's public key exists. Then
+deploy ns1 and control.
+
+- [x] DNSSEC key material on ns1 (KSK/ZSK in Knot's KASP store) — losing it forces
+      an emergency DS rollover at the registrar
+- [ ] VictoriaMetrics TSDB on control (optional, retention is 180d) — deferred;
+      regenerable over time and control is the backup server, so this needs a
+      second client→server pair (e.g. control→ns2) rather than the same topology
+
+## 3. Blackbox DNS probing (done — pending deploy)
+
+`blackbox_exporter` on control (loopback `:9115`), probing each nameserver's
+public v4+v6 address for every zone: an SOA query (zone served?) and a DNSKEY
+query (still signed?). Blackbox has no DO-bit option, so signing is checked by
+asking for DNSKEY directly and asserting the RRset is present. Probe defs live
+in `modules/monitoring/blackbox-probes.nix`, shared by the exporter
+(`blackbox.nix`) and the VM scrape jobs (`server.nix`). Verified live against
+ns1/ns2: SOA + DNSKEY succeed on both servers over v4 and v6.
+
+- [x] `blackbox_exporter` on control doing real DNS + DNSSEC-validation queries
+      against ns1/ns2 — catches outside-in resolution failures the Knot stats miss
+- [x] paired with alerts (`DNSResolutionProbeFailed` / `DNSSECProbeFailed` in
+      `alerts.nix`) and a "DNS probes (outside-in)" row on the CNX DNS dashboard
+
+## 4. Third secondary off Hetzner (resilience)
+
+- [ ] A secondary nameserver on a different provider/network so a single-provider
+      outage doesn't take all authoritative DNS down (architectural — new machine)
+
+## 5. Centralized logs (done — pending deploy)
+
+VictoriaLogs on control (`:9428`, 30d retention, mesh-scoped) in
+`modules/monitoring/server.nix`, plus a VictoriaLogs Grafana datasource. All
+three hosts ship journald with systemd's own `services.journald.upload` to the
+`/insert/journald` endpoint (`modules/monitoring/exporters.nix`) — no extra
+agent. control uploads over loopback; ns1/ns2 over the mesh.
+
+- [x] VictoriaLogs on control to grep journald across all three hosts, pairing
+      with the existing VictoriaMetrics setup

clan.nix

@@ -1,39 +1,106 @@
+{ ... }:
+let
+  hosts = import ./modules/hosts.nix;
+
+  # Single source of the machine list: inventory AND the per-machine baseline
+  # below are both derived from it, so no machine can be added without the
+  # baseline (e.g. admins group encryption for all its generated secrets).
+  fleet = {
+    control = { };
+    ns1 = { };
+    ns2 = { };
+    mx1 = { };
+    web01 = { };
+    # Site gateways (Topton 1U routers): dynamic PPPoE WAN, so they are NOT in
+    # modules/hosts.nix / the `internet` instance — clan reaches them over the
+    # zerotier mesh (or Tor) instead.
+    gw-cnx-1 = { };
+  };
+in
 {
+  # Site gateways: the router service instance lives in inventory.nix.
+  imports = [ ./inventory.nix ];
+
   # Ensure this is unique among all clans you want to use.
   meta.name = "cnx-network-clan";
   meta.domain = "cnx-network.internal";
 
-  inventory.machines = {
-	control = {};
-	ns1 = {};
-	ns2 = {};
-  };
+  inventory.machines = fleet;
+
+  # Services defined in this repo (module.input = "self"). See modules/clan/.
+  modules.router = ./modules/clan/router;
 
   inventory.instances = {
 
-    admin = {
-      roles.default.tags.all = { };
-      roles.default.settings.allowedKeys = {
+    # Admin SSH keys + root password, split per the clan-core migration off
+    # the deprecated `admin` service (sshd handles keys, users the password).
+    sshd = {
+      roles.server.tags.all = { };
+      roles.server.settings.authorizedKeys = {
         "berwn" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIENAjhGQGraQoAjJzsomKP8GAmQPeGL1rNRNHgRcLqtT";
-        "kurogeek" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEcZ/p1Ofa9liwIzPWzNtONhJ7+FUWd2lCz33r81t8+w kurogeek@kurogeek";
+        "kurogeek" =
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEcZ/p1Ofa9liwIzPWzNtONhJ7+FUWd2lCz33r81t8+w kurogeek@kurogeek";
+      };
+    };
+
+    root-user = {
+      module = {
+        name = "users";
+        input = "clan-core";
+      };
+      roles.default.tags.all = { };
+      roles.default.settings = {
+        user = "root";
+        prompt = false; # auto-generate, like the old admin service
+        share = false; # per-machine password, not fleet-wide
       };
     };
 
     zerotier = {
       roles.controller.machines."control" = { };
       roles.peer.tags.all = { };
+      # External members admitted by ZeroTier node id (stable per device).
+      # Inventory machines are auto-accepted; this is only for peers outside the
+      # clan. Node id comes from `zerotier-cli info` on the joining device.
+      roles.controller.settings.allowedIds = [
+        "8802c8d7e0" # alex-nixos
+        "2bd36db8cc" # kurogeek-thinkpad
+      ];
     };
 
     tor = {
       roles.server.tags.nixos = { };
     };
 
+    # Direct SSH to public IPs — clan's priority-1 connection path, with the
+    # ZeroTier mesh and Tor kept as automatic fallbacks. Raw IPs (from
+    # modules/hosts.nix, not the ns1/ns2 DNS names) so reaching these hosts never
+    # depends on their own DNS being up.
+    internet.roles.default.machines = builtins.mapAttrs (_: h: {
+      settings.host = h.ipv4;
+    }) hosts;
+
     # Recovery root password for console access when a machine fails to boot.
     emergency-access = {
       roles.default.tags.nixos = { };
     };
+
+    # Encrypted, deduplicating backups. control hosts the repos; ns1 is the
+    # only client, backing up its declared clan.core.state (the Knot DNSSEC
+    # keystore) over the mesh. Repo lives at /var/lib/borgbackup/ns1 on control.
+    # Cross-host so an ns1 loss is recoverable; repokey encryption means control
+    # never holds plaintext. Run `clan vars generate ns1` (YubiKey) before deploy.
+    borgbackup = {
+      roles.server.machines.control = { };
+      roles.client.machines.ns1 = { };
+    };
   };
 
-  machines = {
-  };
+  # Fleet-wide baseline applied to every machine. Secrets minted by
+  # `clan vars generate` are encrypted for the admins group from the very
+  # first run — generating before this took effect is what forced the
+  # re-encryption dance (`clan vars fix`) on gw-cnx-1.
+  machines = builtins.mapAttrs (_: _: {
+    clan.core.sops.defaultGroups = [ "admins" ];
+  }) fleet;
 }

docs/book.toml

@@ -0,0 +1,12 @@
+[book]
+title = "CNX Infra Runbook"
+description = "Operational docs for the cnx-network clan: DNS, ZeroTier mesh, monitoring."
+authors = ["B4L"]
+src = "src"
+language = "en"
+
+[output.html]
+default-theme = "navy"
+preferred-dark-theme = "navy"
+git-repository-url = "https://git.b4l.co.th/B4L/cnx-network-clan"
+edit-url-template = "https://git.b4l.co.th/B4L/cnx-network-clan/_edit/main/docs/{path}"

docs/src/SUMMARY.md

@@ -0,0 +1,9 @@
+# Summary
+
+- [Overview](./overview.md)
+- [ZeroTier mesh](./mesh.md)
+- [DNS](./dns.md)
+- [Mail](./mail.md)
+- [Site gateways](./gateways.md)
+- [Monitoring](./monitoring.md)
+- [Backups](./backups.md)

docs/src/backups.md

@@ -0,0 +1,61 @@
+# Backups
+
+Encrypted, deduplicating backups via clan's `borgbackup` service, declared in
+`clan.nix`. The only critical, non-regenerable state is the **Knot DNSSEC
+keystore** on `ns1` (the KSK/ZSK private keys under `/var/lib/knot`); losing it
+forces an emergency DS rollover at the registrar.
+
+## Topology
+
+- **control** is the borgbackup **server** — it hosts the repos under
+  `/var/lib/borgbackup/<client>` (so `ns1`'s repo is `/var/lib/borgbackup/ns1`).
+- **ns1** is the **client**. It backs up everything it declares as clan state
+  (`clan.core.state.knot.folders = [ "/var/lib/knot" ]`) once a day at 01:00,
+  over the ZeroTier mesh.
+
+The backup is cross-host so that losing `ns1` is recoverable, and stays
+self-contained (no third-party storage). Encryption is `repokey` with a
+generated passphrase, so `control` only ever stores ciphertext.
+
+Mesh peers have no name resolution, so `ns1` maps the `control` machine name to
+its ZeroTier address via `networking.hosts`; that is how the `borg@control` repo
+URL resolves.
+
+## Secrets
+
+The borgbackup ssh keypair and repokey passphrase are clan vars, generated once
+(needs the YubiKey). `control` will not evaluate until `ns1`'s public key
+exists, so generate before the first deploy:
+
+```
+clan vars generate ns1
+clan machines update ns1
+clan machines update control
+```
+
+## Operating
+
+Backups are driven by systemd on `ns1` (`borgbackup-job-control.timer`).
+
+```
+# trigger a backup now (on ns1)
+borgbackup-create
+
+# list archives (on ns1)
+borgbackup-list
+
+# restore selected folders from an archive (on ns1)
+NAME='<archive-name>' FOLDERS=/var/lib/knot borgbackup-restore
+```
+
+Retention is pruned automatically: all archives from the last day, then 7 daily
+and 4 weekly.
+
+## Monitoring
+
+The **CNX Backups** Grafana dashboard
+(`modules/monitoring/dashboards/backups.json`) tracks job health, time since the
+last successful run, and per-job state — all from the node_exporter systemd
+collector on the client. There is no dedicated borg metrics exporter; the unit
+state and the timer's last-trigger timestamp are enough to catch a backup that
+stops running or fails.

docs/src/dns.md

@@ -0,0 +1,95 @@
+# DNS
+
+Authoritative DNS for three zones, served by Knot:
+
+- `cnx.network`
+- `buildfor.life`
+- `cnx.email`
+
+Add a zone in `modules/dns/domains.nix` **and** drop a matching `<domain>.zone`
+file in `modules/dns/zones/`.
+
+## Primary / secondary
+
+- **`ns1` = primary (master).** Loads each zone from its file, signs it, and
+  notifies `ns2`. Config in `machines/ns1/configuration.nix`.
+- **`ns2` = secondary (slave).** Pulls every zone from `ns1` (AXFR/IXFR) and
+  accepts its NOTIFY. Config in `machines/ns2/configuration.nix`.
+
+Zone transfers run **over the ZeroTier mesh**, authenticated with a shared TSIG
+key (`dns-tsig`, a clan var copied to both machines).
+
+## Serial handling
+
+`ns1` uses `zonefile-load = difference-no-serial` with `serial-policy = unixtime`:
+edit records without touching the SOA serial — Knot diffs the file, assigns a
+strictly-monotonic unixtime serial, signs, and transfers. `journal-content = all`
+holds the live signed zone (required by `difference-no-serial`).
+
+## DNSSEC
+
+Automatic signing on `ns1` only, policy `cnx`: ECDSA P-256/SHA-256. The ZSK
+auto-rolls; the KSK is kept stable, so the DS at the registrar only changes on a
+manual KSK rollover.
+
+### Registrar DS records
+
+Knot manages all key material itself on `ns1` (the only signer); the KSK/ZSK
+private keys live in the KASP keystore under `/var/lib/knot` (backed up nightly —
+see [Backups](./backups.md)). You never touch the private keys directly.
+
+What a registrar needs is the **DS record** for a zone's KSK, which anchors the
+zone into the parent's chain of trust. Generate it on `ns1` — the `keymgr` wrapper
+is already pointed at Knot's config, and it runs as the `knot` user that owns the
+keystore:
+
+```
+sudo -u knot keymgr <zone> ds
+```
+
+e.g. `sudo -u knot keymgr cnx.email ds`. Paste the printed DS record (key tag,
+algorithm 13, digest type, digest) into the registrar's DNSSEC form for that
+domain. Repeat per signed zone (`cnx.network`, `buildfor.life`, `cnx.email`) at
+whichever registrar holds each delegation. After submitting, confirm the parent
+publishes it with `dig +short DS <zone>`.
+
+The ZSK rolls automatically and needs no registrar action; only a **KSK rollover**
+requires re-submitting the DS.
+
+> **Pending (manual):** submit DS records for `buildfor.life` and `cnx.email`
+> once they're at a DNSSEC-capable registrar.
+
+## ACME DNS-01
+
+Certificates are issued by `_acme-challenge` TXT updates that `ns1` accepts over
+TSIG, signs, and transfers to `ns2` (which never needs these keys). Each consumer
+gets its **own** key, scoped by an ACL to exactly the owner names it needs and
+attached only to the zone it lives in — so a leaked key can write nothing but its
+own challenges.
+
+- **`acme_ddns`** (`acl_acme`) — the general key, scoped to `TXT` at or under
+  `_acme-challenge.<zone>` and attached to every zone. Client config:
+  ```
+  clan vars get ns1 dns-acme-tsig/acme.conf
+  ```
+- **`acme_mx1`** (`acl_acme_mx1`) — held only by `mx1`, scoped to
+  `_acme-challenge.{mx1,mta-sts,mail}` and attached only to `cnx.email` (the mail
+  cert plus its MTA-STS and client-alias SANs). Secret shared via the
+  `dns-acme-mx1-secret` generator.
+- **`acme_web01`** (`acl_acme_web01`) — held only by `web01`, scoped to
+  `_acme-challenge` and attached only to `cnx.network` (where the wildcard
+  `*.cnx.network` challenge lands, at the apex). Secret shared via the
+  `dns-acme-web01-secret` generator.
+
+## Runbook: stale secondary
+
+If `ns2` serves stale records while SOA serials match (e.g. after a manual zone
+edit that didn't bump the serial as expected), force a fresh transfer on `ns2`:
+
+```
+knotc zone-retransfer <zone>
+```
+
+Watch the **CNX DNS** Grafana dashboard: the per-nameserver SOA serial table
+should agree across `ns1`/`ns2`, and "seconds until zone expiry" on the secondary
+should reset on each successful transfer rather than counting toward zero.

docs/src/gateways.md

@@ -0,0 +1,168 @@
+# Site gateways
+
+NixOS routers (OPNsense replacements) on Topton 1U fanless boxes — Intel N300,
+4x i226-V 2.5G ports (some units add 2x 10G SFP+), 16GB DDR5, 256GB mSATA.
+Four sites are planned; the first is `gw-cnx-1` (Chiang Mai). Everything lives
+in the `router` clan service (`modules/clan/router/`, registered as
+`modules.router` in `clan.nix`); a site joins by adding its machine to the
+`router` inventory instance in `inventory.nix` with settings for port names,
+VLANs, and feature flags (`roles.default.machines.<gw>.settings`).
+
+Naming: `gw-<city>-<n>`, e.g. `gw-cnx-1`.
+
+The service has a NixOS VM test (`modules/clan/router/tests/vm/`): an ISP
+node runs a PPPoE access concentrator, a `client` sits on the tagged lan/iot
+VLANs behind the trunk, an `admin` on the untagged mgmt access port, and a
+simulated radio (`mac80211_hwsim`) carries two SSIDs with a WPA3 station. It
+checks the PPPoE dial-in, leases/reservations, Blocky, NAT, `allowWan`, the
+mgmt-only trust model, SSH exposure and the Wi-Fi bridge ports. Run it with
+`nix build .#checks.x86_64-linux.router` (also part of `nix flake check`).
+
+## What each gateway runs
+
+| Function     | Implementation                                                                                                                                                                                                                                                                                                                |
+| ------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| WAN          | PPPoE (`pppd`), per-site ISP credentials via clan vars prompts; `wan.vlanId` when the ISP tags the session (AIS: 10); `wan.macAddress` to clone the old router's MAC if the ISP has it pinned                                                                                                                                 |
+| LAN          | VLAN-filtering bridge `br0` over the trunk ports (networkd); `accessPorts` pin a port untagged to one VLAN — convention: the last copper port is an untagged `mgmt` recovery port; `stagingPort` turns a spare port into a DHCP uplink into the old LAN while the box runs alongside the router it replaces (drop at cutover) |
+| Firewall/NAT | nftables: default-deny WAN, no inter-VLAN, MSS clamp, v4 NAT                                                                                                                                                                                                                                                                  |
+| DHCP         | Kea, one subnet per VLAN                                                                                                                                                                                                                                                                                                      |
+| DNS          | Blocky (blocklist resolver), metrics on :4000 scraped by control                                                                                                                                                                                                                                                              |
+| IPv6         | DHCPv6-PD on ppp0, /64 per VLAN via SLAAC                                                                                                                                                                                                                                                                                     |
+| Bans         | Optional per site (`crowdsec.enable`): CrowdSec + nftables bouncer (sshd log parsing)                                                                                                                                                                                                                                         |
+| Omada        | Optional per site: TP-Link Omada controller as a podman container                                                                                                                                                                                                                                                             |
+| Wi-Fi        | Optional: hostapd on the router's radios; each SSID (`wifi.networks`) is an untagged access port of its VLAN, passphrases via vars prompts — see `modules/clan/router/README.md`                                                                                                                                              |
+| Proxy        | Optional: Caddy reverse proxy for internal services under `*.<site><n>.cnx.network` with a real Let's Encrypt wildcard (DNS-01 against ns1)                                                                                                                                                                                   |
+| Diagnostics  | iperf3 server on 5201, reachable from every VLAN and the mesh (`iperf3 -c <router-vlan-addr>`); CLI toolkit: tcpdump, mtr, ethtool, conntrack, kdig, iftop, librespeed-cli                                                                                                                                                    |
+| Speed test   | Hourly librespeed run (`speedtest.timer`) → `speedtest_*` metrics via node_exporter; vmalert flags download < 50% of the link's own 7-day median (`WANSpeedDegraded`)                                                                                                                                                         |
+| Management   | ZeroTier mesh: SSH, node_exporter, journald upload — like the fleet                                                                                                                                                                                                                                                           |
+| Storage      | Single-disk ZFS (zstd, `/var` its own dataset for snapshots)                                                                                                                                                                                                                                                                  |
+
+## Addressing plan
+
+Each site owns `10.<siteId>.0.0/16`. Defaults per VLAN: subnet
+`10.<siteId>.<vlanId>.0/24`, router at `.1`, DHCP pool `.100–.199`
+(`.2–.99` static/infra, `.200–.254` reserved).
+
+VLAN id convention (fleet-wide): **10 = mgmt**, **20 = lan** (both mandatory at
+every site), 30 = guest, 40 = iot (reserved). Sites add their own beyond that.
+
+Larger subnets (e.g. public-WiFi guest networks that outgrow a /24) are carved
+from the **upper half** `10.<siteId>.128.0/17` and set explicitly on the VLAN.
+The lower half stays reserved for /24s indexed by VLAN id. High-churn VLANs
+should also shorten `dhcp.leaseTime` (default 86400 s) so the pool recycles.
+First user: `gw-cnx-2` (site 2) runs the public WiFi — guest VLAN 30 at
+`10.2.128.0/22`, pool `10.2.128.100 – 10.2.131.250`, `dhcp.leaseTime = 3600`.
+
+| Site | siteId | mgmt           | lan            | site-specific VLANs                                             |
+| ---- | ------ | -------------- | -------------- | --------------------------------------------------------------- |
+| cnx  | 1      | `10.1.10.0/24` | `10.1.20.0/24` | iot 40, voip 50, dmz 60, unit1–5 110/120/130/140/150 (all /24s) |
+
+Static DHCP leases are declared per VLAN via `dhcp.reservations` (attribute
+name = hostname, plus `hwAddress`/`ipAddress`); park them outside the pool,
+in the `.2–.99` infra range or `.200–.254`.
+
+Trust model: mgmt → everything; other VLANs → router DNS/DHCP + internet only
+(no inter-VLAN); WAN → nothing inbound; mesh → SSH, metrics, Omada UI.
+
+## Adding a site
+
+1. Copy `machines/gw-cnx-1/` to `machines/gw-<city>-<n>/` (disko, facter,
+   hostId, timezone). Fill the real disk id in `disko.nix` (from the
+   installer: `ls -l /dev/disk/by-id/`).
+2. Add the machine to `inventory.machines` in `clan.nix`, to the `router`
+   instance in `inventory.nix` (`roles.default.machines.gw-<city>-<n>.settings`: `site`,
+   `siteId` (next free number), port names, VLANs, `omada.enable`, `crowdsec.enable`; keep the
+   `mgmt`/`lan` VLANs), and to the machine list in `modules/mesh-hosts.nix`.
+   Do **not** add it to `modules/hosts.nix` (dynamic PPPoE IP; clan connects
+   over the mesh).
+3. Add its node_exporter and Blocky scrape targets in
+   `modules/monitoring/server.nix`.
+4. If the site uses the internal proxy (`proxy.enable` in its settings), wire
+   ns1: import `(import ../../modules/clan/router/acme-secret.nix
+"gw-<city>-<n>")` in ns1's configuration (the gateway declares the same
+   shared generator itself), and on ns1 add the `dns-acme-gw-<city>-<n>-knot`
+   generator, its `keyFiles` entry, and an `acl_acme_gw_<city>_<n>` scoped to
+   `_acme-challenge.<city><n>` on the `cnx.network` zone (copy the `gw-cnx-1`
+   blocks). Then `clan vars generate ns1` and redeploy ns1.
+5. `git add` the new machine directory — flake evaluation only sees
+   git-tracked files, so an untracked `machines/gw-…/` is silently ignored.
+6. `clan vars generate gw-<city>-<n>` — prompts for the site's PPPoE
+   credentials, mints the ZeroTier identity etc. (`nix flake check` fails
+   until this has run, because mesh-hosts reads the ZeroTier IP var.)
+7. Boot the box from a NixOS installer USB on the local network, then:
+   `clan machines install gw-<city>-<n> --target-host root@<lan-ip>`
+8. Check `facter.json` for the real NIC names, fix `wan.interface` /
+   `trunkPorts` if the enumeration differs, and
+   `clan machines update gw-<city>-<n>` (rides the mesh from then on).
+9. Add a row to the site table above and to the machines table in
+   [Overview](./overview.md).
+
+## Omada controller
+
+Sites with TP-Link Omada APs/switches set `omada.enable = true` in their
+router settings. The controller runs as a podman container
+(`mbentley/omada-controller` — there is no nixpkgs package) with host
+networking on the mgmt VLAN, where adoption
+broadcasts (UDP 29810) live. UI: `https://<mgmt-addr>:8043` from mgmt, or the
+gateway's mesh address from an admin machine. Controller state is under
+`/var/lib/omada`, declared as clan state (`clan.core.state.omada`); wiring it
+into the borgbackup instance is still a follow-up.
+
+## Internal reverse proxy
+
+`proxy.enable` in a site's router settings puts Caddy on the gateway,
+terminating TLS for `<service>.<site><n>.cnx.network` (e.g.
+`https://omada.cnx1.cnx.network`) and forwarding to internal backends:
+
+```nix
+proxy = {
+  enable = true;
+  services.omada = {
+    backend = "https://127.0.0.1:8043";
+    insecureSkipVerify = true; # Omada's cert is self-signed
+  };
+};
+```
+
+- **Cert**: one real Let's Encrypt wildcard `*.<site><n>.cnx.network` per
+  gateway, issued via ACME DNS-01 (lego/RFC2136) against ns1 — works behind
+  PPPoE with zero inbound reachability, and browsers trust it without a CA
+  install. Each gateway has its own TSIG key (`acme_gw_<city>_<n>`) that ns1
+  scopes to `_acme-challenge.<site><n>` TXT records only. The zone, ns1's
+  address and the ACME contact are fleet-wide role settings in `inventory.nix`
+  (`proxy.domain`, `proxy.acme.nameserver`, `proxy.acme.email`).
+- **Resolution**: the names exist only internally — Blocky answers
+  `*.<site><n>.cnx.network` with the router's `lan` address; the public
+  `cnx.network` zone never carries them.
+- **Access**: `proxy.allowVlans` (default `mgmt` + `lan`) get 443 (and 80 for
+  the HTTP→HTTPS redirect). Not exposed to WAN, guest VLANs, or the mesh.
+
+## Using the service from another clan
+
+The service is exported as `clan.modules.router` of this flake; add this
+repo as a flake input and reference it with `module.input = "<input name>"`.
+`modules/clan/router/README.md` is the consumer-facing description (it is
+also the module's manifest readme, shown by `clan modules`).
+
+## Runbook
+
+- **PPPoE down**: `systemctl status pppd-wan`, `journalctl -u pppd-wan` on the
+  gateway (over the mesh). ISP credentials live in the `pppoe-credentials`
+  vars generator; re-enter with `clan vars generate gw-<city>-<n> --regenerate`.
+- **A VLAN gets no leases**: `systemctl status kea-dhcp4-server`; check the
+  port's `bridgeVLANs` tagging and that the switch trunk carries the VLAN.
+- **DNS/blocklist issues**: Blocky metrics are in VictoriaMetrics (job
+  `blocky`); `journalctl -u blocky` on the gateway.
+- **Slow internet reported / `WANSpeedDegraded` firing**: check the
+  `speedtest_download_mbps` / `speedtest_ping_ms` trend in VictoriaMetrics,
+  then `librespeed-cli` and `mtr <isp-gw>` on the gateway to separate ISP
+  degradation from local saturation (`iftop` on ppp0 shows who's eating it).
+- **Banned yourself**: `cscli decisions list` / `cscli decisions delete --ip <ip>`
+  on the gateway. The ZeroTier mesh `/88` is whitelisted at the parser stage
+  (`cnx/mesh-whitelist`), so admin access over the mesh can never be banned.
+
+## Follow-ups
+
+- Borgbackup client for Omada + Kea lease state.
+- Suricata (IDS-only) if CPU headroom allows — deliberately skipped for now.
+- Remaining three sites.

docs/src/mail.md

@@ -0,0 +1,90 @@
+# Mail
+
+A single mail server, `mx1`, runs the [Simple NixOS Mailserver](https://nixos-mailserver.readthedocs.io/)
+stack (Postfix + Dovecot + Rspamd + OpenDKIM) for the `cnx.email` domain. All of
+it is declared in `modules/mail.nix`, imported by `machines/mx1/configuration.nix`.
+
+Mailboxes are **virtual** (not system users): each address is a login account
+whose password is a four-word passphrase auto-generated by a clan vars generator
+(e.g. `otter-lantern-cobalt-driftwood-42`). The generator stores both the
+passphrase (handed to the user) and its sha-512 hash (consumed by SNM); plaintext
+never lands in the repo.
+
+The mail cert is obtained by `mx1` itself via ACME DNS-01 against `ns1` (key
+`acme_mx1`), and carries `mx1.cnx.email` plus the `mta-sts.cnx.email` and
+`mail.cnx.email` SANs. See [DNS → ACME DNS-01](./dns.md#acme-dns-01).
+
+## Add a mailbox
+
+1. Append the address to `accounts` in `modules/mail.nix`:
+   ```nix
+   accounts = [
+     "postmaster@cnx.email"
+     "alice@cnx.email"      # new
+   ];
+   ```
+2. Mint its generated password (user-run, YubiKey touch):
+   ```
+   clan vars generate mx1
+   ```
+3. Redeploy: `clan machines update mx1`.
+4. Hand the passphrase to the user. The generator name replaces `@`→`-at-` and
+   `.`→`-`:
+   ```
+   clan vars get mx1 mail-passwd-alice-at-cnx-email/passphrase
+   ```
+
+The DMARC report inbox (`dmarc@cnx.email`) is special: its password comes from
+the **shared** `mail-dmarc-cred` generator (not the per-machine set) so parsedmarc
+on `control` can read the same passphrase over the mesh.
+
+## Add a domain
+
+Adding a second mail domain is more than a config line — it needs its own DNS
+records and DKIM key. Steps:
+
+1. **Serve the zone** (if not already): add it in `modules/dns/domains.nix` and
+   drop a `<domain>.zone` file — see [DNS](./dns.md).
+2. **Declare the domain** in `modules/mail.nix`:
+   ```nix
+   mailserver.domains = [ "cnx.email" "newdomain.tld" ];
+   ```
+   Add at least a `postmaster@newdomain.tld` account (RFC 5321 requires it) the
+   same way as above.
+3. **Publish the mail DNS records** in the new zone, mirroring `cnx.email`'s
+   `; ---- Mail ----` block:
+   - `MX 10 mx1.cnx.email.` — point at the existing MX; do **not** stand up a new
+     hostname unless you really want a separate server.
+   - SPF: `@ TXT "v=spf1 mx -all"`.
+   - DKIM: SNM generates a key per domain on first start under
+     `mailserver.dkimKeyDirectory` (selector `mail`). After deploy, read the
+     public key off `mx1` and paste it as `mail._domainkey TXT`. The private keys
+     are declared as clan state and backed up.
+   - DMARC: `_dmarc TXT "v=DMARC1; p=quarantine; rua=mailto:dmarc@cnx.email; ..."`
+     — reports can keep flowing to the existing `dmarc@cnx.email` inbox.
+4. **DANE / MTA-STS are tied to `mx1.cnx.email`**, the MX hostname — they do **not**
+   need duplicating per sending-domain. A new domain that uses `mx1.cnx.email` as
+   its MX inherits the existing TLSA and MTA-STS policy. Only add new TLSA/MTA-STS
+   records if you introduce a new MX hostname.
+5. `clan vars generate mx1` (if you added accounts), then
+   `clan machines update mx1`.
+
+## DKIM, DANE, MTA-STS reminders
+
+These three are easy to get subtly wrong; the live records for `cnx.email` and
+the exact commands to regenerate them are in `modules/dns/zones/cnx.email.zone`:
+
+- **DKIM** key rotation = regenerate under `dkimKeyDirectory`, then republish the
+  `mail._domainkey TXT`.
+- **DANE TLSA** (`_25._tcp.mx1 TLSA 3 1 1 …`) is the SHA-256 of `mx1`'s cert
+  **public key**. It stays valid across renewals because lego runs with
+  `--reuse-key`; only recompute it if the key changes. The `openssl` one-liner is
+  in the zone file.
+- **MTA-STS**: bump the `_mta-sts TXT` id **every time** the policy in
+  `modules/mail.nix` changes, or senders keep the cached policy.
+
+## DMARC reporting
+
+`mx1` delivers aggregate/forensic DMARC reports to `dmarc@cnx.email`; parsedmarc
+on `control` polls that mailbox over the mesh and feeds the results into Grafana.
+See [Monitoring](./monitoring.md).

docs/src/mesh.md

@@ -0,0 +1,37 @@
+# ZeroTier mesh
+
+A private IPv6 overlay that every machine (and admin laptops) shares. DNS zone
+transfers and metrics scraping ride this mesh, never the public net.
+
+- **Controller:** `control` (the `zerotier` instance in `clan.nix`).
+- **Peers:** every machine (`roles.peer.tags.all`).
+- **Prefix:** `fd06:1bad:ece2:92ad:ba99:9300::/88` (RFC 4193: `fd` + network id + `0x9993`).
+
+## The mesh map
+
+`modules/mesh-hosts.nix` does **not** hardcode addresses. It reads each machine's
+IP from the public clan vars that clan-core's zerotier service writes. As of
+clan-core 26.05 these are shared, instance-scoped generators
+(`vars/shared/zerotier-ip-<machine>-zerotier/ip/value`), and the `/88` subnet is
+derived from the instance network id
+(`vars/shared/zerotier-network-zerotier/network-id/value`). Regenerate or re-key a
+node and the map follows automatically.
+
+Consumers: `modules/dns/authoritative.nix` (transfer ACLs), `modules/monitoring/*`
+(scrape targets and firewall scoping).
+
+## Admitting external members
+
+Inventory machines are auto-accepted. External devices (admin laptops) are listed
+in `clan.nix` under the controller's `allowedIds` — clan-core 26.05 admits members
+by their 10-char ZeroTier node id directly, so no IP derivation is needed:
+
+```nix
+roles.controller.settings.allowedIds = [
+  "8802c8d7e0" # alex-nixos
+  "2bd36db8cc" # kurogeek-thinkpad
+];
+```
+
+A device's 10-char node id comes from `zerotier-cli info` on that device. After
+editing, deploy `control`; the controller admits the new member on its next run.

docs/src/monitoring.md

@@ -0,0 +1,122 @@
+# Monitoring
+
+Metrics and logs live on `control` over the ZeroTier mesh; the Grafana dashboards
+are also published publicly through `web01` (see [Dashboards](#dashboards)).
+
+## Collection
+
+- **node_exporter** (`:9100`) on every machine — CPU, memory, disk, systemd units.
+  Binds all interfaces; the scrape ports are firewall-scoped to the mesh subnet
+  (`modules/monitoring/exporters.nix`).
+- **knot-exporter** (`:9433`) on `ns1`/`ns2` only — reads Knot's control socket,
+  fed by the `mod-stats` module (query/response counters per zone).
+- **blackbox_exporter** (`127.0.0.1:9115`) on `control` only — outside-in DNS
+  probes. For every zone it queries each nameserver's **public** address (v4 and
+  v6) for SOA (is the zone served?) and DNSKEY (is it still signed?). This is the
+  resolver's-eye view that the Knot stats can't see. Probe definitions are shared
+  between the exporter and the scrape jobs in `modules/monitoring/blackbox-probes.nix`.
+
+## Storage & scraping
+
+**VictoriaMetrics** on `control`, bound to `127.0.0.1:8428`, 180-day retention
+(`modules/monitoring/server.nix`). It scrapes `control` over loopback and
+`ns1`/`ns2`/`mx1`/`web01` over the mesh.
+
+> The scraper dials IPv4-only by default, so mesh (IPv6) targets need
+> `extraOptions = [ "-enableTCP6" ]`. Without it, ns1/ns2 are dropped with
+> "no suitable address found". Check live target health on `control`:
+>
+> ```
+> curl -s http://127.0.0.1:8428/api/v1/targets | jq '.data.activeTargets[] | {i:.labels.instance, h:.health, e:.lastError}'
+> ```
+
+## Dashboards
+
+**Grafana** on `control` (`:3000`), anonymous access disabled. Reachable directly
+over the mesh, and publicly at `https://grafana.cnx.network` via `web01`'s reverse
+proxy (TLS termination — see [Overview](./overview.md)). The admin password is a
+clan var:
+
+```
+clan vars get control grafana-admin/password
+```
+
+Dashboards are provisioned from `modules/monitoring/dashboards/` (any JSON file
+there is picked up):
+
+- **CNX DNS** (`dns.json`) — firing alerts, per-nameserver SOA serials, zone
+  expiry countdowns, query/response rates, host CPU/memory/disk/load, and the
+  outside-in DNS probes.
+- **CNX Backups** (`backups.json`) — borgbackup job health, time since the last
+  run, and per-job state. See [Backups](./backups.md).
+- **CNX Uptime** (`uptime.json`) — per-host up/down status, current uptime,
+  availability over the selected window, and up/down history. Label-driven, so
+  every scraped host appears automatically.
+- **CNX WAN Speed** (`wan.json`) — per-gateway WAN speed from the hourly
+  librespeed runs: latest download/upload/ping, throughput history with each
+  link's 7-day median as reference, latency/jitter, and test status. See
+  [Site gateways](./gateways.md).
+- **parsedmarc** — DMARC aggregate/forensic report viewer. Auto-provisioned by
+  the `parsedmarc` module (not from `dashboards/`); reads its own Elasticsearch
+  datasource, not VictoriaMetrics. See [DMARC reports](#dmarc-reports) below.
+
+## DMARC reports
+
+The `cnx.email` DMARC record (`rua`/`ruf`) points at the `dmarc@cnx.email`
+mailbox on `mx1`. **parsedmarc** on `control` (`modules/monitoring/parsedmarc.nix`)
+polls that mailbox over IMAPS, parses the XML reports, and stores them in a local
+**Elasticsearch** (`127.0.0.1:9200`, loopback-only); Grafana renders them via the
+parsedmarc dashboard + two Elasticsearch datasources (`dmarc-ag`/`dmarc-fo`).
+
+> Two Grafana-13 gotchas, both handled in `modules/monitoring/parsedmarc.nix`:
+>
+> 1. **The Elasticsearch data source plugin is no longer bundled.** Grafana 13
+>    externalised its core data sources and nixpkgs ships no `grafanaPlugins`
+>    entry for Elasticsearch, so a bare ES datasource fails with "Plugin not
+>    registered" and every panel errors (and Explore silently falls back to the
+>    default datasource). We pull the official, Grafana-signed `elasticsearch`
+>    plugin from the catalog via `services.grafana.declarativePlugins`.
+> 2. **Datasource variables now resolve by UID, not name.** The bundled dashboard
+>    binds its datasource through template variables whose stored value is the
+>    datasource _name_ (`dmarc-ag`/`dmarc-fo`); Grafana 11+ matches that against
+>    the datasource **UID**. So we provision the two datasources ourselves with
+>    their **UID pinned to their name** (and leave the module's own
+>    `provision.grafana.datasource` off to avoid a duplicate, uid-less pair).
+>    `deleteDatasources` clears any stale uid-less records left by an earlier
+>    deploy so the pinned-UID versions can be re-created.
+
+The IMAP fetch rides the **mesh**, not the public net: `control` pins
+`mx1.cnx.email` to mx1's mesh address in `/etc/hosts`, so TLS still validates
+against the public cert while the bytes stay on the overlay. The mailbox
+passphrase is the shared `mail-dmarc-cred` clan var (so both mx1's mailserver and
+control's parsedmarc see the same value):
+
+```
+clan vars get mx1 mail-dmarc-cred/passphrase
+```
+
+## Logs
+
+**VictoriaLogs** on `control` (`:9428`), 30-day retention
+(`modules/monitoring/server.nix`). All three hosts ship journald to it via
+systemd's own `services.journald.upload` → the `/insert/journald` endpoint
+(`modules/monitoring/exporters.nix`); no extra agent. `control` uploads over
+loopback so its logs survive a mesh outage, the other hosts push over the mesh,
+and 9428 is firewall-scoped to the mesh like everything else.
+
+> Same IPv4-only default as the scraper: VictoriaLogs binds `0.0.0.0:9428` for a
+> bare `:9428`, so mesh (IPv6) pushes from ns1/ns2 are refused until you pass
+> `extraOptions = [ "-enableTCP6" ]` (binds `[::]`). Verify the bind on `control`:
+>
+> ```
+> ss -tlnp | grep 9428   # want [::]:9428, not 0.0.0.0:9428
+> ```
+
+Query logs from Grafana via the provisioned **VictoriaLogs** datasource (Explore
+view, LogsQL), or directly in the built-in UI at `http://[control]:9428/select/vmui`.
+Logs are tagged with `_HOSTNAME` and `_SYSTEMD_UNIT`, so to follow one service
+across hosts:
+
+```
+_SYSTEMD_UNIT:"knot.service"
+```

docs/src/overview.md

@@ -0,0 +1,37 @@
+# Overview
+
+This is the operational runbook for the **cnx-network** clan. Everything here is
+managed declaratively from the [clan repo](https://git.b4l.co.th/B4L/cnx-network-clan);
+this book is built from `docs/` and served on `control` over the ZeroTier mesh.
+
+## Machines
+
+| Machine    | Role                                   | Public IPv4      | Public IPv6             |
+| ---------- | -------------------------------------- | ---------------- | ----------------------- |
+| `control`  | ZeroTier controller, monitoring, docs  | `77.42.68.181`   | `2a01:4f9:c013:e6d0::1` |
+| `ns1`      | Knot DNS **primary** (master)          | `46.224.170.206` | `2a01:4f8:c014:b5c5::1` |
+| `ns2`      | Knot DNS **secondary** (slave)         | `157.180.70.82`  | `2a01:4f9:c014:6d87::1` |
+| `mx1`      | Mail server (**MX** for cnx.email)     | `5.223.65.38`    | `2a01:4ff:2f0:1963::1`  |
+| `web01`    | Public reverse proxy (TLS termination) | `5.223.55.246`   | `2a01:4ff:2f0:2d8f::1`  |
+| `gw-cnx-1` | Site gateway Chiang Mai (router)       | dynamic (PPPoE)  | —                       |
+
+## Access
+
+- Admin SSH and all internal services ride the **ZeroTier mesh**, not the public
+  net. Public SSH (22) is intentionally closed at the Hetzner cloud firewall.
+- clan reaches machines by their public IPs first (the `internet` instance), with
+  the mesh and Tor as automatic fallbacks.
+
+## Host baseline
+
+Every machine sets a fixed-offset `time.timeZone` and runs **chrony** for time
+sync (`services.chrony.enable`). chrony is the single NTP implementation across
+the fleet — do **not** use `systemd-timesyncd`; the two are mutually exclusive
+and we standardise on chrony everywhere. New hosts must set both the timezone and
+chrony as part of their baseline config.
+
+## Editing these docs
+
+Commit-to-edit: change the markdown under `docs/src/`, commit, and redeploy
+`control`. There is no in-browser editor by design — the docs are versioned and
+reviewed alongside the config that they describe.

flake.lock

@@ -1,5 +1,21 @@
 {
   "nodes": {
+    "blobs": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1604995301,
+        "narHash": "sha256-wcLzgLec6SGJA8fx1OEN1yV/Py5b+U5iyYpksUY/yLw=",
+        "owner": "simple-nixos-mailserver",
+        "repo": "blobs",
+        "rev": "2cccdf1ca48316f2cfd1c9a0017e8de5a7156265",
+        "type": "gitlab"
+      },
+      "original": {
+        "owner": "simple-nixos-mailserver",
+        "repo": "blobs",
+        "type": "gitlab"
+      }
+    },
     "clan-core": {
       "inputs": {
         "data-mesher": "data-mesher",
@@ -7,22 +23,21 @@
         "flake-parts": "flake-parts",
         "nix-darwin": "nix-darwin",
         "nix-select": "nix-select",
-        "nixos-facter-modules": "nixos-facter-modules",
         "nixpkgs": "nixpkgs",
         "sops-nix": "sops-nix",
         "systems": "systems",
         "treefmt-nix": "treefmt-nix"
       },
       "locked": {
-        "lastModified": 1779116933,
-        "narHash": "sha256-BoSQT4vbuWbGcDbXp4nqBd1I9X8r+dLSPtTbc5yJd28=",
-        "rev": "10d86d5d8291fe5162a5535f433a4d50efc4b2bc",
+        "lastModified": 1783497933,
+        "narHash": "sha256-TxmwEews6URFPqOWEHNychtXbFDgLZjbOfEXtvtOm6U=",
+        "rev": "3dc0221ca09033599fe98055e9bbc81bdf32732a",
         "type": "tarball",
-        "url": "https://git.clan.lol/api/v1/repos/clan/clan-core/archive/10d86d5d8291fe5162a5535f433a4d50efc4b2bc.tar.gz"
+        "url": "https://git.clan.lol/api/v1/repos/clan/clan-core/archive/3dc0221ca09033599fe98055e9bbc81bdf32732a.tar.gz"
       },
       "original": {
         "type": "tarball",
-        "url": "https://git.clan.lol/clan/clan-core/archive/25.11.tar.gz"
+        "url": "https://git.clan.lol/clan/clan-core/archive/26.05.tar.gz"
       }
     },
     "data-mesher": {
@@ -41,11 +56,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1767582502,
-        "narHash": "sha256-WVcYGWcAlWzVt38OaTC5i5Q3QkIKJKZsJ7LcQZVVxeE=",
-        "rev": "31f2e3ecf207fd2760e3cebf7c2cf3cb7170ea3d",
+        "lastModified": 1778718524,
+        "narHash": "sha256-pXLoI6Ax0EnUK6r34UM1vibVC7CfTu6j72R2692ZzPs=",
+        "rev": "12c552ad547d87254f33f33bddd1a2cdbeac754d",
         "type": "tarball",
-        "url": "https://git.clan.lol/api/v1/repos/clan/data-mesher/archive/31f2e3ecf207fd2760e3cebf7c2cf3cb7170ea3d.tar.gz"
+        "url": "https://git.clan.lol/api/v1/repos/clan/data-mesher/archive/12c552ad547d87254f33f33bddd1a2cdbeac754d.tar.gz"
       },
       "original": {
         "type": "tarball",
@@ -60,11 +75,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1766150702,
-        "narHash": "sha256-P0kM+5o+DKnB6raXgFEk3azw8Wqg5FL6wyl9jD+G5a4=",
+        "lastModified": 1780290312,
+        "narHash": "sha256-eTAlX0CwgB84Ts3GaBd944A3DRXVMzgA0EqroZBISUo=",
         "owner": "nix-community",
         "repo": "disko",
-        "rev": "916506443ecd0d0b4a0f4cf9d40a3c22ce39b378",
+        "rev": "115e5211780054d8a890b41f0b7734cafad54dfe",
         "type": "github"
       },
       "original": {
@@ -73,6 +88,22 @@
         "type": "github"
       }
     },
+    "flake-compat": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1767039857,
+        "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=",
+        "owner": "NixOS",
+        "repo": "flake-compat",
+        "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
     "flake-parts": {
       "inputs": {
         "nixpkgs-lib": [
@@ -81,11 +112,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1777988971,
-        "narHash": "sha256-qIoWPDs+0/8JecyYgE3gpKQxW/4bLW/gp45vow9ioCQ=",
+        "lastModified": 1778716662,
+        "narHash": "sha256-m1Yf0wZ8j1OHjTc2UwHwyQRSnNeSgLJOd7q5Y45hzi4=",
         "owner": "hercules-ci",
         "repo": "flake-parts",
-        "rev": "0678d8986be1661af6bb555f3489f2fdfc31f6ff",
+        "rev": "f7c1a2d347e4c52d5fb8d10cb4d94b5884e546fb",
         "type": "github"
       },
       "original": {
@@ -94,6 +125,54 @@
         "type": "github"
       }
     },
+    "git-hooks": {
+      "inputs": {
+        "flake-compat": [
+          "nixos-mailserver",
+          "flake-compat"
+        ],
+        "gitignore": "gitignore",
+        "nixpkgs": [
+          "nixos-mailserver",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1778507602,
+        "narHash": "sha256-kTwur1wV+01SdqskVMSo6JMEpg71ps3HpbFY2GsflKs=",
+        "owner": "cachix",
+        "repo": "git-hooks.nix",
+        "rev": "61ab0e80d9c7ab14c256b5b453d8b3fb0189ba0a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "git-hooks.nix",
+        "type": "github"
+      }
+    },
+    "gitignore": {
+      "inputs": {
+        "nixpkgs": [
+          "nixos-mailserver",
+          "git-hooks",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1709087332,
+        "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=",
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "rev": "637db329424fd7e46cf4185293b9cc8c88c95394",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "type": "github"
+      }
+    },
     "nix-darwin": {
       "inputs": {
         "nixpkgs": [
@@ -102,16 +181,15 @@
         ]
       },
       "locked": {
-        "lastModified": 1772129556,
-        "narHash": "sha256-Utk0zd8STPsUJPyjabhzPc5BpPodLTXrwkpXBHYnpeg=",
+        "lastModified": 1779036909,
+        "narHash": "sha256-zXcwYQGCT6pzinK+1dBB2ekTVtfxGZAapb3Evdcu4fY=",
         "owner": "nix-darwin",
         "repo": "nix-darwin",
-        "rev": "ebec37af18215214173c98cf6356d0aca24a2585",
+        "rev": "56c666e108467d87d13508936aade6d567f2a501",
         "type": "github"
       },
       "original": {
         "owner": "nix-darwin",
-        "ref": "nix-darwin-25.11",
         "repo": "nix-darwin",
         "type": "github"
       }
@@ -129,44 +207,56 @@
         "url": "https://git.clan.lol/clan/nix-select/archive/main.tar.gz"
       }
     },
-    "nixos-facter-modules": {
+    "nixos-mailserver": {
+      "inputs": {
+        "blobs": "blobs",
+        "flake-compat": "flake-compat",
+        "git-hooks": "git-hooks",
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
       "locked": {
-        "lastModified": 1773858690,
-        "narHash": "sha256-oW0/lC0oRG5H5LaK6Rmh9L1wmkn9TbenM4bXwnIEDKA=",
-        "owner": "nix-community",
-        "repo": "nixos-facter-modules",
-        "rev": "139dcef4dfc97009629c445806f197883351ab4a",
-        "type": "github"
+        "lastModified": 1785198005,
+        "narHash": "sha256-falBPi+PJtMx0vwII8L24wjDGXNZMwRiVXsw8WTXLEo=",
+        "owner": "simple-nixos-mailserver",
+        "repo": "nixos-mailserver",
+        "rev": "d357b9f048c5532ec81b0e0034c0b8463d5ddd46",
+        "type": "gitlab"
       },
       "original": {
-        "owner": "nix-community",
-        "repo": "nixos-facter-modules",
-        "type": "github"
+        "owner": "simple-nixos-mailserver",
+        "ref": "nixos-26.05",
+        "repo": "nixos-mailserver",
+        "type": "gitlab"
       }
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1778003029,
-        "narHash": "sha256-q/nkKLDtHIyLjZpKhWk3cSK5IYsFqtMd6UtXF3ddjgA=",
-        "owner": "nixos",
-        "repo": "nixpkgs",
-        "rev": "0c88e1f2bdb93d5999019e99cb0e61e1fe2af4c5",
-        "type": "github"
+        "lastModified": 1780453794,
+        "narHash": "sha256-hhAl/iKiurXPn7rdzDgiSuRB8tqOB6f0buWkh8Y9mkY=",
+        "rev": "6b316287bae2ee04c9b93c8c858d930fd07d7338",
+        "type": "tarball",
+        "url": "https://releases.nixos.org/nixos/26.05/nixos-26.05.1183.6b316287bae2/nixexprs.tar.xz"
       },
       "original": {
-        "owner": "nixos",
-        "ref": "nixos-25.11",
-        "repo": "nixpkgs",
-        "type": "github"
+        "type": "tarball",
+        "url": "https://nixos.org/channels/nixos-26.05/nixexprs.tar.xz"
       }
     },
     "root": {
       "inputs": {
         "clan-core": "clan-core",
+        "flake-parts": [
+          "clan-core",
+          "flake-parts"
+        ],
+        "nixos-mailserver": "nixos-mailserver",
         "nixpkgs": [
           "clan-core",
           "nixpkgs"
-        ]
+        ],
+        "treefmt-nix": "treefmt-nix_2"
       }
     },
     "sops-nix": {
@@ -177,11 +267,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1777944972,
-        "narHash": "sha256-VfGRo1qTBKOe3s2gOv8LSoA6Fk19PvBlwQ1ECN0Evn8=",
+        "lastModified": 1780547341,
+        "narHash": "sha256-Gq8KNx5A7hBB3uGJaj6eQfLDIz5YdLu92gqBcvHvoUo=",
         "owner": "Mic92",
         "repo": "sops-nix",
-        "rev": "c591bf665727040c6cc5cb409079acb22dcce33c",
+        "rev": "9ed65852b6257fbeae4355bc24ecfea307ca759a",
         "type": "github"
       },
       "original": {
@@ -192,15 +282,16 @@
     },
     "systems": {
       "locked": {
-        "lastModified": 1681028828,
-        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "lastModified": 1774449309,
+        "narHash": "sha256-brhZ8DmuGtzkCYHJg4HEd602amKm89Y9ytsFZ5uWD1w=",
         "owner": "nix-systems",
         "repo": "default",
-        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "rev": "c29398b59d2048c4ab79345812849c9bd15e9150",
         "type": "github"
       },
       "original": {
         "owner": "nix-systems",
+        "ref": "future-26.11",
         "repo": "default",
         "type": "github"
       }
@@ -213,11 +304,31 @@
         ]
       },
       "locked": {
-        "lastModified": 1768158989,
-        "narHash": "sha256-67vyT1+xClLldnumAzCTBvU0jLZ1YBcf4vANRWP3+Ak=",
+        "lastModified": 1780220602,
+        "narHash": "sha256-eynAfOmbmxJnkp7YewvCEbShNnnYJ9gLLqkzsYtBPeM=",
         "owner": "numtide",
         "repo": "treefmt-nix",
-        "rev": "e96d59dff5c0d7fddb9d113ba108f03c3ef99eca",
+        "rev": "db947814a175b7ca6ded66e21383d938df01c227",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "type": "github"
+      }
+    },
+    "treefmt-nix_2": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1784369104,
+        "narHash": "sha256-47cxbcZODibHv3rELFQ9vZly0vUNkND/atn/U7HLeb0=",
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "rev": "df3c0640565d04a0261253cdd89fce78ec50168a",
         "type": "github"
       },
       "original": {

flake.nix

@@ -1,14 +1,22 @@
 {
-  inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/25.11.tar.gz";
+  inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/26.05.tar.gz";
   inputs.nixpkgs.follows = "clan-core/nixpkgs";
+  inputs.treefmt-nix.url = "github:numtide/treefmt-nix";
+  inputs.treefmt-nix.inputs.nixpkgs.follows = "nixpkgs";
+  # Simple NixOS Mailserver, pinned to the branch matching clan-core's nixpkgs.
+  inputs.nixos-mailserver.url = "gitlab:simple-nixos-mailserver/nixos-mailserver/nixos-26.05";
+  inputs.nixos-mailserver.inputs.nixpkgs.follows = "nixpkgs";
+
+  inputs.flake-parts.follows = "clan-core/flake-parts";
 
   outputs =
-    {
+    inputs@{
       self,
       clan-core,
       nixpkgs,
+      flake-parts,
       ...
-    }@inputs:
+    }:
     let
       # Usage see: https://docs.clan.lol
       clan = clan-core.lib.clan {
@@ -26,29 +34,74 @@
         #     };
         #     overlays = [];
         #   };
-	secrets.age.plugins = [
-        "age-plugin-yubikey"
-        "age-plugin-fido2-hmac"
-      ];
+        secrets.age.plugins = [
+          "age-plugin-yubikey"
+          "age-plugin-fido2-hmac"
+        ];
       };
     in
-    {
-      inherit (clan.config) nixosConfigurations nixosModules clanInternals;
-      clan = clan.config;
-      # Add the Clan cli tool to the dev shell.
-      # Use "nix develop" to enter the dev shell.
-      devShells =
-        nixpkgs.lib.genAttrs
-          [
-            "x86_64-linux"
-            "aarch64-linux"
-            "aarch64-darwin"
-            "x86_64-darwin"
-          ]
-          (system: {
-            default = clan-core.inputs.nixpkgs.legacyPackages.${system}.mkShell {
-              packages = [ clan-core.packages.${system}.clan-cli ];
-            };
-          });
+    flake-parts.lib.mkFlake { inherit inputs; } {
+      systems = [
+        "x86_64-linux"
+        "aarch64-linux"
+        "aarch64-darwin"
+        "x86_64-darwin"
+      ];
+
+      # perSystem.clan.nixosTests: NixOS VM tests of our own clan services.
+      imports = [ clan-core.flakeModules.testModule ];
+
+      flake = {
+        inherit (clan.config) nixosConfigurations nixosModules clanInternals;
+        clan = clan.config;
+      };
+
+      perSystem =
+        { system, ... }:
+        let
+          pkgs = clan-core.inputs.nixpkgs.legacyPackages.${system};
+          treefmtEval = inputs.treefmt-nix.lib.evalModule pkgs ./fmt.nix;
+          clanCli = clan-core.packages.${system}.clan-cli;
+          # `clan machines update a b c` normally runs machines in parallel,
+          # which interleaves their output and buries the YubiKey PIN prompts.
+          # This wrapper (first in PATH) runs them one at a time instead; any
+          # flags fall through to the real CLI untouched.
+          clanSequential = pkgs.writeShellScriptBin "clan" ''
+            if [ "$#" -gt 3 ] && [ "$1" = machines ] && [ "$2" = update ]; then
+              shift 2
+              for arg in "$@"; do
+                case "$arg" in
+                  -*) exec ${clanCli}/bin/clan machines update "$@" ;;
+                esac
+              done
+              for machine in "$@"; do
+                ${clanCli}/bin/clan machines update "$machine" || exit
+              done
+              exit 0
+            fi
+            exec ${clanCli}/bin/clan "$@"
+          '';
+        in
+        {
+          # `nix fmt` and the `nix flake check` formatting gate.
+          formatter = treefmtEval.config.build.wrapper;
+          checks.formatting = treefmtEval.config.build.check self;
+
+          # `nix build .#checks.x86_64-linux.router` (VM test, Linux only).
+          clan.nixosTests.router = {
+            imports = [ ./modules/clan/router/tests/vm ];
+            clan.modules.router = ./modules/clan/router;
+          };
+
+          # Add the Clan cli tool to the dev shell.
+          # Use "nix develop" to enter the dev shell.
+          devShells.default = pkgs.mkShell {
+            packages = [
+              clanSequential
+              clanCli
+              treefmtEval.config.build.wrapper
+            ];
+          };
+        };
     };
 }

fmt.nix

@@ -0,0 +1,33 @@
+# treefmt config, evaluated per-system in flake.nix and exposed as
+# `nix fmt` (the formatter) plus a `nix flake check` formatting gate.
+{ ... }:
+{
+  projectRootFile = "flake.nix";
+
+  programs = {
+    nixfmt.enable = true;
+    prettier.enable = true;
+    yamlfmt.enable = true;
+    shfmt.enable = true;
+  };
+
+  settings = {
+    on-unmatched = "fatal";
+    global.excludes = [
+      # Secrets and clan-managed state — never reformat.
+      "sops/*"
+      "vars/*"
+      "inventory.json"
+
+      # Generated — don't reformat (regeneration would churn the diff).
+      "*facter.json"
+
+      # No formatter, or reformatting would corrupt them.
+      "*.zone" # Knot zone files
+      "docs/book.toml" # mdBook config; no TOML formatter enabled
+      "flake.lock"
+      ".envrc"
+      ".gitignore"
+    ];
+  };
+}

inventory.json

@@ -8,6 +8,15 @@
     },
     "ns2": {
       "installedAt": 1781418857
+    },
+    "mx1": {
+      "installedAt": 1781757322
+    },
+    "web01": {
+      "installedAt": 1781983723
+    },
+    "gw-cnx-1": {
+      "installedAt": 1785494267
     }
   }
 }

inventory.nix

@@ -0,0 +1,95 @@
+# Site gateways: the `router` service instance (modules/clan/router). One
+# instance for the fleet; each site adds its machine with the site's port /
+# VLAN layout under roles.default.machines.<gw>.settings. Everything else
+# (PPPoE, Kea/Blocky, nftables, IPv6-PD, CrowdSec, iperf3, speedtest) follows
+# from the service. Imported by clan.nix.
+{ config, lib, ... }:
+let
+  hosts = import ./modules/hosts.nix;
+  mesh = import ./modules/mesh-hosts.nix {
+    dir = config.directory;
+    inherit lib;
+  };
+in
+{
+  inventory.instances.router = {
+    module = {
+      name = "router";
+      input = "self";
+    };
+
+    # Fleet-wide settings: admin paths (SSH, scrapes, iperf3, Omada UI) are
+    # opened to the mesh; internal proxy names live under
+    # <site><n>.cnx.network with wildcard certs issued via DNS-01 against
+    # ns1 (which must declare the gateway's acme-secret.nix generator).
+    roles.default.settings = {
+      mesh.subnet = mesh.subnet;
+      proxy = {
+        domain = "cnx.network";
+        acme.nameserver = hosts.ns1.ipv4;
+        acme.email = "postmaster@cnx.email";
+      };
+    };
+
+    # Chiang Mai (site 1): Topton 1U, Intel N300, 4x i226-V 2.5G. Port roles
+    # use the expected igc names — verify against facter.json after the
+    # first install and adjust if the box enumerates differently.
+    roles.default.machines.gw-cnx-1.settings = {
+      site = "cnx";
+      siteId = 1;
+      wan.interface = "enp1s0";
+      wan.vlanId = null; # this ISP runs PPPoE untagged on the port
+      wan.macAddress = "a8:b8:e0:01:06:87";
+      trunkPorts = [ "enp2s0" ];
+      # STAGING (remove at cutover, move enp3s0 back into trunkPorts):
+      # DHCP-client uplink into the existing OPNsense LAN so the box has
+      # internet + mesh while it runs alongside the old router.
+      stagingPort = "enp3s0";
+      # Dedicated on-site recovery port: untagged mgmt, always available even
+      # if the switch config is broken.
+      accessPorts.enp4s0 = "mgmt";
+      # Replaces the newedge.house OPNsense box; renumbered to the fleet
+      # convention (10.1.<vlanId>.0/24, router .1, pool .100-.199). The old
+      # untagged LAN becomes tagged mgmt — infra switch ports get PVID 10.
+      vlans = {
+        mgmt = {
+          id = 10; # 10.1.10.0/24 — servers, APs, switches, Omada, admin
+          dhcp.reservations.storinator01 = {
+            hwAddress = "7c:c2:55:e0:d6:40";
+            ipAddress = "10.1.10.53";
+          };
+        };
+        lan.id = 20; # 10.1.20.0/24 — trusted clients
+        iot.id = 40; # 10.1.40.0/24
+        voip.id = 50; # 10.1.50.0/24
+        dmz.id = 60; # 10.1.60.0/24
+        unit1.id = 110; # 10.1.110.0/24
+        unit2.id = 120; # 10.1.120.0/24
+        unit3.id = 130; # 10.1.130.0/24
+        unit4.id = 140; # 10.1.140.0/24
+        unit5 = {
+          id = 150; # 10.1.150.0/24
+          dhcp.reservations.newt = {
+            hwAddress = "7c:d3:0a:21:58:0b";
+            ipAddress = "10.1.150.22";
+          };
+        };
+      };
+      # This site runs the Omada controller for its APs/switches.
+      omada.enable = true;
+      # sshd ban engine (was unconditional before the option existed).
+      crowdsec.enable = true;
+
+      # Internal reverse proxy: real wildcard cert *.cnx1.cnx.network; Blocky
+      # resolves the names to the router's LAN address for mgmt+lan clients.
+      proxy = {
+        enable = true;
+        services.omada = {
+          # Omada's UI is HTTPS with a self-signed cert on the host network.
+          backend = "https://127.0.0.1:8043";
+          insecureSkipVerify = true;
+        };
+      };
+    };
+  };
+}

machines/control/configuration.nix

@@ -1,12 +1,29 @@
+{ config, ... }:
+let
+  hosts = import ../../modules/hosts.nix;
+in
 {
   imports = [
     ../../modules/hetzner-firewall.nix
+    ../../modules/static-ipv6.nix
+    ../../modules/monitoring/exporters.nix
+    ../../modules/monitoring/server.nix
+    ../../modules/monitoring/blackbox.nix
+    ../../modules/monitoring/alerts.nix
+    ../../modules/monitoring/parsedmarc.nix
+    ../../modules/docs.nix
   ];
 
   clan.core.sops.defaultGroups = [ "admins" ];
 
+  # Public IPv6 (from modules/hosts.nix); SLAAC doesn't bring it up here.
+  cnx.staticIPv6 = {
+    enable = true;
+    address = hosts.${config.networking.hostName}.ipv6;
+  };
+
   time.timeZone = "Etc/GMT-3"; # UTC+3 (fixed offset, no DST)
-  services.timesyncd.enable = true;
+  services.chrony.enable = true;
 
   # Public Hetzner Cloud firewalls, synced from this config on every deploy.
   # Rules live in their own data file; see that file for the no-public-SSH note.

machines/control/disko.nix

@@ -1,7 +1,7 @@
 # ---
 # schema = "single-disk"
 # [placeholders]
-# mainDisk = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_120729781" 
+# mainDisk = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_120729781"
 # ---
 # This file was automatically generated!
 # CHANGING this configuration requires wiping and reinstalling the machine

machines/gw-cnx-1/configuration.nix

@@ -0,0 +1,21 @@
+# Site gateway Chiang Mai (site 1): Topton 1U, Intel N300, 4x i226-V 2.5G.
+# The router itself (ports, VLANs, features) is the `router` service instance
+# in clan.nix; only machine-local bits live here.
+{ config, lib, ... }:
+{
+  imports = [ ../../modules/monitoring/exporters.nix ];
+
+  clan.core.sops.defaultGroups = [ "admins" ];
+
+  # Until the install generates facter.json (which normally provides this).
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+
+  # ZFS (disko.nix) needs a stable machine-unique hostId; derive it from the
+  # hostname so every gateway gets one for free when copied for a new site.
+  networking.hostId = builtins.substring 0 8 (
+    builtins.hashString "sha256" config.networking.hostName
+  );
+
+  time.timeZone = "Etc/GMT-7"; # UTC+7 (Thailand, fixed offset, no DST)
+  services.chrony.enable = true;
+}

machines/gw-cnx-1/disko.nix

@@ -0,0 +1,97 @@
+{ ... }:
+let
+  hashDisk = disk: "os-${builtins.substring 0 5 (builtins.hashString "sha256" disk)}";
+  os = "/dev/disk/by-id/ata-TS256GMSA230S_J927900030";
+in
+{
+
+  boot.loader = {
+    systemd-boot = {
+      enable = true;
+    };
+    efi = {
+      canTouchEfiVariables = true;
+    };
+  };
+
+  boot.zfs.forceImportRoot = true;
+
+  disko.devices = {
+    disk = {
+      "os-${hashDisk os}" = {
+        type = "disk";
+        device = os;
+        content = {
+          type = "gpt";
+          partitions = {
+            ESP = {
+              size = "1G";
+              type = "EF00";
+              content = {
+                type = "filesystem";
+                format = "vfat";
+                mountpoint = "/boot";
+                mountOptions = [ "nofail" ];
+              };
+            };
+            system = {
+              size = "100%";
+              content = {
+                type = "zfs";
+                pool = "zroot";
+              };
+            };
+            swap = {
+              size = "16G";
+              content = {
+                type = "swap";
+              };
+            };
+          };
+        };
+      };
+    };
+    zpool = {
+      zroot = {
+        type = "zpool";
+        rootFsOptions = {
+          mountpoint = "none";
+          compression = "lz4";
+          acltype = "posixacl";
+          xattr = "sa";
+          "com.sun:auto-snapshot" = "true";
+        };
+        options.ashift = "12";
+        datasets = {
+          "root" = {
+            type = "zfs_fs";
+            options.mountpoint = "none";
+          };
+          "root/nixos" = {
+            type = "zfs_fs";
+            options.mountpoint = "/";
+            mountpoint = "/";
+          };
+          "root/home" = {
+            type = "zfs_fs";
+            options.mountpoint = "/home";
+            mountpoint = "/home";
+          };
+          "root/tmp" = {
+            type = "zfs_fs";
+            mountpoint = "/tmp";
+            options = {
+              mountpoint = "/tmp";
+              sync = "disabled";
+            };
+          };
+          "root/var" = {
+            type = "zfs_fs";
+            options.mountpoint = "/var";
+            mountpoint = "/var";
+          };
+        };
+      };
+    };
+  };
+}

machines/mx1/configuration.nix

@@ -0,0 +1,23 @@
+{ config, inputs, ... }:
+let
+  hosts = import ../../modules/hosts.nix;
+in
+{
+  imports = [
+    inputs.nixos-mailserver.nixosModules.default
+    ../../modules/mail.nix
+    ../../modules/static-ipv6.nix
+    ../../modules/monitoring/exporters.nix
+  ];
+
+  clan.core.sops.defaultGroups = [ "admins" ];
+
+  # Public IPv6 (from modules/hosts.nix); SLAAC doesn't bring it up here.
+  cnx.staticIPv6 = {
+    enable = true;
+    address = hosts.${config.networking.hostName}.ipv6;
+  };
+
+  time.timeZone = "Etc/GMT-8"; # UTC+8 (Singapore, fixed offset, no DST)
+  services.chrony.enable = true;
+}

machines/mx1/disko.nix

@@ -0,0 +1,50 @@
+# ---
+# schema = "single-disk"
+# [placeholders]
+# mainDisk = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_117494657"
+# ---
+# This file was automatically generated!
+# CHANGING this configuration requires wiping and reinstalling the machine
+{
+
+  boot.loader.grub.efiSupport = true;
+  boot.loader.grub.efiInstallAsRemovable = true;
+  boot.loader.grub.enable = true;
+  disko.devices = {
+    disk = {
+      main = {
+        name = "main-5a0919ffeb6044a39b7d44bba8895ff2";
+        device = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_117494657";
+        type = "disk";
+        content = {
+          type = "gpt";
+          partitions = {
+            "boot" = {
+              size = "1M";
+              type = "EF02"; # for grub MBR
+              priority = 1;
+            };
+            ESP = {
+              type = "EF00";
+              size = "500M";
+              content = {
+                type = "filesystem";
+                format = "vfat";
+                mountpoint = "/boot";
+                mountOptions = [ "umask=0077" ];
+              };
+            };
+            root = {
+              size = "100%";
+              content = {
+                type = "filesystem";
+                format = "ext4";
+                mountpoint = "/";
+              };
+            };
+          };
+        };
+      };
+    };
+  };
+}

machines/ns1/configuration.nix

@@ -1,22 +1,53 @@
-{ config, pkgs, ... }:
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
 let
   domains = import ../../modules/dns/domains.nix;
+  mesh = import ../../modules/mesh-hosts.nix {
+    dir = config.clan.core.settings.directory;
+    inherit lib;
+  };
+  hosts = import ../../modules/hosts.nix;
 in
 {
   imports = [
     ../../modules/dns/authoritative.nix
+    ../../modules/dns/acme-mx1-secret.nix
+    ../../modules/dns/acme-web01-secret.nix
+    # Same shared TSIG secret the gateway's router service declares (proxy).
+    (import ../../modules/clan/router/acme-secret.nix "gw-cnx-1")
+    ../../modules/static-ipv6.nix
+    ../../modules/monitoring/exporters.nix
   ];
 
   clan.core.sops.defaultGroups = [ "admins" ];
 
-  time.timeZone = "Etc/GMT-1"; # UTC+1 (fixed offset, no DST)
-  services.timesyncd.enable = true;
+  # Knot's state dir holds the non-regenerable DNSSEC key material (KSK/ZSK
+  # private keys in the KASP keystore). Declaring it as clan state makes the
+  # borgbackup client back it up; losing it forces an emergency DS rollover at
+  # the registrar. mode 0700 owned by knot, but borg runs as root so it reads it.
+  clan.core.state.knot.folders = [ "/var/lib/knot" ];
 
-  # ACME DNS-01 (RFC 2136): a dedicated TSIG key, scoped to ns1 only, that an
-  # external ACME client uses to write _acme-challenge TXT records. acl_acme
-  # (referenced by each zone below) limits the key to TXT updates at or under
-  # _acme-challenge.<zone>; Knot then signs the record and transfers it to ns2,
-  # which never needs this key. Retrieve the secret for the client with:
+  # The borgbackup repo is addressed as `borg@control`; mesh peers have no name
+  # resolution, so map the control machine name to its ZeroTier mesh address.
+  networking.hosts.${mesh.hosts.control} = [ "control" ];
+
+  # Public IPv6 (from modules/hosts.nix; matches the ns1 AAAA glue); SLAAC
+  # doesn't bring it up here.
+  cnx.staticIPv6 = {
+    enable = true;
+    address = hosts.${config.networking.hostName}.ipv6;
+  };
+
+  time.timeZone = "Etc/GMT-1"; # UTC+1 (fixed offset, no DST)
+  services.chrony.enable = true;
+
+  # ACME DNS-01 (RFC 2136), general key. A dedicated TSIG key scoped by acl_acme
+  # (referenced by every zone below) to TXT updates at or under _acme-challenge.
+  # Retrieve the client config with:
   #   clan vars get ns1 dns-acme-tsig/acme.conf
   clan.core.vars.generators.dns-acme-tsig = {
     files."acme.conf" = {
@@ -30,8 +61,66 @@ in
     '';
   };
 
+  # ACME DNS-01, dedicated mx1 key. A *separate* TSIG key (acme_mx1) that only
+  # mx1 holds, rendered from the shared secret (generator dns-acme-mx1-secret,
+  # imported above). acl_acme_mx1 scopes it to TXT updates at exactly
+  # _acme-challenge.{mx1,mta-sts,mail} (the mail cert and its MTA-STS + client-
+  # alias SANs), and it is attached only to the cnx.email zone below — so this
+  # credential can write nothing but mx1's own cert challenges.
+  clan.core.vars.generators.dns-acme-mx1-knot = {
+    files."acme.conf" = {
+      secret = true;
+      owner = "knot";
+      group = "knot";
+    };
+    dependencies = [ "dns-acme-mx1-secret" ];
+    script = ''
+      printf 'key:\n  - id: acme_mx1\n    algorithm: hmac-sha256\n    secret: %s\n' \
+        "$(cat "$in"/dns-acme-mx1-secret/secret)" > "$out"/acme.conf
+    '';
+  };
+
+  # ACME DNS-01, dedicated web01 key. A *separate* TSIG key (acme_web01) that only
+  # web01 holds, rendered from the shared secret (generator dns-acme-web01-secret,
+  # imported above). acl_acme_web01 scopes it to TXT updates at _acme-challenge on
+  # the cnx.network zone — the owner the wildcard *.cnx.network challenge uses — so
+  # this credential can write nothing but web01's own cert challenges.
+  clan.core.vars.generators.dns-acme-web01-knot = {
+    files."acme.conf" = {
+      secret = true;
+      owner = "knot";
+      group = "knot";
+    };
+    dependencies = [ "dns-acme-web01-secret" ];
+    script = ''
+      printf 'key:\n  - id: acme_web01\n    algorithm: hmac-sha256\n    secret: %s\n' \
+        "$(cat "$in"/dns-acme-web01-secret/secret)" > "$out"/acme.conf
+    '';
+  };
+
+  # ACME DNS-01, dedicated gateway keys. Same pattern as web01: each gateway
+  # holds its own TSIG key (acme_gw_<site>_<n>), rendered from the shared
+  # secret generator imported above. acl_acme_gw_<site>_<n> scopes it to TXT
+  # updates at _acme-challenge.<site><n> on cnx.network — the owner its internal
+  # wildcard *.<site><n>.cnx.network challenge uses — and nothing else.
+  clan.core.vars.generators.dns-acme-gw-cnx-1-knot = {
+    files."acme.conf" = {
+      secret = true;
+      owner = "knot";
+      group = "knot";
+    };
+    dependencies = [ "dns-acme-gw-cnx-1-secret" ];
+    script = ''
+      printf 'key:\n  - id: acme_gw_cnx_1\n    algorithm: hmac-sha256\n    secret: %s\n' \
+        "$(cat "$in"/dns-acme-gw-cnx-1-secret/secret)" > "$out"/acme.conf
+    '';
+  };
+
   services.knot.keyFiles = [
     config.clan.core.vars.generators.dns-acme-tsig.files."acme.conf".path
+    config.clan.core.vars.generators.dns-acme-mx1-knot.files."acme.conf".path
+    config.clan.core.vars.generators.dns-acme-web01-knot.files."acme.conf".path
+    config.clan.core.vars.generators.dns-acme-gw-cnx-1-knot.files."acme.conf".path
   ];
 
   services.knot.settings.acl = [
@@ -44,6 +133,41 @@ in
       "update-owner-match" = "sub-or-equal";
       "update-owner-name" = [ "_acme-challenge" ];
     }
+    {
+      id = "acl_acme_mx1";
+      key = "acme_mx1";
+      action = [ "update" ];
+      "update-type" = [ "TXT" ];
+      "update-owner" = "name";
+      "update-owner-match" = "sub-or-equal";
+      "update-owner-name" = [
+        "_acme-challenge.mx1"
+        "_acme-challenge.mta-sts"
+        "_acme-challenge.mail"
+      ];
+    }
+    {
+      id = "acl_acme_web01";
+      key = "acme_web01";
+      action = [ "update" ];
+      "update-type" = [ "TXT" ];
+      "update-owner" = "name";
+      "update-owner-match" = "sub-or-equal";
+      # Wildcard *.cnx.network places its challenge at _acme-challenge.cnx.network,
+      # i.e. _acme-challenge at the cnx.network apex (where this acl is attached).
+      "update-owner-name" = [ "_acme-challenge" ];
+    }
+    {
+      id = "acl_acme_gw_cnx_1";
+      key = "acme_gw_cnx_1";
+      action = [ "update" ];
+      "update-type" = [ "TXT" ];
+      "update-owner" = "name";
+      "update-owner-match" = "sub-or-equal";
+      # The internal wildcard *.cnx1.cnx.network places its challenge at
+      # _acme-challenge.cnx1.cnx.network.
+      "update-owner-name" = [ "_acme-challenge.cnx1" ];
+    }
   ];
 
   # Automatic DNSSEC signing policy (primary only). ECDSA P-256/SHA-256 with
@@ -58,18 +182,30 @@ in
 
   # ns1 = primary (master): loads each zone from its file and serves it to ns2.
   # zonefile-load = difference-no-serial lets us edit records without touching the
-  # SOA serial; Knot diffs the file, assigns a date-based serial, signs the zone,
-  # then notifies ns2 and lets it pull the signed zone via AXFR/IXFR.
+  # SOA serial; Knot diffs the file, assigns a unixtime serial, signs the zone,
+  # then notifies ns2 and lets it pull the signed zone via AXFR/IXFR. unixtime is
+  # strictly monotonic per reload, so two zone versions can never share a serial
+  # (the failure mode dateserial's 2-digit daily counter allowed after a journal reset).
   services.knot.settings.zone = map (d: {
     domain = d;
     file = ../../modules/dns/zones + "/${d}.zone";
     "zonefile-load" = "difference-no-serial";
     "zonefile-sync" = "-1";
     "journal-content" = "all"; # required by difference-no-serial; holds the live signed zone
-    "serial-policy" = "dateserial";
+    "serial-policy" = "unixtime";
     "dnssec-signing" = true;
     "dnssec-policy" = "cnx";
     notify = [ "ns2" ];
-    acl = [ "acl_ns2" "acl_acme" ]; # ns2 transfers; acme_ddns key does DNS-01 updates
+    # ns2 transfers; acme_ddns does general DNS-01 updates. The dedicated
+    # acme_mx1 key is attached only to cnx.email, so it can't touch other zones.
+    acl = [
+      "acl_ns2"
+      "acl_acme"
+    ]
+    ++ lib.optionals (d == "cnx.email") [ "acl_acme_mx1" ]
+    ++ lib.optionals (d == "cnx.network") [
+      "acl_acme_web01"
+      "acl_acme_gw_cnx_1"
+    ];
   }) domains;
 }

machines/ns1/disko.nix

@@ -1,7 +1,7 @@
 # ---
 # schema = "single-disk"
 # [placeholders]
-# mainDisk = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_120730960" 
+# mainDisk = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_120730960"
 # ---
 # This file was automatically generated!
 # CHANGING this configuration requires wiping and reinstalling the machine

machines/ns2/configuration.nix

@@ -1,16 +1,26 @@
-{ ... }:
+{ config, ... }:
 let
   domains = import ../../modules/dns/domains.nix;
+  hosts = import ../../modules/hosts.nix;
 in
 {
   imports = [
     ../../modules/dns/authoritative.nix
+    ../../modules/static-ipv6.nix
+    ../../modules/monitoring/exporters.nix
   ];
 
   clan.core.sops.defaultGroups = [ "admins" ];
 
+  # Public IPv6 (from modules/hosts.nix; matches the ns2 AAAA glue); SLAAC
+  # doesn't bring it up here.
+  cnx.staticIPv6 = {
+    enable = true;
+    address = hosts.${config.networking.hostName}.ipv6;
+  };
+
   time.timeZone = "Etc/GMT-3"; # UTC+3 (fixed offset, no DST)
-  services.timesyncd.enable = true;
+  services.chrony.enable = true;
 
   # ns2 = secondary (slave): pulls every zone from ns1 and accepts its NOTIFY.
   services.knot.settings.zone = map (d: {

machines/ns2/disko.nix

@@ -1,7 +1,7 @@
 # ---
 # schema = "single-disk"
 # [placeholders]
-# mainDisk = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_120731321" 
+# mainDisk = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_120731321"
 # ---
 # This file was automatically generated!
 # CHANGING this configuration requires wiping and reinstalling the machine

machines/web01/configuration.nix

@@ -0,0 +1,22 @@
+{ config, ... }:
+let
+  hosts = import ../../modules/hosts.nix;
+in
+{
+  imports = [
+    ../../modules/static-ipv6.nix
+    ../../modules/monitoring/exporters.nix
+    ../../modules/web-proxy.nix
+  ];
+
+  clan.core.sops.defaultGroups = [ "admins" ];
+
+  # Public IPv6 (from modules/hosts.nix); SLAAC doesn't bring it up here.
+  cnx.staticIPv6 = {
+    enable = true;
+    address = hosts.${config.networking.hostName}.ipv6;
+  };
+
+  time.timeZone = "Etc/GMT-8"; # UTC+8 (Singapore, fixed offset, no DST)
+  services.chrony.enable = true;
+}

machines/web01/disko.nix

@@ -0,0 +1,50 @@
+# ---
+# schema = "single-disk"
+# [placeholders]
+# mainDisk = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_108706511"
+# ---
+# This file was automatically generated!
+# CHANGING this configuration requires wiping and reinstalling the machine
+{
+
+  boot.loader.grub.efiSupport = true;
+  boot.loader.grub.efiInstallAsRemovable = true;
+  boot.loader.grub.enable = true;
+  disko.devices = {
+    disk = {
+      main = {
+        name = "main-ddd46ebf135244608078712d6ec76691";
+        device = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_108706511";
+        type = "disk";
+        content = {
+          type = "gpt";
+          partitions = {
+            "boot" = {
+              size = "1M";
+              type = "EF02"; # for grub MBR
+              priority = 1;
+            };
+            ESP = {
+              type = "EF00";
+              size = "500M";
+              content = {
+                type = "filesystem";
+                format = "vfat";
+                mountpoint = "/boot";
+                mountOptions = [ "umask=0077" ];
+              };
+            };
+            root = {
+              size = "100%";
+              content = {
+                type = "filesystem";
+                format = "ext4";
+                mountpoint = "/";
+              };
+            };
+          };
+        };
+      };
+    };
+  };
+}

modules/clan/router/README.md

@@ -0,0 +1,92 @@
+# router
+
+Turns a machine with several NICs into a site gateway: PPPoE WAN (ISP
+credentials via vars prompts), a VLAN-filtering bridge over the LAN ports with
+one L3 interface per VLAN, Kea DHCP and Blocky DNS per VLAN, nftables
+firewall/NAT, DHCPv6-PD, an iperf3 server and a WAN speed-test timer.
+Optional: a Wi-Fi access point on the router's own radios (hostapd), CrowdSec
+with the nftables bouncer (sshd log parsing), the TP-Link Omada controller
+(podman) and an internal Caddy reverse proxy with a real wildcard certificate
+(ACME DNS-01).
+
+Addressing convention: a site owns `10.<siteId>.0.0/16`; VLAN `<id>` defaults
+to `10.<siteId>.<id>.0/24`, router at `.1`, DHCP pool `.100-.199`. The `mgmt`
+and `lan` VLANs are mandatory. Trust model: mgmt reaches everything; other
+VLANs get router DNS/DHCP and (with `allowWan`) the internet, no inter-VLAN;
+WAN nothing inbound; the admin mesh (`mesh.subnet`) gets SSH, metrics, iperf3
+and the Omada UI.
+
+## Usage from another clan
+
+```nix
+# flake.nix
+inputs.cnx-network.url = "git+https://<host>/B4L/cnx-network-clan";
+
+# clan.nix
+inventory.instances.router = {
+  module = { name = "router"; input = "cnx-network"; };
+  roles.default.settings.mesh.subnet = "fd..::/88"; # your admin overlay
+  roles.default.machines.gw-1.settings = {
+    site = "ams";
+    siteId = 1;
+    wan.interface = "enp1s0";
+    wan.vlanId = 10;                 # or null for untagged PPPoE
+    trunkPorts = [ "enp2s0" ];
+    accessPorts.enp4s0 = "mgmt";     # untagged on-site recovery port
+    # stagingPort = "enp3s0";       # DHCP uplink into the old LAN until cutover
+    vlans = {
+      mgmt.id = 10;
+      lan.id = 20;
+      iot = { id = 40; allowWan = false; };
+    };
+  };
+};
+```
+
+Then `clan vars generate gw-1` prompts for the PPPoE username/password.
+
+### Wi-Fi access point
+
+If the box has wireless cards, the router can be the site's AP. An SSID is
+defined once and behaves like an untagged access port of its VLAN; radios
+choose what to broadcast, so a dual-band card serves the same SSID twice:
+
+```nix
+wifi = {
+  enable = true;
+  countryCode = "TH";
+  networks = {
+    home.vlan = "lan";                                   # WPA3 with WPA2 fallback
+    things = { vlan = "iot"; security = "wpa2"; };       # legacy IoT
+    guest = { vlan = "guest"; isolateClients = true; };
+  };
+  radios = {
+    wlp5s0 = { band = "2g"; channel = 6; macAddress = "…"; networks = [ "home" "things" ]; };
+    wlp6s0 = { band = "5g"; channel = 36; networks = [ "home" ]; };
+  };
+};
+```
+
+Passphrases are vars prompts (`wifi-<name>-passphrase`), asked once at `clan
+vars generate`. A radio broadcasting more than one SSID needs its hardware
+`macAddress`: hostapd wants a fixed BSSID per extra SSID, derived from it.
+`security = "wpa3-transition"` (the default) offers SAE and WPA2-PSK-SHA256;
+devices that only speak classic WPA2-PSK need `security = "wpa2"`.
+
+### Internal proxy
+
+`proxy.enable` serves `<name>.<site><siteId>.<proxy.domain>` under a wildcard
+certificate obtained via RFC 2136 DNS-01 against `proxy.acme.nameserver`. The
+gateway signs updates with TSIG key `acme_<hostname with _>`, whose secret is
+the shared `dns-acme-<hostname>-secret` generator declared by this service.
+The nameserver machine must declare the same generator so both sides hold one
+secret — import `acme-secret.nix` from this directory with the gateway's name:
+
+```nix
+imports = [ (import "${inputs.cnx-network}/modules/clan/router/acme-secret.nix" "gw-1") ];
+```
+
+and load the key with an acl scoped to `_acme-challenge.<site><siteId>`.
+
+The service does not open the WAN to anything; reach gateways over your mesh.
+One instance per machine.

modules/clan/router/acme-secret.nix

@@ -0,0 +1,22 @@
+# Shared TSIG secret for a gateway's dedicated ACME key (function: machine
+# name -> NixOS module). The acme_<machine> key lets that gateway — and only
+# it — write _acme-challenge.<site><siteId> TXT records on the authoritative
+# nameserver to obtain its internal wildcard cert via DNS-01 (proxy.nix).
+#
+# The router service declares it on the gateway automatically when
+# proxy.enable is set. The nameserver machine must declare the very same
+# generator so both sides share one secret:
+#   imports = [ (import <router-service>/acme-secret.nix "gw-cnx-1") ];
+# and then load it into its DNS server as key acme_gw_cnx_1 (hmac-sha256)
+# with an acl scoped to that gateway's _acme-challenge label.
+machine:
+{ pkgs, ... }:
+{
+  clan.core.vars.generators."dns-acme-${machine}-secret" = {
+    share = true;
+    files."secret".secret = true;
+    runtimeInputs = [ pkgs.openssl ];
+    # 32 random bytes, base64 — a valid hmac-sha256 TSIG secret.
+    script = ''openssl rand -base64 32 | tr -d '\n' > "$out"/secret'';
+  };
+}

modules/clan/router/crowdsec.nix

@@ -0,0 +1,49 @@
+# CrowdSec security engine + nftables bouncer: parses sshd auth attempts from
+# the journal and bans offending source IPs at the firewall. Log-based (no
+# inline DPI) so it costs the N300 next to nothing. Opt-in per site
+# (`crowdsec.enable`): the hub sync needs internet at activation time.
+{ settings }:
+{ lib, ... }:
+let
+  cfg = settings;
+in
+{
+  config = lib.mkIf cfg.crowdsec.enable {
+    services.crowdsec = {
+      enable = true;
+      autoUpdateService = true;
+      hub.collections = [
+        "crowdsecurity/linux"
+        "crowdsecurity/sshd"
+      ];
+      localConfig = {
+        acquisitions = [
+          {
+            source = "journalctl";
+            journalctl_filter = [ "_SYSTEMD_UNIT=sshd.service" ];
+            labels.type = "syslog";
+          }
+        ];
+        # Never ban the ZeroTier mesh — it is the only admin path to these
+        # boxes (no public SSH), so a false positive would lock us out.
+        # Parser-stage whitelist: mesh events are dropped before any scenario.
+        parsers.s02Enrich = [
+          {
+            name = "cnx/mesh-whitelist";
+            description = "Whitelist the ZeroTier management mesh";
+            whitelist = {
+              reason = "ZeroTier mesh is the admin path";
+              cidr = [ cfg.mesh.subnet ];
+            };
+          }
+        ];
+      };
+    };
+
+    services.crowdsec-firewall-bouncer = {
+      enable = true;
+      registerBouncer.enable = true;
+      settings.mode = "nftables";
+    };
+  };
+}

modules/clan/router/default.nix

@@ -0,0 +1,67 @@
+# Site gateway (OPNsense replacement) for the Topton 1U boxes, as a clan
+# service: PPPoE WAN, VLAN-filtering bridge over the LAN ports, per-VLAN
+# DHCP/DNS, firewall/NAT, and the optional Omada controller / internal proxy.
+#
+# Exported as `clan.modules.router` of this flake; used here with
+# `module.input = "self"` and from other clans with `module.input =
+# "<this flake's input name>"`. README.md has the consumer view. A site joins
+# through the inventory:
+#
+#   inventory.instances.router = {
+#     module = { name = "router"; input = "self"; };
+#     roles.default.settings.mesh.subnet = ...;   # fleet-wide
+#     roles.default.machines.gw-<city>-<n>.settings = { site = ...; ... };
+#   };
+#
+# The settings schema lives in interface.nix. Implementation files that need
+# the settings are functions `{ settings }: <NixOS module>`; the evaluated
+# settings are handed in with importApply so nothing goes through
+# machine-level options. The rest are plain NixOS modules.
+{ lib, ... }:
+{
+  _class = "clan.service";
+  manifest.name = "router";
+  manifest.description = "Site gateway: PPPoE WAN, VLAN bridge, DHCP/DNS, firewall/NAT";
+  manifest.categories = [ "Network" ];
+  manifest.readme = builtins.readFile ./README.md;
+
+  roles.default = {
+    description = "Turns the machine into the site's router (one instance per machine).";
+    interface = ./interface.nix;
+
+    perInstance =
+      { settings, machine, ... }:
+      {
+        nixosModule.imports = [
+          ./ipv6.nix
+        ]
+        # The proxy's TSIG secret is shared with the nameserver (acme-secret.nix).
+        ++ lib.optional settings.proxy.enable (import ./acme-secret.nix machine.name)
+        ++ map (file: lib.modules.importApply file { inherit settings; }) [
+          ./network.nix
+          ./pppoe.nix
+          ./firewall.nix
+          ./dns-dhcp.nix
+          ./crowdsec.nix
+          ./omada.nix
+          ./proxy.nix
+          ./iperf.nix
+          ./speedtest.nix
+          ./wifi.nix
+        ];
+      };
+  };
+
+  # A machine has exactly one WAN port and one VLAN layout; two instances would
+  # both claim br0/ppp0 and fight over Kea/Blocky/nftables.
+  perMachine =
+    { instances, machine, ... }:
+    {
+      nixosModule.assertions = [
+        {
+          assertion = lib.length (lib.attrNames instances) == 1;
+          message = "router: ${machine.name} is a gateway in several instances (${lib.concatStringsSep ", " (lib.attrNames instances)}); a machine can only be one router.";
+        }
+      ];
+    };
+}

modules/clan/router/dns-dhcp.nix

@@ -0,0 +1,98 @@
+# LAN DHCP (Kea) and DNS (Blocky). Fully declarative: one Kea subnet per VLAN
+# with dhcp.enable, Blocky as the blocklist resolver every DHCP lease points
+# at. Blocky's HTTP listener (:4000) serves Prometheus metrics, scraped by
+# control over the mesh (firewall.nix scopes it to the mesh subnet).
+{ settings }:
+{ lib, ... }:
+let
+  cfg = settings;
+  dhcpVlans = lib.filterAttrs (_: vlan: vlan.dhcp.enable) cfg.vlans;
+in
+{
+  services.kea.dhcp4 = {
+    enable = true;
+    settings = {
+      interfaces-config = {
+        interfaces = lib.mapAttrsToList (name: _: "vlan-${name}") dhcpVlans;
+        # The unit orders after network-online.target, which under networkd
+        # only waits for the WAN carrier (the vlan-* links are
+        # RequiredForOnline=no), so Kea can start before vlan-* have their
+        # addresses. By default it then logs the failed bind and runs with no
+        # socket at all: clients' DISCOVERs reach vlan-lan and nobody answers.
+        # Insist on every socket and keep retrying while networkd catches up;
+        # if it still cannot bind, exit and let systemd restart the unit.
+        service-sockets-require-all = true;
+        service-sockets-max-retries = 60;
+        service-sockets-retry-wait-time = 1000;
+      };
+      lease-database = {
+        type = "memfile";
+        persist = true;
+        name = "/var/lib/kea/dhcp4.leases";
+      };
+      valid-lifetime = 86400;
+      subnet4 = lib.mapAttrsToList (name: vlan: {
+        id = vlan.id;
+        subnet = vlan.subnet;
+        interface = "vlan-${name}";
+        valid-lifetime = vlan.dhcp.leaseTime;
+        pools = [ { pool = "${vlan.dhcp.pool.from} - ${vlan.dhcp.pool.to}"; } ];
+        reservations = lib.mapAttrsToList (host: res: {
+          hostname = host;
+          hw-address = res.hwAddress;
+          ip-address = res.ipAddress;
+        }) vlan.dhcp.reservations;
+        option-data = [
+          {
+            name = "routers";
+            data = vlan.address;
+          }
+          {
+            name = "domain-name-servers";
+            data = vlan.address;
+          }
+        ];
+      }) dhcpVlans;
+    };
+  };
+
+  # The nixpkgs unit already has Restart=on-failure; space the restarts out so
+  # a persistent bind failure does not trip the start-rate limit.
+  systemd.services.kea-dhcp4-server.serviceConfig.RestartSec = 5;
+
+  services.blocky = {
+    enable = true;
+    settings = {
+      ports = {
+        dns = 53;
+        http = 4000;
+      };
+      upstreams.groups.default = [
+        "9.9.9.9"
+        "149.112.112.112"
+        "2620:fe::fe"
+      ];
+      blocking = {
+        denylists.ads = [
+          "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts"
+        ];
+        clientGroupsBlock.default = [ "ads" ];
+      };
+      caching = {
+        minTime = "5m";
+        prefetching = true;
+      };
+      prometheus.enable = true;
+    };
+  };
+
+  # The router itself resolves via public resolvers, not via Blocky, so DNS
+  # for deploys/updates survives a broken local resolver. networkd would
+  # enable systemd-resolved by default, whose stub listener on 127.0.0.53:53
+  # makes Blocky's wildcard :53 bind fail — plain resolv.conf instead.
+  services.resolved.enable = false;
+  networking.nameservers = [
+    "9.9.9.9"
+    "1.1.1.1"
+  ];
+}

modules/clan/router/firewall.nix

@@ -0,0 +1,91 @@
+# Router firewall/NAT policy (nftables). Trust model:
+#   mgmt VLAN  -> trusted: router services, all VLANs, WAN
+#   other VLANs -> DNS/DHCP on the router + WAN (if allowWan); no inter-VLAN
+#   WAN (ppp0) -> nothing inbound beyond established/related
+#   mesh       -> admin SSH + metrics scrapes (same trust boundary as the fleet)
+#   staging    -> admin SSH only inbound (pre-cutover uplink into the old LAN);
+#                 allowWan VLANs are NATed out through it while ppp0 is down
+{ settings }:
+{ lib, ... }:
+let
+  cfg = settings;
+
+  vlanIfs = lib.mapAttrsToList (name: _: "vlan-${name}") cfg.vlans;
+  wanVlanIfs = lib.mapAttrsToList (name: _: "vlan-${name}") (
+    lib.filterAttrs (_: vlan: vlan.allowWan) cfg.vlans
+  );
+  nonMgmtIfs = lib.filter (i: i != "vlan-mgmt") vlanIfs;
+
+  # allowWan VLANs may also leave through the staging uplink. Same set as
+  # networking.nat.internalInterfaces below, so `allowWan` holds on both
+  # exits. The kernel picks the exit: ppp0 (metric 0, see pppoe.nix) while
+  # the session is up, the staging DHCP route (metric 1024) otherwise.
+  stagingExit = cfg.stagingPort != null && wanVlanIfs != [ ];
+  wanVlanSet = "{ ${lib.concatMapStringsSep ", " (i: ''"${i}"'') wanVlanIfs} }";
+in
+{
+  networking.nftables.enable = true;
+
+  # SSH reachable only from the mgmt VLAN (trusted) and the mesh — never
+  # from the WAN or the other VLANs.
+  services.openssh.openFirewall = false;
+
+  networking.firewall = {
+    enable = true;
+    filterForward = true;
+    trustedInterfaces = [ "vlan-mgmt" ];
+
+    # Non-mgmt VLANs may only talk to the router's DNS and DHCP; the staging
+    # uplink (old LAN, pre-cutover) gets admin SSH so the box can be reached
+    # before the mgmt VLAN or the mesh are up.
+    interfaces =
+      lib.genAttrs nonMgmtIfs (_: {
+        allowedTCPPorts = [ 53 ];
+        allowedUDPPorts = [
+          53
+          67
+        ];
+      })
+      // lib.optionalAttrs (cfg.stagingPort != null) {
+        ${cfg.stagingPort}.allowedTCPPorts = [ 22 ];
+      };
+
+    extraInputRules = ''
+      ip6 saddr ${cfg.mesh.subnet} tcp dport 22 accept comment "admin ssh over the mesh"
+      ip6 saddr ${cfg.mesh.subnet} tcp dport 4000 accept comment "blocky metrics scrape from control"
+    '';
+
+    extraForwardRules = ''
+      tcp flags syn tcp option maxseg size set rt mtu comment "MSS clamp for PPPoE mtu 1492"
+      iifname "vlan-mgmt" accept comment "mgmt reaches all VLANs and the WAN"
+    ''
+    + lib.optionalString stagingExit ''
+      iifname ${wanVlanSet} oifname "${cfg.stagingPort}" accept comment "allowWan VLANs out via the staging uplink"
+    '';
+  };
+
+  # networking.nat both masquerades and opens forward-to-WAN for exactly its
+  # internalInterfaces — so this list, not a rule of our own, is where
+  # `allowWan` is enforced. Listing every VLAN here would silently let
+  # allowWan = false VLANs out.
+  networking.nat = {
+    enable = true;
+    externalInterface = "ppp0";
+    internalInterfaces = wanVlanIfs;
+  };
+
+  # networking.nat only masquerades on its single externalInterface; the
+  # staging uplink needs its own postrouting chain (nixos-nat's is
+  # oifname-scoped to ppp0, so the two never both apply).
+  networking.nftables.tables = lib.optionalAttrs stagingExit {
+    router-staging-nat = {
+      family = "ip";
+      content = ''
+        chain post {
+          type nat hook postrouting priority srcnat;
+          iifname ${wanVlanSet} oifname "${cfg.stagingPort}" masquerade comment "allowWan VLANs out via the staging uplink"
+        }
+      '';
+    };
+  };
+}

modules/clan/router/interface.nix

@@ -0,0 +1,382 @@
+# Settings of the `router` service (inventory `roles.default.settings`).
+# Pure schema: no machine config is reachable here; the implementation files
+# get the evaluated result as `settings`.
+#
+# Fleet addressing convention: each site owns 10.<siteId>.0.0/16. A VLAN's
+# subnet defaults to 10.<siteId>.<vlanId>.0/24 with the router at .1 and the
+# DHCP pool at .100-.199. VLANs that need more space (e.g. public-wifi guest)
+# override `subnet`/`address`/`dhcp.pool` and take a wider block from the
+# upper half (10.<siteId>.128.0/17), e.g. guest -> 10.<siteId>.128.0/22.
+# VLAN ids: 10 = mgmt, 20 = lan (mandatory); 30 = guest, 40 = iot (reserved).
+{ config, lib, ... }:
+let
+  site = toString config.siteId;
+
+  vlanModule =
+    { config, ... }:
+    let
+      octet = toString config.id;
+    in
+    {
+      options = {
+        id = lib.mkOption {
+          type = lib.types.ints.between 1 4094;
+          description = "802.1Q VLAN id (fleet convention: 10 mgmt, 20 lan, 30 guest, 40 iot).";
+        };
+        address = lib.mkOption {
+          type = lib.types.str;
+          default = "10.${site}.${octet}.1";
+          defaultText = lib.literalExpression ''"10.<siteId>.<id>.1"'';
+          description = "Router address on this VLAN.";
+        };
+        prefixLength = lib.mkOption {
+          type = lib.types.ints.between 8 30;
+          default = 24;
+        };
+        subnet = lib.mkOption {
+          type = lib.types.str;
+          default = "10.${site}.${octet}.0/24";
+          defaultText = lib.literalExpression ''"10.<siteId>.<id>.0/24"'';
+          description = "The VLAN's network in CIDR form (must contain `address`).";
+        };
+        dhcp = {
+          enable = lib.mkOption {
+            type = lib.types.bool;
+            default = true;
+          };
+          pool = {
+            from = lib.mkOption {
+              type = lib.types.str;
+              default = "10.${site}.${octet}.100";
+              defaultText = lib.literalExpression ''"10.<siteId>.<id>.100"'';
+            };
+            to = lib.mkOption {
+              type = lib.types.str;
+              default = "10.${site}.${octet}.199";
+              defaultText = lib.literalExpression ''"10.<siteId>.<id>.199"'';
+            };
+          };
+          leaseTime = lib.mkOption {
+            type = lib.types.ints.positive;
+            default = 86400;
+            description = ''
+              Lease validity in seconds. Lower it for high-churn networks,
+              e.g. public-WiFi guest VLANs (3600-7200), so the pool recycles.
+            '';
+          };
+          reservations = lib.mkOption {
+            type = lib.types.attrsOf (
+              lib.types.submodule {
+                options = {
+                  hwAddress = lib.mkOption {
+                    type = lib.types.str;
+                    example = "aa:bb:cc:dd:ee:ff";
+                    description = "Client MAC address.";
+                  };
+                  ipAddress = lib.mkOption {
+                    type = lib.types.str;
+                    description = "Fixed address handed to this client (inside the VLAN's subnet, outside the pool).";
+                  };
+                };
+              }
+            );
+            default = { };
+            description = "Static DHCP leases; the attribute name becomes the client's hostname.";
+          };
+        };
+        allowWan = lib.mkOption {
+          type = lib.types.bool;
+          default = true;
+          description = "Whether clients on this VLAN may reach the internet.";
+        };
+      };
+    };
+
+  proxyServiceModule = {
+    options = {
+      backend = lib.mkOption {
+        type = lib.types.str;
+        example = "https://127.0.0.1:8043";
+        description = "URL Caddy forwards to (internal/mesh address).";
+      };
+      insecureSkipVerify = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = "Skip TLS verification towards the backend (self-signed upstreams like Omada).";
+      };
+    };
+  };
+
+  wifiNetworkModule =
+    { name, ... }:
+    {
+      options = {
+        ssid = lib.mkOption {
+          type = lib.types.str;
+          default = name;
+          defaultText = lib.literalExpression "<attribute name>";
+          description = "SSID as advertised; defaults to the attribute name.";
+        };
+        vlan = lib.mkOption {
+          type = lib.types.str;
+          example = "lan";
+          description = "VLAN (by name) the clients of this SSID land in, like an untagged access port.";
+        };
+        security = lib.mkOption {
+          type = lib.types.enum [
+            "wpa3"
+            "wpa3-transition"
+            "wpa2"
+            "open"
+          ];
+          default = "wpa3-transition";
+          description = ''
+            - `wpa3`: WPA3-Personal (SAE) only.
+            - `wpa3-transition`: WPA3 with WPA2 fallback for older clients.
+            - `wpa2`: WPA2-PSK only, for legacy IoT devices.
+            - `open`: no encryption (captive/guest use; pair with an isolated VLAN).
+            Except for `open`, the passphrase is a vars prompt
+            (`wifi-<name>-passphrase`, entered at `clan vars generate`).
+          '';
+        };
+        hidden = lib.mkOption {
+          type = lib.types.bool;
+          default = false;
+          description = "Do not advertise the SSID in beacons (clients must know it).";
+        };
+        isolateClients = lib.mkOption {
+          type = lib.types.bool;
+          default = false;
+          description = "Keep wireless clients of this SSID from talking to each other (guest networks).";
+        };
+      };
+    };
+
+  wifiRadioModule = {
+    options = {
+      band = lib.mkOption {
+        type = lib.types.enum [
+          "2g"
+          "5g"
+          "6g"
+        ];
+        default = "2g";
+        description = "Frequency band of this radio; a dual-band card exposes one radio interface per band.";
+      };
+      channel = lib.mkOption {
+        type = lib.types.ints.unsigned;
+        default = 0;
+        description = "Channel; 0 lets hostapd pick one (ACS) — not every driver supports that.";
+      };
+      wifi6 = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = "Enable 802.11ax (WiFi 6) on this radio; WiFi 4/5 are always on.";
+      };
+      macAddress = lib.mkOption {
+        type = lib.types.nullOr lib.types.str;
+        default = null;
+        example = "aa:bb:cc:dd:ee:ff";
+        description = ''
+          Hardware address of the radio (facter.json / `ip link`). Needed when
+          the radio serves more than one network: hostapd wants a fixed BSSID
+          per extra network, derived from this address (locally administered
+          variants of its first octet).
+        '';
+      };
+      networks = lib.mkOption {
+        type = lib.types.listOf lib.types.str;
+        example = [
+          "home"
+          "iot"
+        ];
+        description = "Networks (from `wifi.networks`) this radio broadcasts; at most four per radio.";
+      };
+    };
+  };
+in
+{
+  options = {
+    site = lib.mkOption {
+      type = lib.types.str;
+      description = "City code of the site, e.g. \"cnx\".";
+    };
+
+    siteId = lib.mkOption {
+      type = lib.types.ints.between 1 254;
+      description = "Site number; drives the 10.<siteId>.<vlan>.0/24 addressing.";
+    };
+
+    wan.interface = lib.mkOption {
+      type = lib.types.str;
+      description = "Physical WAN port the PPPoE session runs on.";
+    };
+
+    wan.vlanId = lib.mkOption {
+      type = lib.types.nullOr (lib.types.ints.between 1 4094);
+      default = null;
+      description = ''
+        802.1Q tag the ISP requires for the PPPoE session (AIS Thailand: 10);
+        null for untagged PPPoE directly on the port. Unrelated to the LAN
+        VLANs — this tag exists only on the WAN port.
+      '';
+    };
+
+    wan.macAddress = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      example = "aa:bb:cc:dd:ee:ff";
+      description = ''
+        Spoofed MAC for the WAN port, e.g. to keep the MAC the ISP has
+        pinned (cloned from the old router). null keeps the hardware MAC.
+      '';
+    };
+
+    trunkPorts = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      description = "LAN ports carrying all VLANs tagged (incl. any 10G SFP+ ports).";
+    };
+
+    accessPorts = lib.mkOption {
+      type = lib.types.attrsOf lib.types.str;
+      default = { };
+      example = {
+        enp4s0 = "mgmt";
+      };
+      description = ''
+        Ports acting as untagged access ports on a single VLAN (port name ->
+        VLAN name). Frames are untagged on the wire; the bridge tags them with
+        the VLAN's PVID. Use for an always-available on-site mgmt port.
+      '';
+    };
+
+    stagingPort = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      example = "enp3s0";
+      description = ''
+        Temporary DHCPv4-client uplink into the existing LAN while the box
+        runs alongside the router it replaces: gives it (and, NATed, the
+        allowWan VLANs) internet + mesh before the WAN port is cabled; once
+        the PPPoE session is up its default route wins, and the staging
+        route only carries traffic again if the session drops (PPPoE simply
+        retries until then). The port is in no VLAN zone; inbound, the
+        firewall admits only SSH on it. Do NOT connect the trunk ports to
+        the production switch while staging — Kea on the mgmt tag would
+        fight the old router's DHCP in one broadcast domain. Set to null at
+        cutover (and usually hand the port back to `trunkPorts`).
+      '';
+    };
+
+    vlans = lib.mkOption {
+      type = lib.types.attrsOf (lib.types.submodule vlanModule);
+      description = "VLANs served at this site; `mgmt` and `lan` are mandatory.";
+    };
+
+    mesh.subnet = lib.mkOption {
+      type = lib.types.str;
+      example = "fd12:3456:789a:bcde:f000::/88";
+      description = ''
+        IPv6 prefix of the admin mesh (the ZeroTier overlay; see
+        modules/mesh-hosts.nix). Admin SSH, metrics scrapes, iperf3 and the
+        Omada UI accept connections from it, and CrowdSec never bans it.
+      '';
+    };
+
+    omada.enable = lib.mkEnableOption "TP-Link Omada SDN controller (podman container)";
+
+    crowdsec.enable = lib.mkEnableOption "CrowdSec (sshd log parsing) with the nftables bouncer";
+
+    proxy = {
+      enable = lib.mkEnableOption "internal reverse proxy (Caddy, wildcard cert via DNS-01)";
+
+      domain = lib.mkOption {
+        type = lib.types.str;
+        example = "example.net";
+        description = ''
+          Parent zone of the proxy names: services are served as
+          <name>.<site><siteId>.<domain> under a wildcard certificate.
+        '';
+      };
+
+      acme = {
+        nameserver = lib.mkOption {
+          type = lib.types.str;
+          example = "203.0.113.53";
+          description = ''
+            Authoritative nameserver of `domain` that accepts RFC 2136
+            updates for _acme-challenge.<site><siteId> with this gateway's
+            TSIG key (acme_<hostname with _>, secret from the shared
+            dns-acme-<hostname>-secret generator, see acme-secret.nix).
+          '';
+        };
+        email = lib.mkOption {
+          type = lib.types.str;
+          example = "postmaster@example.net";
+          description = "ACME account contact.";
+        };
+      };
+
+      services = lib.mkOption {
+        type = lib.types.attrsOf (lib.types.submodule proxyServiceModule);
+        default = { };
+        description = "Proxied services; attr name becomes <name>.<site><siteId>.<domain>.";
+      };
+
+      allowVlans = lib.mkOption {
+        type = lib.types.listOf lib.types.str;
+        default = [
+          "mgmt"
+          "lan"
+        ];
+        description = "VLANs whose clients may reach the proxy (443, plus 80 for the redirect).";
+      };
+    };
+
+    wifi = {
+      enable = lib.mkEnableOption "a Wi-Fi access point on the router's own radios (hostapd)";
+
+      countryCode = lib.mkOption {
+        type = lib.types.nullOr lib.types.str;
+        default = null;
+        example = "TH";
+        description = "ISO 3166-1 country code for the regulatory domain; required when enabled.";
+      };
+
+      networks = lib.mkOption {
+        type = lib.types.attrsOf (lib.types.submodule wifiNetworkModule);
+        default = { };
+        example = {
+          home.vlan = "lan";
+          things = {
+            vlan = "iot";
+            security = "wpa2";
+          };
+        };
+        description = ''
+          Wireless networks (SSIDs). Each one behaves like an untagged access
+          port on its VLAN; the radios below choose which to broadcast.
+        '';
+      };
+
+      radios = lib.mkOption {
+        type = lib.types.attrsOf (lib.types.submodule wifiRadioModule);
+        default = { };
+        example = {
+          wlp5s0 = {
+            band = "5g";
+            channel = 36;
+            networks = [ "home" ];
+          };
+        };
+        description = "Wireless radios of the router (interface name -> config); at least one when enabled.";
+      };
+    };
+
+    speedtest.interval = lib.mkOption {
+      type = lib.types.str;
+      default = "hourly";
+      description = "systemd OnCalendar spec for the WAN speed test.";
+    };
+  };
+}

modules/clan/router/iperf.nix

@@ -0,0 +1,22 @@
+# iperf3 server on every gateway, for throughput testing from any LAN segment
+# (e.g. validating AP/switch links: `iperf3 -c 10.<siteId>.<vlan>.1`) and from
+# admin machines over the mesh. Never reachable from the WAN (default-deny).
+{ settings }:
+{ lib, ... }:
+let
+  cfg = settings;
+  vlanIfs = lib.mapAttrsToList (name: _: "vlan-${name}") cfg.vlans;
+in
+{
+  services.iperf3.enable = true;
+
+  networking.firewall.interfaces = lib.genAttrs vlanIfs (_: {
+    allowedTCPPorts = [ 5201 ];
+    allowedUDPPorts = [ 5201 ];
+  });
+
+  networking.firewall.extraInputRules = ''
+    ip6 saddr ${cfg.mesh.subnet} tcp dport 5201 accept comment "iperf3 over the mesh"
+    ip6 saddr ${cfg.mesh.subnet} udp dport 5201 accept comment "iperf3 over the mesh"
+  '';
+}

modules/clan/router/ipv6.nix

@@ -0,0 +1,22 @@
+# IPv6 on the PPPoE uplink: run networkd's DHCPv6 client on ppp0 to obtain a
+# delegated prefix; each vlan-* interface (network.nix) carves a /64 out of it
+# via DHCPPrefixDelegation and announces it to clients with SLAAC.
+{ lib, ... }:
+{
+  systemd.network.networks."45-ppp0" = {
+    matchConfig.Name = "ppp0";
+    networkConfig = {
+      DHCP = "ipv6";
+      # pppd owns the v4 address/route on this link; don't let networkd
+      # tear them down.
+      KeepConfiguration = "static";
+      # Default v6 route comes from the ISP's RA when they send one.
+      IPv6AcceptRA = true;
+    };
+    # Many PPPoE ISPs never send an RA with the M flag; solicit regardless.
+    dhcpV6Config.WithoutRA = "solicit";
+    linkConfig.RequiredForOnline = "no";
+  };
+
+  boot.kernel.sysctl."net.ipv6.conf.all.forwarding" = lib.mkDefault 1;
+}

modules/clan/router/network.nix

@@ -0,0 +1,170 @@
+# L2/L3 of the gateway: PPPoE WAN port (optionally on an ISP VLAN), a
+# VLAN-filtering bridge over the LAN ports, and one L3 interface per VLAN.
+{ settings }:
+{ lib, pkgs, ... }:
+let
+  cfg = settings;
+
+  vlanIf = name: "vlan-${name}";
+in
+{
+  assertions = [
+    {
+      assertion = cfg.vlans ? mgmt && cfg.vlans ? lan;
+      message = "router: every site must define the `mgmt` and `lan` VLANs.";
+    }
+    {
+      assertion = lib.all (v: cfg.vlans ? ${v}) (lib.attrValues cfg.accessPorts);
+      message = "router: every accessPorts value must name a defined VLAN.";
+    }
+    {
+      assertion = lib.all (p: !(cfg.accessPorts ? ${p})) cfg.trunkPorts;
+      message = "router: a port cannot be both a trunk and an access port.";
+    }
+    {
+      assertion =
+        cfg.stagingPort == null
+        || !(
+          cfg.stagingPort == cfg.wan.interface
+          || lib.elem cfg.stagingPort cfg.trunkPorts
+          || cfg.accessPorts ? ${cfg.stagingPort}
+        );
+      message = "router: stagingPort ${toString cfg.stagingPort} is also the WAN, a trunk or an access port.";
+    }
+  ];
+
+  # Router diagnostics toolkit: packets (tcpdump), path (mtr), link
+  # negotiation (ethtool), NAT state (conntrack), DNS (kdig), per-flow
+  # bandwidth (iftop), WAN throughput (librespeed-cli; iperf3 covers LAN).
+  environment.systemPackages = with pkgs; [
+    tcpdump
+    mtr
+    ethtool
+    conntrack-tools
+    knot-dns
+    iftop
+    librespeed-cli
+  ];
+
+  networking.useNetworkd = true;
+  networking.useDHCP = false;
+  systemd.network.enable = true;
+
+  systemd.network.netdevs = {
+    "20-br0" = {
+      netdevConfig = {
+        Name = "br0";
+        Kind = "bridge";
+      };
+      bridgeConfig.VLANFiltering = true;
+    };
+  }
+  // lib.optionalAttrs (cfg.wan.vlanId != null) {
+    "15-wan-vlan" = {
+      netdevConfig = {
+        Name = "wan-vlan";
+        Kind = "vlan";
+      };
+      vlanConfig.Id = cfg.wan.vlanId;
+    };
+  }
+  // lib.mapAttrs' (
+    name: vlan:
+    lib.nameValuePair "30-${vlanIf name}" {
+      netdevConfig = {
+        Name = vlanIf name;
+        Kind = "vlan";
+      };
+      vlanConfig.Id = vlan.id;
+    }
+  ) cfg.vlans;
+
+  systemd.network.networks =
+    let
+      taggedAll = lib.mapAttrsToList (_: vlan: { VLAN = vlan.id; }) cfg.vlans;
+    in
+    lib.optionalAttrs (cfg.stagingPort != null) {
+      # Staging uplink (see interface.nix): plain DHCPv4 client on a spare
+      # port, no bridge/VLAN membership, so the firewall treats it as untrusted.
+      "05-staging" = {
+        matchConfig.Name = cfg.stagingPort;
+        networkConfig.DHCP = "ipv4";
+      };
+    }
+    // {
+      # WAN port carries only the PPPoE session; no IP config of its own.
+      "10-wan" = {
+        matchConfig.Name = cfg.wan.interface;
+        networkConfig.LinkLocalAddressing = "no";
+        vlan = lib.optional (cfg.wan.vlanId != null) "wan-vlan";
+        linkConfig = {
+          RequiredForOnline = "carrier";
+        }
+        # The wan-vlan subinterface (and thus the PPPoE session) inherits
+        # the parent port's MAC, so spoofing here covers both cases.
+        // lib.optionalAttrs (cfg.wan.macAddress != null) {
+          MACAddress = cfg.wan.macAddress;
+        };
+      };
+    }
+    // lib.optionalAttrs (cfg.wan.vlanId != null) {
+      # The ISP-side VLAN subinterface pppd dials on (e.g. AIS tags PPPoE).
+      "15-wan-vlan" = {
+        matchConfig.Name = "wan-vlan";
+        networkConfig.LinkLocalAddressing = "no";
+        linkConfig.RequiredForOnline = "no";
+      };
+    }
+    // {
+      # The bridge itself is L2-only; L3 lives on the vlan-* interfaces,
+      # which hang off the bridge (tagged on the bridge "self" port).
+      "20-br0" = {
+        matchConfig.Name = "br0";
+        networkConfig.LinkLocalAddressing = "no";
+        vlan = lib.mapAttrsToList (name: _: vlanIf name) cfg.vlans;
+        bridgeVLANs = taggedAll;
+        linkConfig.RequiredForOnline = "no";
+      };
+    }
+    // lib.listToAttrs (
+      map (port: {
+        name = "25-trunk-${port}";
+        value = {
+          matchConfig.Name = port;
+          networkConfig.Bridge = "br0";
+          bridgeVLANs = taggedAll;
+          linkConfig.RequiredForOnline = "no";
+        };
+      }) cfg.trunkPorts
+    )
+    // lib.mapAttrs' (
+      port: vlanName:
+      lib.nameValuePair "25-access-${port}" {
+        matchConfig.Name = port;
+        networkConfig.Bridge = "br0";
+        bridgeVLANs = [
+          {
+            VLAN = cfg.vlans.${vlanName}.id;
+            PVID = cfg.vlans.${vlanName}.id;
+            EgressUntagged = cfg.vlans.${vlanName}.id;
+          }
+        ];
+        linkConfig.RequiredForOnline = "no";
+      }
+    ) cfg.accessPorts
+    // lib.mapAttrs' (
+      name: vlan:
+      lib.nameValuePair "40-${vlanIf name}" {
+        matchConfig.Name = vlanIf name;
+        address = [ "${vlan.address}/${toString vlan.prefixLength}" ];
+        networkConfig = {
+          IPv6AcceptRA = false;
+          # Announce a /64 carved from the DHCPv6-PD prefix on ppp0 (SLAAC).
+          IPv6SendRA = true;
+          DHCPPrefixDelegation = true;
+        };
+        dhcpPrefixDelegationConfig.SubnetId = "auto";
+        linkConfig.RequiredForOnline = "no";
+      }
+    ) cfg.vlans;
+}

modules/clan/router/omada.nix

@@ -0,0 +1,36 @@
+# TP-Link Omada SDN controller for sites with Omada APs/switches. There is no
+# nixpkgs package, so it runs as a podman container (mbentley/omada-controller,
+# the de-facto standard image). Host networking because device adoption relies
+# on L2 broadcast discovery (UDP 29810) on the mgmt VLAN; the default-deny
+# input firewall keeps its ports unreachable from WAN and non-mgmt VLANs.
+{ settings }:
+{ config, lib, ... }:
+let
+  cfg = settings;
+in
+{
+  config = lib.mkIf cfg.omada.enable {
+    virtualisation.podman.enable = true;
+    virtualisation.oci-containers = {
+      backend = "podman";
+      containers.omada = {
+        image = "docker.io/mbentley/omada-controller:5.15";
+        extraOptions = [ "--network=host" ];
+        environment.TZ = config.time.timeZone;
+        volumes = [
+          "/var/lib/omada/data:/opt/tplink/EAPController/data"
+          "/var/lib/omada/logs:/opt/tplink/EAPController/logs"
+        ];
+      };
+    };
+
+    # Admin UI (8043) also reachable over the mesh, like Grafana on control.
+    networking.firewall.extraInputRules = ''
+      ip6 saddr ${cfg.mesh.subnet} tcp dport 8043 accept comment "omada ui over the mesh"
+    '';
+
+    # Controller state (adopted devices, site config, cert) — declared as clan
+    # state so a borgbackup client can pick it up; backup wiring is a later step.
+    clan.core.state.omada.folders = [ "/var/lib/omada" ];
+  };
+}

modules/clan/router/pppoe.nix

@@ -0,0 +1,67 @@
+# PPPoE WAN session. ISP credentials are entered once at `clan vars generate`
+# (prompts). Both are secret — AIS often uses the same string for username and
+# password — so neither may land in the Nix store: pppd reads the username from
+# an included secret options file and the password from chap/pap-secrets.
+{ settings }:
+{ config, ... }:
+let
+  cfg = settings;
+  creds = config.clan.core.vars.generators.pppoe-credentials;
+  # Interface pppd dials on: the WAN port itself, or its ISP VLAN (network.nix).
+  pppInterface = if cfg.wan.vlanId == null then cfg.wan.interface else "wan-vlan";
+in
+{
+  clan.core.vars.generators.pppoe-credentials = {
+    prompts.username = {
+      description = "PPPoE username (from the ISP)";
+      type = "hidden";
+    };
+    prompts.password = {
+      description = "PPPoE password (from the ISP)";
+      type = "hidden";
+    };
+    files."user-opts".secret = true;
+    files."chap-secrets".secret = true;
+    script = ''
+      user="$(cat "$prompts"/username)"
+      pass="$(cat "$prompts"/password)"
+      printf 'user "%s"\n' "$user" > "$out"/user-opts
+      printf '"%s" * "%s"\n' "$user" "$pass" > "$out"/chap-secrets
+    '';
+  };
+
+  # defaultroute-metric 0: pppd refuses `defaultroute` while any other
+  # default route exists (e.g. the staging uplink's DHCP route, metric 1024,
+  # network.nix) unless given a metric; with 0 it only checks for a metric-0
+  # route, installs its own as the preferred exit, and removes it again on
+  # hangup so the staging route takes over.
+  services.pppd = {
+    enable = true;
+    peers.wan = {
+      autostart = true;
+      config = ''
+        plugin pppoe.so ${pppInterface}
+        ifname ppp0
+        file ${creds.files."user-opts".path}
+        noipdefault
+        defaultroute
+        defaultroute-metric 0
+        noauth
+        hide-password
+        persist
+        maxfail 0
+        holdoff 5
+        lcp-echo-interval 15
+        lcp-echo-failure 3
+        +ipv6
+        mtu 1492
+        mru 1492
+      '';
+    };
+  };
+
+  # pppd looks up the password for `user` in these files at dial time; both
+  # point at the same generated `"<user>" * "<pass>"` line (PAP and CHAP).
+  environment.etc."ppp/chap-secrets".source = creds.files."chap-secrets".path;
+  environment.etc."ppp/pap-secrets".source = creds.files."chap-secrets".path;
+}

modules/clan/router/proxy.nix

@@ -0,0 +1,104 @@
+# Internal reverse proxy for the gateway: Caddy terminates TLS for
+# <service>.<site><siteId>.<proxy.domain> (e.g. omada.cnx1.cnx.network) and
+# forwards to backends by their internal address. The cert is a real Let's
+# Encrypt wildcard (*.<site><siteId>.<domain>) obtained via ACME DNS-01
+# against proxy.acme.nameserver with a gateway-scoped TSIG key, so browsers
+# trust it without any CA install; the names only *resolve* internally —
+# Blocky answers them with the router's LAN address, the public zone never
+# carries them.
+#
+# The TSIG secret is the shared dns-acme-<hostname>-secret generator
+# (acme-secret.nix, declared here via default.nix); the nameserver machine
+# must declare the same generator and load the key (this fleet: ns1).
+{ settings }:
+{ config, lib, ... }:
+let
+  cfg = settings;
+  hostname = config.networking.hostName;
+  tsigKey = "acme_${lib.replaceStrings [ "-" ] [ "_" ] hostname}";
+  certName = "${cfg.site}${toString cfg.siteId}.${cfg.proxy.domain}";
+  nameserver = "${cfg.proxy.acme.nameserver}:53";
+in
+{
+  config = lib.mkIf cfg.proxy.enable {
+    assertions = [
+      {
+        assertion = lib.all (v: cfg.vlans ? ${v}) cfg.proxy.allowVlans;
+        message = "router: proxy.allowVlans must name VLANs defined in vlans.";
+      }
+    ];
+
+    # Render the shared per-gateway TSIG secret into a lego rfc2136 env file,
+    # scoped on the nameserver to _acme-challenge.<site><siteId> TXT records.
+    clan.core.vars.generators."dns-acme-${hostname}-rfc2136" = {
+      files."rfc2136.env".secret = true; # root-owned; systemd reads it as root
+      dependencies = [ "dns-acme-${hostname}-secret" ];
+      script = ''
+        printf 'RFC2136_NAMESERVER=${nameserver}\nRFC2136_TSIG_ALGORITHM=hmac-sha256.\nRFC2136_TSIG_KEY=${tsigKey}\nRFC2136_TSIG_SECRET=%s\n' \
+          "$(cat "$in"/dns-acme-${hostname}-secret/secret)" > "$out"/rfc2136.env
+      '';
+    };
+
+    security.acme = {
+      acceptTerms = true;
+      defaults.email = cfg.proxy.acme.email;
+      # One wildcard for every proxied service; DNS-01, so issuance works
+      # behind PPPoE with no inbound reachability at all.
+      certs.${certName} = {
+        domain = "*.${certName}";
+        dnsProvider = "rfc2136";
+        environmentFile =
+          config.clan.core.vars.generators."dns-acme-${hostname}-rfc2136".files."rfc2136.env".path;
+        # Only that nameserver accepts this key's UPDATE; check propagation
+        # against it directly rather than a public resolver.
+        dnsResolver = nameserver;
+        # Caddy reads the cert from explicit file paths (tls directive below),
+        # so it won't notice a renewal on its own.
+        reloadServices = [ "caddy.service" ];
+      };
+    };
+
+    # The lego-issued cert is owned group=acme; Caddy needs to read the key.
+    users.users.caddy.extraGroups = [ "acme" ];
+
+    # The explicit `tls cert key` points Caddy at the wildcard cert and disables
+    # its automatic ACME, so no extra issuance happens.
+    services.caddy = {
+      enable = true;
+      virtualHosts = lib.mapAttrs' (
+        name: svc:
+        lib.nameValuePair "${name}.${certName}" {
+          extraConfig = ''
+            tls /var/lib/acme/${certName}/cert.pem /var/lib/acme/${certName}/key.pem
+            ${
+              if svc.insecureSkipVerify then
+                ''
+                  reverse_proxy ${svc.backend} {
+                    transport http {
+                      tls_insecure_skip_verify
+                    }
+                  }''
+              else
+                "reverse_proxy ${svc.backend}"
+            }
+          '';
+        }
+      ) cfg.proxy.services;
+    };
+
+    # Blocky answers <anything>.<site><siteId>.cnx.network (customDNS covers
+    # subdomains) with the router's LAN address — clients on any allowed VLAN
+    # reach that address through the router's input path.
+    services.blocky.settings.customDNS.mapping.${certName} = cfg.vlans.lan.address;
+
+    # 443 serves the proxy; 80 only carries Caddy's automatic HTTP->HTTPS
+    # redirect. mgmt is already a trusted interface; listed anyway so shrinking
+    # trustedInterfaces later doesn't silently break the proxy.
+    networking.firewall.interfaces = lib.genAttrs (map (v: "vlan-${v}") cfg.proxy.allowVlans) (_: {
+      allowedTCPPorts = [
+        80
+        443
+      ];
+    });
+  };
+}

modules/clan/router/speedtest.nix

@@ -0,0 +1,55 @@
+# Periodic WAN speed test so ISP degradation shows up as a trend instead of a
+# complaint. A timer runs librespeed-cli and writes the results as Prometheus
+# metrics into node_exporter's textfile collector — they ride the existing
+# 9100 scrape to VictoriaMetrics, where alerts.nix compares each run against
+# the link's own 7-day median (no per-site threshold to maintain).
+{ settings }:
+{ pkgs, ... }:
+let
+  cfg = settings;
+  textfileDir = "/var/lib/speedtest";
+in
+{
+  services.prometheus.exporters.node.extraFlags = [
+    "--collector.textfile.directory=${textfileDir}"
+  ];
+
+  systemd.services.speedtest = {
+    description = "WAN speed test to Prometheus textfile metrics";
+    after = [ "network-online.target" ];
+    wants = [ "network-online.target" ];
+    path = [
+      pkgs.librespeed-cli
+      pkgs.jq
+    ];
+    serviceConfig = {
+      Type = "oneshot";
+      StateDirectory = "speedtest";
+      # One test at boot would race PPPoE and log a spurious failure.
+      ExecCondition = "${pkgs.iproute2}/bin/ip link show ppp0";
+    };
+    script = ''
+      tmp="${textfileDir}/.speedtest.prom.tmp"
+      if result=$(librespeed-cli --json); then
+        jq -r '.[0]
+          | "speedtest_download_mbps \(.download)",
+            "speedtest_upload_mbps \(.upload)",
+            "speedtest_ping_ms \(.ping)",
+            "speedtest_jitter_ms \(.jitter)",
+            "speedtest_success 1"' <<<"$result" > "$tmp"
+      else
+        echo "speedtest_success 0" > "$tmp"
+      fi
+      mv "$tmp" "${textfileDir}/speedtest.prom"
+    '';
+  };
+
+  systemd.timers.speedtest = {
+    wantedBy = [ "timers.target" ];
+    timerConfig = {
+      OnCalendar = cfg.speedtest.interval;
+      RandomizedDelaySec = "10m";
+      Persistent = true;
+    };
+  };
+}

modules/clan/router/tests/vm/default.nix

@@ -0,0 +1,359 @@
+# End-to-end VM test of the router service: a PPPoE access concentrator plays
+# the ISP on the WAN port, a trunk carries tagged lan/iot VLANs to `client`,
+# an untagged access port carries mgmt to `admin`, and `oldlan` is the DHCP
+# network the box is staged in before cutover.
+#
+#   isp ---(vlan 1: PPPoE)--- wan [gw] trunk ---(vlan 2: tagged 20/40)--- client
+#                                    access --(vlan 3: untagged mgmt)--- admin
+#                                    staging -(vlan 4: DHCP client)--- oldlan
+#
+# What is proven: PPPoE dial-in with the vars-provided credentials, bridge
+# VLAN tagging/untagging, Kea leases and reservations per VLAN, Blocky
+# answering on the VLAN with the blocklist active, NAT to the WAN, the
+# firewall trust model (allowWan, mgmt-only SSH, no inter-VLAN forwarding),
+# and the staging uplink as NATed fallback exit behind ppp0.
+{ pkgs, lib, ... }:
+let
+  # The vars mock answers every prompt with "mock-prompt-value-<name>"; the
+  # ISP side must accept exactly those.
+  chapSecrets = ''"mock-prompt-value-username" * "mock-prompt-value-password" *'';
+  ispAddress = "192.0.2.1";
+
+  clientMac = "02:00:00:00:00:20";
+  clientAddress = "10.9.20.50";
+  adminMac = "02:00:00:00:00:10";
+  adminAddress = "10.9.10.50";
+  oldlanAddress = "192.168.88.1";
+  # Only reachable through oldlan's router role, i.e. via gw's staging
+  # default route (metric 1024); ppp0's metric-0 default must win while up.
+  beyondStaging = "203.0.113.1";
+in
+{
+  name = "router";
+
+  clan = {
+    directory = ./.;
+    # Bridges, VLAN netdevs, PPPoE and nftables need a real kernel.
+    test.useContainers = false;
+    inventory = {
+      # Every node is a clan machine (the test framework's defaults require
+      # it); only gw gets the router role.
+      machines = {
+        gw = { };
+        isp = { };
+        client = { };
+        admin = { };
+        oldlan = { };
+      };
+
+      instances.router = {
+        module.name = "router";
+        module.input = "self";
+        roles.default.machines.gw.settings = {
+          site = "tst";
+          siteId = 9;
+          mesh.subnet = "fd00:7e57:c1a1:c0de::/64";
+          wan.interface = "wan";
+          trunkPorts = [ "trunk" ];
+          accessPorts.access = "mgmt";
+          stagingPort = "staging";
+          vlans = {
+            mgmt = {
+              id = 10;
+              dhcp.reservations.admin = {
+                hwAddress = adminMac;
+                ipAddress = adminAddress;
+              };
+            };
+            lan = {
+              id = 20;
+              dhcp.reservations.client = {
+                hwAddress = clientMac;
+                ipAddress = clientAddress;
+              };
+            };
+            iot = {
+              id = 40;
+              allowWan = false;
+            };
+          };
+          # Access point on the simulated radio wlan0 (mac80211_hwsim, address
+          # 02:00:00:00:00:00): two SSIDs, one per VLAN.
+          wifi = {
+            enable = true;
+            countryCode = "US";
+            networks = {
+              home.vlan = "lan";
+              things = {
+                vlan = "iot";
+                security = "wpa2";
+                hidden = true;
+                isolateClients = true;
+              };
+            };
+            radios.wlan0 = {
+              channel = 6;
+              macAddress = "02:00:00:00:00:00";
+              networks = [
+                "home"
+                "things"
+              ];
+            };
+          };
+        };
+      };
+    };
+  };
+
+  nodes = {
+    gw = {
+      virtualisation.interfaces = {
+        wan = {
+          vlan = 1;
+          assignIP = false;
+        };
+        trunk = {
+          vlan = 2;
+          assignIP = false;
+        };
+        access = {
+          vlan = 3;
+          assignIP = false;
+        };
+        staging = {
+          vlan = 4;
+          assignIP = false;
+        };
+      };
+
+      # Something must listen on 22 for the mgmt-only SSH rule to be observable
+      # (a refused and a dropped connection look the same to the client).
+      services.openssh.enable = true;
+
+      # The sandbox has no internet: serve the blocklist from a local file
+      # instead of GitHub. (CrowdSec, whose hub sync needs the network too,
+      # is opt-in and stays off.)
+      services.blocky.settings.blocking.denylists.ads = lib.mkForce [
+        (toString (pkgs.writeText "ads.hosts" "0.0.0.0 ads.example.com\n"))
+      ];
+
+      # Two simulated radios: wlan0 is the AP (settings above), wlan1 plays a
+      # wireless client. It lives in its own network namespace, like the
+      # separate host it stands in for — otherwise its lease would add a
+      # second 10.9.20.0/24 route to the router's own table. Its DHCP lease
+      # must come from Kea on the SSID's VLAN: wlan1 -> air -> wlan0 -> br0 ->
+      # vlan-lan. The mock passphrase is what the vars mock fed into the
+      # wifi-home-passphrase generator.
+      boot.kernelModules = [ "mac80211_hwsim" ];
+      systemd.services.wifi-station = {
+        wantedBy = [ "multi-user.target" ];
+        # No BindsTo: the device unit vanishes once wlan1 moves into the netns.
+        after = [ "sys-subsystem-net-devices-wlan1.device" ];
+        path = [
+          pkgs.iproute2
+          pkgs.iw
+        ];
+        preStart = ''
+          ip netns add sta
+          iw phy phy1 set netns name sta
+          ip netns exec sta ip link set lo up
+          mkdir -p /run/wpa_supplicant/client # nixpkgs' wpa_cli keeps its sockets here
+        '';
+        serviceConfig.ExecStart =
+          "${pkgs.iproute2}/bin/ip netns exec sta ${pkgs.wpa_supplicant}/bin/wpa_supplicant -i wlan1 -c "
+          + pkgs.writeText "sta.conf" ''
+            ctrl_interface=/run/wpa_supplicant/control
+            network={
+              ssid="home"
+              key_mgmt=SAE
+              sae_password="mock-prompt-value-passphrase"
+              ieee80211w=2
+              scan_freq=2437
+            }
+          '';
+      };
+      environment.systemPackages = [
+        pkgs.wpa_supplicant
+        (pkgs.writeShellScriptBin "sta-dhcp" ''
+          # One DHCP round on the station, applying the offered address.
+          exec ${pkgs.iproute2}/bin/ip netns exec sta ${pkgs.busybox}/bin/udhcpc -i wlan1 -n -q -f \
+            -s ${pkgs.writeShellScript "udhcpc-apply" ''
+              case "$1" in
+                bound|renew) ${pkgs.iproute2}/bin/ip addr replace "$ip/$mask" dev "$interface" ;;
+              esac
+            ''}
+        '')
+      ];
+    };
+
+    isp = {
+      virtualisation.interfaces.wan = {
+        vlan = 1;
+        assignIP = false;
+      };
+      # PPPoE access concentrator: one session, peer gets 192.0.2.10.
+      systemd.services.pppoe-server = {
+        wantedBy = [ "multi-user.target" ];
+        after = [ "network.target" ];
+        serviceConfig.ExecStart =
+          "${pkgs.rp-pppoe}/sbin/pppoe-server -F -O /etc/ppp/pppoe-server-options"
+          + " -q ${pkgs.ppp}/sbin/pppd -I wan -L ${ispAddress} -R 192.0.2.10";
+      };
+      environment.etc = {
+        "ppp/pppoe-server-options".text = ''
+          plugin pppoe.so
+          require-chap
+          lcp-echo-interval 10
+          lcp-echo-failure 2
+          nobsdcomp
+          noccp
+          novj
+        '';
+        "ppp/chap-secrets" = {
+          text = chapSecrets;
+          mode = "0640";
+        };
+      };
+    };
+
+    client = {
+      virtualisation.interfaces.trunk = {
+        vlan = 2;
+        assignIP = false;
+      };
+      networking.useDHCP = false;
+      networking.vlans = {
+        lan0 = {
+          id = 20;
+          interface = "trunk";
+        };
+        iot0 = {
+          id = 40;
+          interface = "trunk";
+        };
+      };
+      networking.interfaces.lan0 = {
+        useDHCP = true;
+        macAddress = clientMac;
+      };
+      networking.interfaces.iot0.useDHCP = true;
+      # Dual-homed on purpose (lan + iot). Both leases bring a default route:
+      # prefer lan0 so replies to other VLANs and the WAN leave where the
+      # router expects them (its rp-filter would drop them on vlan-iot), and
+      # loosen this client's own rp-filter so a WAN reply on iot0 would be
+      # seen — the negative allowWan check must fail at the router, not here.
+      systemd.network.networks."40-iot0".dhcpV4Config.RouteMetric = 2048;
+      networking.firewall.checkReversePath = "loose";
+      environment.systemPackages = [
+        pkgs.dnsutils
+        pkgs.netcat
+      ];
+    };
+
+    admin = {
+      virtualisation.interfaces.access = {
+        vlan = 3;
+        assignIP = false;
+      };
+      networking.useDHCP = false;
+      networking.interfaces.access = {
+        useDHCP = true;
+        macAddress = adminMac;
+      };
+      environment.systemPackages = [ pkgs.netcat ];
+    };
+
+    # The LAN the box is staged in: a DHCP server handing gw its uplink
+    # lease, plus an address that is only reachable via that uplink's
+    # default route. No route back to 10.9.0.0/16: replies only reach the
+    # clients if gw masquerades them.
+    oldlan = {
+      virtualisation.interfaces.staging = {
+        vlan = 4;
+        assignIP = false;
+      };
+      networking.useDHCP = false;
+      networking.useNetworkd = true;
+      systemd.network.networks."10-staging" = {
+        matchConfig.Name = "staging";
+        address = [
+          "${oldlanAddress}/24"
+          "${beyondStaging}/32"
+        ];
+        networkConfig.DHCPServer = true;
+        dhcpServerConfig = {
+          PoolOffset = 100;
+          PoolSize = 50;
+        };
+      };
+      networking.firewall.allowedUDPPorts = [ 67 ];
+    };
+  };
+
+  testScript = ''
+    start_all()
+
+    with subtest("PPPoE session comes up with the vars credentials"):
+        gw.wait_for_unit("pppd-wan.service")
+        gw.wait_until_succeeds("ping -c1 -W1 ${ispAddress}")
+
+    with subtest("DHCP hands out reserved leases per VLAN"):
+        gw.wait_for_unit("kea-dhcp4-server.service")
+        client.wait_until_succeeds("ip -4 addr show lan0 | grep -q 'inet ${clientAddress}/24'")
+        client.wait_until_succeeds("ip -4 addr show iot0 | grep -q 'inet 10.9.40.1[0-9][0-9]/24'")
+        admin.wait_until_succeeds("ip -4 addr show access | grep -q 'inet ${adminAddress}/24'")
+
+    with subtest("Blocky serves the VLAN and blocks the denylist"):
+        gw.wait_for_unit("blocky.service")
+        answer = client.wait_until_succeeds("dig +short +time=2 @10.9.20.1 ads.example.com")
+        assert answer.strip() == "0.0.0.0", f"expected blocked answer, got: {answer!r}"
+
+    with subtest("NAT to the WAN only for VLANs with allowWan"):
+        client.wait_until_succeeds("ping -c1 -W1 -I lan0 ${ispAddress}")
+        # iot has a route to the WAN; the router is what refuses to forward.
+        client.succeed("ip route show dev iot0 | grep -q ^default")
+        client.fail("ping -c1 -W2 -I iot0 ${ispAddress}")
+
+    with subtest("mgmt reaches other VLANs, other VLANs do not"):
+        admin.succeed("ping -c1 -W2 ${clientAddress}")
+        client.fail("ping -c1 -W2 -I lan0 ${adminAddress}")
+
+    with subtest("SSH on the router only from mgmt"):
+        gw.wait_for_open_port(22)
+        admin.succeed("nc -z -w2 10.9.10.1 22")
+        client.fail("nc -z -w2 10.9.20.1 22")
+
+    with subtest("Wireless SSIDs are bridge access ports of their VLAN"):
+        gw.wait_for_unit("hostapd.service")
+        # Second SSID: derived BSSID, hidden, bridged into the iot VLAN.
+        gw.wait_until_succeeds("ip link show wlan0-1 | grep -q '06:00:00:00:00:00'")
+        gw.wait_until_succeeds("bridge vlan show dev wlan0-1 | grep -q '40 PVID Egress Untagged'")
+        gw.succeed("hostapd_cli -i wlan0-1 get_config | grep -q '^ssid=things'")
+        # The second BSS section of the rendered config carries hidden + isolation.
+        things = gw.succeed("sed -n '/^bss=wlan0-1/,$p' /run/hostapd/wlan0.hostapd.conf")
+        assert "ignore_broadcast_ssid=1" in things and "ap_isolate=1" in things, things
+        assert "sae_password=mock-prompt-value-passphrase" not in things, "wpa2 SSID must not carry SAE entries"
+        # A WPA3 (SAE) station on the first SSID authenticates with the vars
+        # passphrase and gets its lease from Kea on the lan VLAN, through the bridge.
+        gw.wait_for_unit("wifi-station.service")
+        gw.wait_until_succeeds("ip netns exec sta wpa_cli -i wlan1 status | grep -q wpa_state=COMPLETED")
+        gw.succeed("timeout 60 sta-dhcp")
+        gw.succeed("ip netns exec sta ip -4 addr show wlan1 | grep -q 'inet 10.9.20.1[0-9][0-9]/24'")
+
+    with subtest("Staging uplink: NATed exit for allowWan VLANs, behind ppp0 while it is up"):
+        gw.wait_until_succeeds("ip -4 route show default dev staging | grep -q 'via ${oldlanAddress}'")
+        # pppd installs its default route despite the DHCP one (defaultroute-metric 0).
+        gw.succeed("ip route get ${beyondStaging} | grep -q 'dev ppp0'")
+        # On-link old-LAN hosts are reached through the staging port regardless.
+        client.succeed("ping -c1 -W2 -I lan0 ${oldlanAddress}")
+        client.fail("ping -c1 -W2 -I iot0 ${oldlanAddress}")
+        # ppp0 down: the staging route carries the WAN traffic, allowWan still holds.
+        gw.systemctl("stop pppd-wan.service")
+        gw.wait_until_succeeds("ip route get ${beyondStaging} | grep -q 'dev staging'")
+        client.succeed("ping -c1 -W2 -I lan0 ${beyondStaging}")
+        client.fail("ping -c1 -W2 -I iot0 ${beyondStaging}")
+        # ppp0 back: preferred again.
+        gw.systemctl("start pppd-wan.service")
+        gw.wait_until_succeeds("ip route get ${beyondStaging} | grep -q 'dev ppp0'")
+  '';
+}

modules/clan/router/wifi.nix

@@ -0,0 +1,207 @@
+# Wireless access point on the router itself (hostapd). Every SSID is a BSS
+# interface that joins the VLAN bridge as an untagged access port of its VLAN,
+# so wireless clients get exactly the same DHCP/DNS/firewall treatment as a
+# wired port in that VLAN. Passphrases are vars prompts, never in the store.
+#
+# hostapd names BSS interfaces <radio>, <radio>-1, <radio>-2 ... and wants a
+# fixed BSSID for every extra one; they are derived from the radio's hardware
+# address by setting the locally-administered bit and flipping bits 2-3 of
+# the first octet per index (02 -> 06, 0a, 0e), which never collides with the
+# radio's own address.
+{ settings }:
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = settings;
+  wifi = cfg.wifi;
+
+  bssIf = radio: i: if i == 0 then radio else "${radio}-${toString i}";
+
+  hexDigit =
+    c:
+    {
+      "0" = 0;
+      "1" = 1;
+      "2" = 2;
+      "3" = 3;
+      "4" = 4;
+      "5" = 5;
+      "6" = 6;
+      "7" = 7;
+      "8" = 8;
+      "9" = 9;
+      a = 10;
+      b = 11;
+      c = 12;
+      d = 13;
+      e = 14;
+      f = 15;
+    }
+    .${lib.toLower c};
+  octetToInt = s: 16 * hexDigit (builtins.substring 0 1 s) + hexDigit (builtins.substring 1 1 s);
+  intToOctet = n: lib.toLower (lib.fixedWidthString 2 "0" (lib.toHexString n));
+  deriveBssid =
+    mac: i:
+    let
+      octets = lib.splitString ":" mac;
+      first = builtins.bitXor (builtins.bitOr (octetToInt (builtins.head octets)) 2) (i * 4);
+    in
+    lib.concatStringsSep ":" ([ (intToOctet first) ] ++ builtins.tail octets);
+
+  # Every BSS of every radio, flattened.
+  bsses = lib.concatLists (
+    lib.mapAttrsToList (
+      radio: r:
+      lib.imap0 (i: name: {
+        inherit radio name;
+        index = i;
+        iface = bssIf radio i;
+        net = wifi.networks.${name} or null;
+      }) r.networks
+    ) wifi.radios
+  );
+
+  referenced = lib.unique (map (b: b.name) bsses);
+  secured = lib.filter (name: wifi.networks.${name}.security != "open") (
+    lib.filter (name: wifi.networks ? ${name}) referenced
+  );
+  passphraseFile =
+    name: config.clan.core.vars.generators."wifi-${name}-passphrase".files.passphrase.path;
+
+  authentication =
+    name: net:
+    {
+      wpa3 = {
+        mode = "wpa3-sae";
+        saePasswordsFile = passphraseFile name;
+      };
+      wpa3-transition = {
+        mode = "wpa3-sae-transition";
+        saePasswordsFile = passphraseFile name;
+        wpaPasswordFile = passphraseFile name;
+      };
+      wpa2 = {
+        mode = "wpa2-sha1";
+        wpaPasswordFile = passphraseFile name;
+      };
+      open.mode = "none";
+    }
+    .${net.security};
+in
+{
+  config = lib.mkIf wifi.enable {
+    assertions = [
+      {
+        assertion = wifi.radios != { };
+        message = "router: wifi.enable needs at least one radio in wifi.radios.";
+      }
+      {
+        assertion = wifi.countryCode != null;
+        message = "router: wifi.countryCode is required when wifi.enable is set (regulatory domain).";
+      }
+      {
+        assertion = lib.all (b: b.net != null) bsses;
+        message = "router: every wifi.radios.<radio>.networks entry must name a network in wifi.networks.";
+      }
+      {
+        assertion = lib.all (b: b.net == null || cfg.vlans ? ${b.net.vlan}) bsses;
+        message = "router: every wifi.networks.<name>.vlan must name a VLAN in vlans.";
+      }
+      {
+        assertion = lib.all (r: lib.length r.networks <= 4) (lib.attrValues wifi.radios);
+        message = "router: a radio can broadcast at most four networks.";
+      }
+      {
+        assertion = lib.all (r: lib.length r.networks <= 1 || r.macAddress != null) (
+          lib.attrValues wifi.radios
+        );
+        message = "router: wifi.radios.<radio>.macAddress is required for radios broadcasting more than one network.";
+      }
+      {
+        assertion =
+          !config.networking.wireless.enable
+          && !config.networking.wireless.iwd.enable
+          && !config.networking.networkmanager.enable;
+        message = "router: wifi.enable needs the radios for hostapd; disable networking.wireless (wpa_supplicant), iwd and NetworkManager.";
+      }
+    ];
+
+    # Regulatory database for the kernel, so countryCode actually applies.
+    hardware.wirelessRegulatoryDatabase = true;
+
+    clan.core.vars.generators = lib.genAttrs (map (name: "wifi-${name}-passphrase") secured) (
+      gen:
+      let
+        name = lib.removeSuffix "-passphrase" (lib.removePrefix "wifi-" gen);
+      in
+      {
+        prompts.passphrase = {
+          description = "Wi-Fi passphrase for SSID \"${wifi.networks.${name}.ssid}\" (8-63 ASCII characters)";
+          type = "hidden";
+        };
+        files.passphrase.secret = true;
+        # No trailing newline: hostapd turns every line of the file into a
+        # sae_password entry, and an empty one wipes the list.
+        script = ''printf '%s' "$(cat "$prompts"/passphrase)" > "$out"/passphrase'';
+      }
+    );
+
+    services.hostapd = {
+      enable = true;
+      radios = lib.mapAttrs (radio: r: {
+        inherit (r) band channel;
+        inherit (wifi) countryCode;
+        wifi6.enable = r.wifi6;
+        networks = lib.listToAttrs (
+          map (
+            b:
+            lib.nameValuePair b.iface (
+              {
+                inherit (b.net) ssid;
+                ignoreBroadcastSsid = if b.net.hidden then "empty" else "disabled";
+                apIsolate = b.net.isolateClients;
+                authentication = authentication b.name b.net;
+              }
+              // lib.optionalAttrs (lib.length r.networks > 1) {
+                bssid = if b.index == 0 then r.macAddress else deriveBssid r.macAddress b.index;
+              }
+            )
+          ) (lib.filter (b: b.radio == radio) bsses)
+        );
+      }) wifi.radios;
+    };
+
+    # Each BSS is an untagged access port of its VLAN on br0 (cf. accessPorts
+    # in network.nix). The kernel refuses to bridge a wireless interface in
+    # station mode (IFF_DONT_BRIDGE), and a failed enslave parks the link in
+    # networkd's `failed` state for good: networkd only re-evaluates a link
+    # when its matching .network file changes. So match on the AP interface
+    # type as well as the name: the radio's initial station-mode netdev
+    # matches nothing (unmanaged), and once hostapd switches it to AP and
+    # brings the carrier up, networkd matches this file for the first time
+    # and enslaves it. Extra BSSes are created by hostapd in AP mode already.
+    systemd.network.networks = lib.listToAttrs (
+      map (
+        b:
+        lib.nameValuePair "27-wifi-${b.iface}" {
+          matchConfig = {
+            Name = b.iface;
+            WLANInterfaceType = "ap";
+          };
+          networkConfig.Bridge = "br0";
+          bridgeVLANs = [
+            {
+              VLAN = cfg.vlans.${b.net.vlan}.id;
+              PVID = cfg.vlans.${b.net.vlan}.id;
+              EgressUntagged = cfg.vlans.${b.net.vlan}.id;
+            }
+          ];
+          linkConfig.RequiredForOnline = "no";
+        }
+      ) bsses
+    );
+  };
+}

modules/dns/acme-mx1-secret.nix

@@ -0,0 +1,19 @@
+# Shared TSIG secret for the dedicated acme_mx1 key.
+#
+# This key lets mx1 — and only mx1 — write _acme-challenge.mx1.cnx.email TXT
+# records on ns1 to obtain its mail TLS cert via ACME DNS-01. ns1 scopes it with
+# acl_acme_mx1 (attached only to the cnx.email zone) so the credential can touch
+# nothing else. ns1 renders this secret into a Knot key file; mx1 into a lego
+# rfc2136 env file; both must carry the same secret, hence one shared generator
+# with a per-host renderer that depends on it. Imported by ns1 and (via mail.nix)
+# mx1.
+{ pkgs, ... }:
+{
+  clan.core.vars.generators.dns-acme-mx1-secret = {
+    share = true;
+    files."secret".secret = true;
+    runtimeInputs = [ pkgs.openssl ];
+    # 32 random bytes, base64 — a valid hmac-sha256 TSIG secret.
+    script = ''openssl rand -base64 32 | tr -d '\n' > "$out"/secret'';
+  };
+}

modules/dns/acme-web01-secret.nix

@@ -0,0 +1,19 @@
+# Shared TSIG secret for the dedicated acme_web01 key.
+#
+# This key lets web01 — and only web01 — write _acme-challenge.cnx.network TXT
+# records on ns1 to obtain its wildcard (*.cnx.network) TLS cert via ACME DNS-01.
+# ns1 scopes it with acl_acme_web01 (attached only to the cnx.network zone) so the
+# credential can touch nothing else. ns1 renders this secret into a Knot key file;
+# web01 into a lego rfc2136 env file; both must carry the same secret, hence one
+# shared generator with a per-host renderer that depends on it. Imported by ns1
+# and (via web-proxy.nix) web01.
+{ pkgs, ... }:
+{
+  clan.core.vars.generators.dns-acme-web01-secret = {
+    share = true;
+    files."secret".secret = true;
+    runtimeInputs = [ pkgs.openssl ];
+    # 32 random bytes, base64 — a valid hmac-sha256 TSIG secret.
+    script = ''openssl rand -base64 32 | tr -d '\n' > "$out"/secret'';
+  };
+}

modules/dns/authoritative.nix

@@ -1,8 +1,17 @@
-{ config, pkgs, ... }:
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
 let
   # ZeroTier addresses — zone transfers run over the mesh, not the public net.
-  ns1zt = "fd06:1bad:ece2:92ad:ba99:939d:766d:8974";
-  ns2zt = "fd06:1bad:ece2:92ad:ba99:9323:61be:a09e";
+  mesh = import ../mesh-hosts.nix {
+    dir = config.clan.core.settings.directory;
+    inherit lib;
+  };
+  ns1zt = mesh.hosts.ns1;
+  ns2zt = mesh.hosts.ns2;
 in
 {
   # Shared TSIG key, generated once and copied to every machine that imports
@@ -26,24 +35,56 @@ in
 
   # knot binds 0.0.0.0@53 and ::@53, so free port 53 by disabling the
   # systemd-resolved stub listener. Resolution still works via nss-resolve.
-  services.resolved.extraConfig = "DNSStubListener=no";
+  services.resolved.settings.Resolve.DNSStubListener = "no";
 
   services.knot = {
     enable = true;
     # Including the key via keyFiles keeps the secret out of the Nix store.
     keyFiles = [ config.clan.core.vars.generators.dns-tsig.files."tsig.conf".path ];
     settings = {
-      server.listen = [ "0.0.0.0@53" "::@53" ];
-      log = [ { target = "syslog"; any = "info"; } ];
+      server.listen = [
+        "0.0.0.0@53"
+        "::@53"
+      ];
+      log = [
+        {
+          target = "syslog";
+          any = "info";
+        }
+      ];
 
       remote = [
-        { id = "ns1"; address = [ ns1zt ]; key = "cnx_xfr"; }
-        { id = "ns2"; address = [ ns2zt ]; key = "cnx_xfr"; }
+        {
+          id = "ns1";
+          address = [ ns1zt ];
+          key = "cnx_xfr";
+        }
+        {
+          id = "ns2";
+          address = [ ns2zt ];
+          key = "cnx_xfr";
+        }
       ];
 
       acl = [
-        { id = "acl_ns1"; address = [ ns1zt ]; key = "cnx_xfr"; action = [ "transfer" "notify" ]; }
-        { id = "acl_ns2"; address = [ ns2zt ]; key = "cnx_xfr"; action = [ "transfer" "notify" ]; }
+        {
+          id = "acl_ns1";
+          address = [ ns1zt ];
+          key = "cnx_xfr";
+          action = [
+            "transfer"
+            "notify"
+          ];
+        }
+        {
+          id = "acl_ns2";
+          address = [ ns2zt ];
+          key = "cnx_xfr";
+          action = [
+            "transfer"
+            "notify"
+          ];
+        }
       ];
     };
   };

modules/dns/domains.nix

@@ -4,4 +4,5 @@
   "cnx.network"
   "buildfor.life"
   "cnx.email"
+  "newedge.house"
 ]

modules/dns/zones/cnx.email.zone

@@ -11,8 +11,36 @@ $TTL 3600
 @       IN  NS      ns1.cnx.network.
 @       IN  NS      ns2.cnx.network.
 
-; ---- Mail (fill in once the mail host exists) ----
-;@      IN  MX 10   mail.cnx.email.
-;mail   IN  A       <mail-ipv4>
-;@      IN  TXT     "v=spf1 mx -all"
-;_dmarc IN  TXT     "v=DMARC1; p=quarantine; rua=mailto:postmaster@cnx.email"
+; ---- Mail ----
+mx1     IN  A       5.223.65.38
+mx1     IN  AAAA    2a01:4ff:2f0:1963::1
+; Client-facing alias for IMAP/submission (Thunderbird etc.); the cert carries
+; mail.cnx.email as a SAN. The MX must never point here (CNAMEs are illegal MX
+; targets) — server-to-server delivery and DANE stay on mx1.cnx.email.
+mail    IN  CNAME   mx1.cnx.email.
+@       IN  MX 10   mx1.cnx.email.
+@       IN  TXT     "v=spf1 mx -all"
+; Aggregate (rua) + forensic (ruf) reports go to the dmarc@cnx.email mailbox,
+; which parsedmarc on control polls and feeds into Grafana. fo=1 asks reporters
+; to send a forensic report on any SPF/DKIM failure.
+_dmarc  IN  TXT     "v=DMARC1; p=quarantine; rua=mailto:dmarc@cnx.email; ruf=mailto:dmarc@cnx.email; fo=1"
+
+; ---- DANE / TLSA ----
+; "3 1 1" = DANE-EE, SPKI, SHA-256: the digest of mx1's certificate public key.
+; Valid because the zone is DNSSEC-signed and the lego cert uses --reuse-key, so
+; the key (and thus this digest) is stable across renewals. Compute it AFTER the
+; first issuance and paste the hex below:
+;   ssh mx1 'openssl x509 -in /var/lib/acme/mx1.cnx.email/cert.pem -noout -pubkey \
+;     | openssl pkey -pubin -outform DER | openssl dgst -sha256 -binary | xxd -p -c256'
+_25._tcp.mx1  IN  TLSA  3 1 1  bd9a51f60b6d2dd20f18b3553d2795053ac52f87567a46bc892006bb58506404
+
+; ---- MTA-STS ----
+; Policy host (A/AAAA point at mx1); the _mta-sts TXT id MUST be bumped whenever
+; the policy file in modules/mail.nix changes, or senders keep the cached policy.
+mta-sts   IN  A       5.223.65.38
+mta-sts   IN  AAAA    2a01:4ff:2f0:1963::1
+_mta-sts  IN  TXT     "v=STSv1; id=2026061801"
+mail._domainkey IN TXT ( "v=DKIM1; k=rsa; "
+        "p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAr9QxTs5dLtY76bo156+Tp0GUoE554rMwIooIYa2MMYHNs8zPb0thFmaCKGAINdHKNIq2phXAlk51iBTfdqXjx7gVWSrs+ftykqO3b5hUjgImsgqPWGUTzy5/bUgcDELiD9KKEyKYD3+ebZEw6d0uvBvEsA6a1CPzOsufoCDtyKjByCuQzkCBrK25TUHFolGvEYcZexR0LSF+8hMss"
+        "xyw9NYiPpTXVCWQJnrZZpuOBiX0K2l5CAXVyuT/B5RcBXlAUhBTp3390VEhL0wAZMTOnvtvBYK3NnsTIh96fkh6MfWmre7Fi9hEq//xGf40N5/aomMjJrJdqFZJLZpDotb/XwIDAQAB"
+        )

modules/dns/zones/cnx.network.zone

@@ -24,3 +24,11 @@ control IN  AAAA    fd06:1bad:ece2:92ad:ba99:9306:1bad:ece2
 ; ---- Web / apex (fill in once you have a web host) ----
 ;@      IN  A       <web-ipv4>
 ;www    IN  CNAME   cnx.network.
+monitor IN  A       5.223.66.36
+
+; ---- web01 (public reverse proxy / TLS termination) ----
+; Serves a wildcard *.cnx.network TLS cert (ACME DNS-01) and forwards to internal
+; services over the mesh. Add a vhost in modules/web-proxy.nix and a CNAME here.
+web01   IN  A       5.223.55.246
+web01   IN  AAAA    2a01:4ff:2f0:2d8f::1
+grafana IN  CNAME   web01.cnx.network.

modules/dns/zones/newedge.house.zone

@@ -0,0 +1,13 @@
+$ORIGIN newedge.house.
+$TTL 3600
+
+@   IN  SOA ns1.cnx.network. hostmaster.cnx.network. (
+        2026061401  ; serial   (ignored: Knot auto-assigns a dateserial on signing)
+        3600        ; refresh
+        900         ; retry
+        604800      ; expire
+        300 )       ; negative-cache TTL
+
+; Served by the same nameservers (out-of-bailiwick, no glue needed here).
+@       IN  NS      ns1.cnx.network.
+@       IN  NS      ns2.cnx.network.

modules/docs.nix

@@ -0,0 +1,49 @@
+# Infra runbook (mdBook), built at Nix-build time from ./docs and served by Caddy.
+# Reachable only over the ZeroTier mesh (firewall rule below); the public side is
+# already closed by the Hetzner cloud firewall. Imported by control only.
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  mesh = import ./mesh-hosts.nix {
+    dir = config.clan.core.settings.directory;
+    inherit lib;
+  };
+  port = 8080;
+
+  site = pkgs.stdenvNoCC.mkDerivation {
+    name = "cnx-infra-docs";
+    src = ../docs;
+    nativeBuildInputs = [ pkgs.mdbook ];
+    # mdbook writes a state dir under $HOME; the build sandbox has none.
+    buildPhase = ''
+      export HOME=$TMPDIR
+      mdbook build -d $out
+    '';
+    dontInstall = true;
+  };
+in
+{
+  # ":port" makes Caddy serve plain HTTP (no automatic TLS) on all interfaces;
+  # the mesh-scoped firewall rule below is what constrains reachability.
+  services.caddy = {
+    enable = true;
+    virtualHosts.":${toString port}".extraConfig = ''
+      root * ${site}
+      # mdBook doesn't fingerprint asset filenames, and every file in the Nix
+      # store carries an epoch (1970) mtime, so file_server's only validator never
+      # changes across redeploys — conditional requests would 304 forever and pin
+      # browsers to stale docs. no-store sidesteps caching entirely; pages are tiny
+      # and mesh-only, so refetching on each load is free.
+      header Cache-Control "no-store"
+      file_server
+    '';
+  };
+
+  networking.firewall.extraInputRules = ''
+    ip6 saddr ${mesh.subnet} tcp dport ${toString port} accept
+  '';
+}

modules/hetzner-firewall-rules.nix

@@ -4,7 +4,10 @@
 # Public SSH (22) is intentionally absent: admin access rides the ZeroTier mesh
 # (inside UDP 9993), with emergency-access as the console fallback.
 let
-  world = [ "0.0.0.0/0" "::/0" ];
+  world = [
+    "0.0.0.0/0"
+    "::/0"
+  ];
 
   zerotier = {
     direction = "in";
@@ -21,15 +24,78 @@ let
     description = "ICMP (ping / PMTUD)";
   };
 
+  # Public mail ports for mx1 (MX for cnx.email). 25 is server-to-server
+  # delivery; 587/465 are client submission; 143/993 are IMAP. 443 serves only the
+  # MTA-STS policy (https://mta-sts.cnx.email/.well-known/mta-sts.txt); the cert
+  # itself uses ACME DNS-01 so port 80 stays closed. Admin still rides the mesh.
+  mailPort = port: description: {
+    direction = "in";
+    protocol = "tcp";
+    inherit port;
+    source_ips = world;
+    inherit description;
+  };
+  mailRules = [
+    (mailPort "25" "SMTP (inbound mail)")
+    (mailPort "587" "Submission (STARTTLS)")
+    (mailPort "465" "Submission (implicit TLS)")
+    (mailPort "143" "IMAP (STARTTLS)")
+    (mailPort "993" "IMAP (implicit TLS)")
+    (mailPort "443" "MTA-STS policy (HTTPS)")
+  ];
+
+  # web01 is a public reverse proxy with TLS termination. 443 serves the proxy;
+  # 80 only carries Caddy's HTTP->HTTPS redirect (the cert uses ACME DNS-01, not
+  # HTTP-01). Admin rides the mesh.
+  webRules = [
+    {
+      direction = "in";
+      protocol = "tcp";
+      port = "80";
+      source_ips = world;
+      description = "HTTP (redirect to HTTPS)";
+    }
+    {
+      direction = "in";
+      protocol = "tcp";
+      port = "443";
+      source_ips = world;
+      description = "HTTPS (reverse proxy / TLS termination)";
+    }
+  ];
+
   dnsRules = [
-    { direction = "in"; protocol = "udp"; port = "53"; source_ips = world; description = "DNS (UDP)"; }
-    { direction = "in"; protocol = "tcp"; port = "53"; source_ips = world; description = "DNS (TCP)"; }
+    {
+      direction = "in";
+      protocol = "udp";
+      port = "53";
+      source_ips = world;
+      description = "DNS (UDP)";
+    }
+    {
+      direction = "in";
+      protocol = "tcp";
+      port = "53";
+      source_ips = world;
+      description = "DNS (TCP)";
+    }
     zerotier
     ping
   ];
 in
 {
-  "clan-control" = [ zerotier ping ];
+  "clan-control" = [
+    zerotier
+    ping
+  ];
   "clan-ns1" = dnsRules;
   "clan-ns2" = dnsRules;
+  "clan-mx1" = mailRules ++ [
+    zerotier
+    ping
+  ];
+  "clan-web01" = webRules ++ [
+    zerotier
+    ping
+  ];
 }

modules/hetzner-firewall.nix

@@ -1,4 +1,9 @@
-{ config, lib, pkgs, ... }:
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
 let
   cfg = config.cnx.hetznerFirewall;
 in
@@ -29,8 +34,7 @@ in
     tokenFile = lib.mkOption {
       type = lib.types.path;
       default = config.clan.core.vars.generators.hetzner-firewall.files.token.path;
-      defaultText = lib.literalExpression
-        "config.clan.core.vars.generators.hetzner-firewall.files.token.path";
+      defaultText = lib.literalExpression "config.clan.core.vars.generators.hetzner-firewall.files.token.path";
       description = "File holding the Hetzner Cloud API token (Read & Write).";
     };
   };
@@ -48,7 +52,11 @@ in
       description = "Sync Hetzner Cloud firewall rules from Nix config";
       after = [ "network-online.target" ];
       wants = [ "network-online.target" ];
-      path = [ pkgs.curl pkgs.jq pkgs.coreutils ];
+      path = [
+        pkgs.curl
+        pkgs.jq
+        pkgs.coreutils
+      ];
       environment.SSL_CERT_FILE = "/etc/ssl/certs/ca-certificates.crt";
       serviceConfig = {
         Type = "oneshot";
@@ -71,20 +79,22 @@ in
           curl -fsS -H @"$hdr" -H "Content-Type: application/json" "$@"
         }
 
-        ${lib.concatStringsSep "\n" (lib.mapAttrsToList (fwName: rules: ''
-          name=${lib.escapeShellArg fwName}
-          rules=${lib.escapeShellArg (builtins.toJSON rules)}
-          id="$(hapi "$api/firewalls?name=$name" | jq -r '.firewalls[0].id // empty')"
-          if [ -z "$id" ]; then
-            echo "hetzner-firewall: creating $name"
-            jq -n --arg name "$name" --argjson rules "$rules" '{name: $name, rules: $rules}' \
-              | hapi -X POST --data-binary @- "$api/firewalls" > /dev/null
-          else
-            echo "hetzner-firewall: setting rules on $name (id $id)"
-            jq -n --argjson rules "$rules" '{rules: $rules}' \
-              | hapi -X POST --data-binary @- "$api/firewalls/$id/actions/set_rules" > /dev/null
-          fi
-        '') cfg.firewalls)}
+        ${lib.concatStringsSep "\n" (
+          lib.mapAttrsToList (fwName: rules: ''
+            name=${lib.escapeShellArg fwName}
+            rules=${lib.escapeShellArg (builtins.toJSON rules)}
+            id="$(hapi "$api/firewalls?name=$name" | jq -r '.firewalls[0].id // empty')"
+            if [ -z "$id" ]; then
+              echo "hetzner-firewall: creating $name"
+              jq -n --arg name "$name" --argjson rules "$rules" '{name: $name, rules: $rules}' \
+                | hapi -X POST --data-binary @- "$api/firewalls" > /dev/null
+            else
+              echo "hetzner-firewall: setting rules on $name (id $id)"
+              jq -n --argjson rules "$rules" '{rules: $rules}' \
+                | hapi -X POST --data-binary @- "$api/firewalls/$id/actions/set_rules" > /dev/null
+            fi
+          '') cfg.firewalls
+        )}
       '';
     };
 

modules/hosts.nix

@@ -0,0 +1,32 @@
+# Per-host public network facts: single source of truth for each machine's
+# public IPv4 and its static public IPv6. Consumed by clan.nix's `internet`
+# connection hosts (ipv4) and each machine's `cnx.staticIPv6` (ipv6), so an
+# address is written once instead of being duplicated across configs.
+#
+# NOT a driver for the DNS zone files — those stay hand-edited text, so a record
+# here that also appears as A/AAAA glue still needs a matching manual zone edit.
+#
+# ipv6 is the single address to assign from the host's allocated /64 (we take
+# ::1), without prefix length; cnx.staticIPv6 supplies the /64 default.
+{
+  control = {
+    ipv4 = "77.42.68.181";
+    ipv6 = "2a01:4f9:c013:e6d0::1";
+  };
+  ns1 = {
+    ipv4 = "46.224.170.206";
+    ipv6 = "2a01:4f8:c014:b5c5::1";
+  };
+  ns2 = {
+    ipv4 = "157.180.70.82";
+    ipv6 = "2a01:4f9:c014:6d87::1";
+  };
+  mx1 = {
+    ipv4 = "5.223.65.38";
+    ipv6 = "2a01:4ff:2f0:1963::1";
+  };
+  web01 = {
+    ipv4 = "5.223.55.246";
+    ipv6 = "2a01:4ff:2f0:2d8f::1";
+  };
+}

modules/mail-dmarc-cred.nix

@@ -0,0 +1,26 @@
+# Shared credential for the dmarc@cnx.email mailbox.
+#
+# DMARC aggregate/forensic reports are delivered to dmarc@cnx.email on mx1;
+# parsedmarc on control fetches them over IMAPS across the mesh and needs the
+# *plaintext* passphrase, while mx1's mailserver only needs the sha-512 hash.
+# clan vars secrets are per-machine, so this generator is shared (share = true)
+# to make the same value available on both hosts. Files are root-owned: SNM reads
+# the hash as root, and parsedmarc's ExecStartPre reads the passphrase as root.
+# Imported by mx1 (via mail.nix) and control (via monitoring/parsedmarc.nix).
+{ pkgs, ... }:
+{
+  clan.core.vars.generators.mail-dmarc-cred = {
+    share = true;
+    files."passphrase".secret = true; # read by parsedmarc on control
+    files."hash".secret = true; # consumed by the mailserver on mx1
+    runtimeInputs = [
+      pkgs.xkcdpass
+      pkgs.mkpasswd
+    ];
+    script = ''
+      pass="$(xkcdpass --numwords=4 --delimiter=- --case=lower)-$((RANDOM % 90 + 10))"
+      printf '%s' "$pass" > "$out"/passphrase
+      printf '%s' "$pass" | mkpasswd -s -m sha-512 > "$out"/hash
+    '';
+  };
+}

modules/mail.nix

@@ -0,0 +1,163 @@
+# Declarative mail stack for mx1 (Simple NixOS Mailserver: Postfix + Dovecot +
+# Rspamd + OpenDKIM). Imported by machines/mx1 alongside the SNM flake module.
+#
+# Mailboxes are virtual (not system users): each address below is a login account
+# whose password is auto-generated by a clan vars generator as a four-word
+# passphrase with a trailing number (e.g. otter-lantern-cobalt-driftwood-42). The
+# generator stores both the passphrase and its sha-512 hash. To add a mailbox:
+# append the address to `accounts`, run `clan vars generate mx1`, redeploy mx1,
+# then hand the passphrase to the user:
+#   clan vars get mx1 mail-passwd-<addr>/passphrase
+# (addr with @ and . replaced by -at- and -, e.g. mail-passwd-postmaster-at-cnx-email)
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  hosts = import ./hosts.nix;
+  fqdn = "mx1.cnx.email";
+  mtaStsHost = "mta-sts.cnx.email";
+  # Client-facing alias (CNAME -> mx1) so Thunderbird etc. can use mail.cnx.email
+  # for submission/IMAP; added as a cert SAN so TLS validates against that name.
+  clientHost = "mail.cnx.email";
+
+  # MTA-STS policy served at https://mta-sts.cnx.email/.well-known/mta-sts.txt.
+  # enforce = a sending MTA that fetched this must use a valid, MX-matching TLS
+  # cert or refuse to deliver. Bump the _mta-sts TXT id (in the zone) whenever
+  # this changes.
+  mtaStsPolicy = pkgs.writeText "mta-sts.txt" ''
+    version: STSv1
+    mode: enforce
+    mx: ${fqdn}
+    max_age: 604800
+  '';
+
+  # The mailboxes mx1 serves. postmaster is required by RFC 5321.
+  accounts = [
+    "postmaster@cnx.email"
+  ];
+
+  genName = addr: "mail-passwd-" + lib.replaceStrings [ "@" "." ] [ "-at-" "-" ] addr;
+
+  passwdGenerators = lib.listToAttrs (
+    map (addr: {
+      name = genName addr;
+      value = {
+        files."passphrase".secret = true; # retrievable to hand to the user
+        files."hash".secret = true; # consumed by SNM's hashedPasswordFile
+        runtimeInputs = [
+          pkgs.xkcdpass
+          pkgs.mkpasswd
+        ];
+        script = ''
+          pass="$(xkcdpass --numwords=4 --delimiter=- --case=lower)-$((RANDOM % 90 + 10))"
+          printf '%s' "$pass" > "$out"/passphrase
+          printf '%s' "$pass" | mkpasswd -s -m sha-512 > "$out"/hash
+        '';
+      };
+    }) accounts
+  );
+
+  loginAccounts =
+    lib.listToAttrs (
+      map (addr: {
+        name = addr;
+        value.hashedPasswordFile = config.clan.core.vars.generators.${genName addr}.files."hash".path;
+      }) accounts
+    )
+    // {
+      # DMARC report inbox (rua/ruf target in the cnx.email zone). Its password
+      # comes from the *shared* mail-dmarc-cred generator instead of the per-machine
+      # set above, so parsedmarc on control can read the same passphrase over the
+      # mesh. Retrieve it with: clan vars get mx1 mail-dmarc-cred/passphrase
+      "dmarc@cnx.email".hashedPasswordFile =
+        config.clan.core.vars.generators.mail-dmarc-cred.files."hash".path;
+    };
+in
+{
+  imports = [
+    ./dns/acme-mx1-secret.nix
+    ./mail-dmarc-cred.nix
+  ];
+
+  clan.core.vars.generators = passwdGenerators // {
+    # Render the shared acme_mx1 TSIG secret into a lego rfc2136 env file. lego
+    # (via security.acme below) uses it to write the _acme-challenge.mx1.cnx.email
+    # TXT record to ns1, which authorizes the acme_mx1 key for exactly that owner.
+    dns-acme-rfc2136 = {
+      files."rfc2136.env".secret = true; # root-owned; systemd reads it as root
+      dependencies = [ "dns-acme-mx1-secret" ];
+      script = ''
+        printf 'RFC2136_NAMESERVER=${hosts.ns1.ipv4}:53\nRFC2136_TSIG_ALGORITHM=hmac-sha256.\nRFC2136_TSIG_KEY=acme_mx1\nRFC2136_TSIG_SECRET=%s\n' \
+          "$(cat "$in"/dns-acme-mx1-secret/secret)" > "$out"/rfc2136.env
+      '';
+    };
+  };
+
+  mailserver = {
+    enable = true;
+    # Fresh install: declare the latest layout the nixos-26.05 branch ships (3),
+    # so SNM uses the current dovecot mail directory layout with nothing to migrate.
+    stateVersion = 3;
+    inherit fqdn;
+    domains = [ "cnx.email" ];
+    accounts = loginAccounts;
+
+    # Consume a security.acme cert we obtain ourselves via DNS-01 (below); no
+    # web server and no inbound HTTP needed, so port 80 stays closed. The extra
+    # SAN hosts (MTA-STS, client alias) are attached to that cert via
+    # security.acme.certs.${fqdn}.extraDomainNames below.
+    x509.useACMEHost = fqdn;
+
+    dkim.defaults.selector = "mail";
+  };
+
+  security.acme = {
+    acceptTerms = true;
+    defaults.email = "postmaster@cnx.email";
+    certs.${fqdn} = {
+      # The MTA-STS endpoint and client-facing alias ride this one cert as SANs.
+      extraDomainNames = [
+        mtaStsHost
+        clientHost
+      ];
+      dnsProvider = "rfc2136";
+      environmentFile = config.clan.core.vars.generators.dns-acme-rfc2136.files."rfc2136.env".path;
+      # ns1 is the only nameserver that accepts the acme_mx1 UPDATE; check
+      # propagation against it directly rather than a public resolver.
+      dnsResolver = "${hosts.ns1.ipv4}:53";
+      # Keep the private key fixed across renewals so the DANE TLSA "3 1 1"
+      # record (public-key digest, published in the zone) stays valid.
+      extraLegoRenewFlags = [ "--reuse-key" ];
+      # Caddy serves the MTA-STS endpoint from explicit cert file paths, so it
+      # won't notice a renewal on its own — reload it whenever the cert changes.
+      # (Merges with the postfix/dovecot reloads SNM wires up for this cert.)
+      reloadServices = [ "caddy.service" ];
+    };
+  };
+
+  # The mail cert is owned group=acme (SNM adds postfix/dovecot); Caddy serves the
+  # MTA-STS endpoint from the same cert, so it needs to read the key too.
+  users.users.caddy.extraGroups = [ "acme" ];
+
+  # MTA-STS policy endpoint, served by Caddy (same web server as control's docs).
+  # The explicit `tls cert key` points at the lego-issued mail cert (which carries
+  # mta-sts.cnx.email as a SAN) and disables Caddy's automatic ACME, so no extra
+  # issuance happens and the DANE TLSA key stays stable. Only :443 is opened.
+  services.caddy = {
+    enable = true;
+    virtualHosts.${mtaStsHost}.extraConfig = ''
+      tls /var/lib/acme/${fqdn}/cert.pem /var/lib/acme/${fqdn}/key.pem
+      root * ${pkgs.writeTextDir ".well-known/mta-sts.txt" (builtins.readFile mtaStsPolicy)}
+      file_server
+    '';
+  };
+
+  # DKIM private keys are generated on first start under this dir. They're
+  # regenerable (rotate + republish the TXT), but declaring the path as clan
+  # state lets a borg client back it up to avoid a needless DNS round-trip on
+  # restore. Wiring mx1 into the borgbackup instance is a separate step.
+  clan.core.state.mail-dkim.folders = [ config.mailserver.dkimKeyDirectory ];
+}

modules/mesh-hosts.nix

@@ -0,0 +1,40 @@
+# ZeroTier (clan mesh) addresses — the private IPv6 overlay every machine shares.
+# DNS zone transfers and metrics scraping ride this mesh, never the public net.
+#
+# Rather than hardcoding the addresses, we read them from the public clan vars
+# that clan-core's zerotier service writes. As of clan-core 26.05 these are
+# shared, instance-scoped generators: the per-machine IP lives at
+# vars/shared/zerotier-ip-<machine>-<instance>/ip and the network id at
+# vars/shared/zerotier-network-<instance>/network-id (instance = "zerotier",
+# the inventory.instances.zerotier name in clan.nix). This keeps the mesh map in
+# lockstep with the actual identities: regenerate or re-key a node and its
+# address here follows automatically. `dir` is the clan directory: from NixOS
+# `config.clan.core.settings.directory`, from clan.nix `config.directory`.
+{ dir, lib }:
+let
+  instance = "zerotier";
+
+  readIp =
+    machine: builtins.readFile "${dir}/vars/shared/zerotier-ip-${machine}-${instance}/ip/value";
+
+  hosts = lib.genAttrs [
+    "control"
+    "ns1"
+    "ns2"
+    "mx1"
+    "web01"
+    "gw-cnx-1"
+  ] readIp;
+
+  # RFC 4193 prefix of this ZeroTier network: fd + the 8-byte network id + the
+  # 0x9993 marker. The network id is a public, shared var for the instance.
+  # The /88 (11 bytes) covers fd + network id + 0x99 + 0x93, i.e. every mesh peer,
+  # and is used to scope mesh-only firewall rules.
+  networkId = builtins.readFile "${dir}/vars/shared/zerotier-network-${instance}/network-id/value";
+  full = "fd" + networkId + "9993"; # 22 hex chars = 11 bytes
+  hextet = i: builtins.substring (i * 4) 4 full;
+  subnet = "${hextet 0}:${hextet 1}:${hextet 2}:${hextet 3}:${hextet 4}:${builtins.substring 20 2 full}00::/88";
+in
+{
+  inherit hosts subnet;
+}

modules/monitoring/alerts.nix

@@ -0,0 +1,133 @@
+# Alerting rules, evaluated by vmalert against VictoriaMetrics on control.
+# Everything is declared here in git. vmalert remote-writes alert state back to
+# VM, so firing alerts surface as the `ALERTS{alertstate="firing"}` series and
+# can be viewed in Grafana. No notifier is wired yet: notifier.blackhole makes
+# that explicit (vmalert evaluates rules but sends nowhere). To deliver alerts
+# later, drop blackhole and set settings."notifier.url" to an Alertmanager.
+{ ... }:
+let
+  vmUrl = "http://127.0.0.1:8428";
+in
+{
+  services.vmalert.instances.cnx = {
+    enable = true;
+    settings = {
+      "datasource.url" = vmUrl;
+      "remoteWrite.url" = vmUrl; # persists ALERTS / ALERTS_FOR_STATE back to VM
+      "notifier.blackhole" = true;
+      "httpListenAddr" = "127.0.0.1:8880"; # vmalert UI/API, loopback only (like VM)
+    };
+    rules.groups = [
+      {
+        name = "dns";
+        rules = [
+          {
+            alert = "DNSSecondaryOutOfSync";
+            expr = "max by (zone) (knot_zone_serial) - min by (zone) (knot_zone_serial) > 0";
+            for = "15m";
+            labels.severity = "warning";
+            annotations.summary = "Zone {{ $labels.zone }} SOA serial differs between nameservers";
+            annotations.description = "The secondary is out of sync with the primary for {{ $labels.zone }}. `knotc zone-retransfer {{ $labels.zone }}` on ns2 forces a fresh pull.";
+          }
+          {
+            alert = "ZoneExpiryLow";
+            expr = "knot_zone_status_expiration < 3600";
+            for = "5m";
+            labels.severity = "critical";
+            annotations.summary = "Zone {{ $labels.zone }} on {{ $labels.instance }} is within 1h of expiry";
+            annotations.description = "Transfers to the secondary appear to be failing; the zone stops being served when the SOA expire timer hits zero.";
+          }
+        ];
+      }
+      {
+        name = "host";
+        rules = [
+          {
+            alert = "ScrapeTargetDown";
+            expr = "up == 0";
+            for = "5m";
+            labels.severity = "critical";
+            annotations.summary = "{{ $labels.job }} exporter on {{ $labels.instance }} is down";
+            annotations.description = "VictoriaMetrics cannot scrape this target; its metrics are missing.";
+          }
+          {
+            alert = "RootFilesystemFull";
+            expr = ''100 * (1 - node_filesystem_avail_bytes{mountpoint="/",fstype!="tmpfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!="tmpfs"}) > 90'';
+            for = "15m";
+            labels.severity = "warning";
+            annotations.summary = "Root filesystem on {{ $labels.instance }} is over 90% full";
+          }
+        ];
+      }
+      {
+        # WAN quality on the site gateways, fed by the hourly speedtest timer
+        # (modules/clan/router/speedtest.nix) via the node_exporter textfile
+        # collector. Degradation is judged against the link's own 7-day median,
+        # so no per-site/per-ISP-plan threshold needs maintaining.
+        name = "wan";
+        rules = [
+          {
+            alert = "WANSpeedDegraded";
+            expr = "speedtest_download_mbps < 0.5 * quantile_over_time(0.5, speedtest_download_mbps[7d])";
+            for = "3h";
+            labels.severity = "warning";
+            annotations.summary = "WAN download on {{ $labels.instance }} is below half its 7-day median";
+            annotations.description = "Sustained for 3h (several test runs) — likely ISP degradation, line fault, or saturation. Compare speedtest_upload_mbps/speedtest_ping_ms and run `librespeed-cli` manually on the gateway.";
+          }
+          {
+            alert = "WANSpeedTestFailing";
+            expr = "speedtest_success == 0";
+            for = "3h";
+            labels.severity = "warning";
+            annotations.summary = "Speed tests on {{ $labels.instance }} have been failing for 3h";
+            annotations.description = "librespeed-cli cannot complete a test while other traffic may still flow; check `journalctl -u speedtest` on the gateway.";
+          }
+        ];
+      }
+      {
+        name = "backup";
+        rules = [
+          {
+            alert = "BackupJobFailed";
+            expr = ''node_systemd_unit_state{name=~"borgbackup-job-.+\\.service",state="failed"} == 1'';
+            for = "5m";
+            labels.severity = "warning";
+            annotations.summary = "Backup job {{ $labels.name }} on {{ $labels.instance }} failed";
+            annotations.description = "The borgbackup run did not complete. Check `systemctl status {{ $labels.name }}` and `journalctl -u {{ $labels.name }}` on the client; `borgbackup-create` re-runs it.";
+          }
+          {
+            alert = "BackupStale";
+            expr = ''time() - node_systemd_timer_last_trigger_seconds{name=~"borgbackup-job-.+\\.timer"} > 93600'';
+            for = "30m";
+            labels.severity = "warning";
+            annotations.summary = "No successful backup on {{ $labels.instance }} for over 26h";
+            annotations.description = "The daily backup timer {{ $labels.name }} has not fired within its expected window; the most recent archive is stale. A value far above 26h (or no data) means backups have stopped entirely.";
+          }
+        ];
+      }
+      {
+        # Outside-in DNS probes (blackbox on control). The `for` rides out a
+        # single dropped UDP packet; only a sustained failure fires.
+        name = "dns_probe";
+        rules = [
+          {
+            alert = "DNSResolutionProbeFailed";
+            expr = ''probe_success{query="SOA"} == 0'';
+            for = "5m";
+            labels.severity = "critical";
+            annotations.summary = "{{ $labels.zone }} is not resolving from {{ $labels.instance }}";
+            annotations.description = "The blackbox SOA probe to this public nameserver address is failing; from the outside the zone looks unavailable there, which the Knot stats would not show.";
+          }
+          {
+            alert = "DNSSECProbeFailed";
+            expr = ''probe_success{query="DNSKEY"} == 0'';
+            for = "5m";
+            labels.severity = "critical";
+            annotations.summary = "{{ $labels.zone }} DNSKEY missing from {{ $labels.instance }}";
+            annotations.description = "The DNSKEY probe to this public nameserver address is failing: the zone's signing keys are not being served, so validating resolvers will treat answers as bogus.";
+          }
+        ];
+      }
+    ];
+  };
+}

modules/monitoring/blackbox-probes.nix

@@ -0,0 +1,108 @@
+# Blackbox DNS probe definitions, shared between the exporter module
+# (modules/monitoring/blackbox.nix, which renders these into the blackbox
+# config) and the scraper (modules/monitoring/server.nix, which turns them into
+# VictoriaMetrics scrape jobs). Kept in one place so the module list and the
+# scrape jobs can never drift apart.
+#
+# These query the nameservers' PUBLIC addresses, i.e. the path a real internet
+# resolver takes, not the mesh — the whole point is to catch outside-in
+# resolution failures the Knot stats can't see. For each zone we run two probes
+# per endpoint: an SOA query (is the zone being served at all?) and a DNSKEY
+# query (is it still DNSSEC-signed?). Blackbox has no DO-bit option, so we ask
+# for DNSKEY directly — an authoritative signed zone returns it without EDNS0,
+# and its absence means signing has broken.
+{ lib }:
+let
+  domains = import ../dns/domains.nix;
+
+  blackboxAddr = "127.0.0.1:9115";
+
+  # Public endpoints of the authoritative nameservers. The v4 addresses also
+  # appear in the `internet` instance in clan.nix; the v6 ones in each ns
+  # machine's cnx.staticIPv6. IPv6 literals are bracketed for host:port.
+  endpoints = [
+    {
+      instance = "ns1 v4";
+      target = "46.224.170.206:53";
+    }
+    {
+      instance = "ns1 v6";
+      target = "[2a01:4f8:c014:b5c5::1]:53";
+    }
+    {
+      instance = "ns2 v4";
+      target = "157.180.70.82:53";
+    }
+    {
+      instance = "ns2 v6";
+      target = "[2a01:4f9:c014:6d87::1]:53";
+    }
+  ];
+
+  queries = [
+    {
+      name = "soa";
+      type = "SOA";
+    }
+    {
+      name = "dnskey";
+      type = "DNSKEY";
+    }
+  ];
+
+  sanitize = lib.replaceStrings [ "." ] [ "_" ];
+  moduleName = zone: q: "dns_${q.name}_${sanitize zone}";
+
+  modules = lib.listToAttrs (
+    lib.concatMap (
+      zone:
+      map (
+        q:
+        lib.nameValuePair (moduleName zone q) {
+          prober = "dns";
+          timeout = "5s";
+          dns = {
+            query_name = "${zone}.";
+            query_type = q.type;
+            valid_rcodes = [ "NOERROR" ];
+            # Fail unless at least one answer RR of the queried type is present:
+            # a NOERROR with an empty answer (or a missing DNSKEY) still fails.
+            validate_answer_rrs.fail_if_not_matches_regexp = [ "\\s${q.type}\\s" ];
+          };
+        }
+      ) queries
+    ) domains
+  );
+
+  scrapeConfigs = lib.concatMap (
+    zone:
+    map (q: {
+      job_name = "blackbox_${moduleName zone q}";
+      metrics_path = "/probe";
+      params.module = [ (moduleName zone q) ];
+      static_configs = map (e: {
+        targets = [ e.target ];
+        labels = {
+          instance = e.instance;
+          zone = zone;
+          query = q.type;
+        };
+      }) endpoints;
+      # Hand the real DNS server to blackbox as ?target=, then point the scrape
+      # at the exporter itself.
+      relabel_configs = [
+        {
+          source_labels = [ "__address__" ];
+          target_label = "__param_target";
+        }
+        {
+          target_label = "__address__";
+          replacement = blackboxAddr;
+        }
+      ];
+    }) queries
+  ) domains;
+in
+{
+  inherit modules scrapeConfigs blackboxAddr;
+}

modules/monitoring/blackbox.nix

@@ -0,0 +1,24 @@
+# Blackbox exporter on control: outside-in DNS probes against the public
+# nameserver addresses (see blackbox-probes.nix for what and why). Bound to
+# loopback — only VictoriaMetrics on the same host scrapes its /probe endpoint,
+# and the scrape jobs that drive it live in server.nix. The probes leave control
+# over the public internet to reach ns1/ns2, which is the path we want to test.
+{
+  lib,
+  pkgs,
+  ...
+}:
+let
+  probes = import ./blackbox-probes.nix { inherit lib; };
+in
+{
+  services.prometheus.exporters.blackbox = {
+    enable = true;
+    listenAddress = "127.0.0.1";
+    port = 9115;
+    # JSON is valid YAML; enableConfigCheck runs the exporter's own --config.check
+    # against this file at build time, so a malformed prober is caught here.
+    configFile = pkgs.writeText "blackbox.yml" (builtins.toJSON { inherit (probes) modules; });
+    enableConfigCheck = true;
+  };
+}

modules/monitoring/dashboards/backups.json

@@ -0,0 +1,217 @@
+{
+  "uid": "cnx-backups",
+  "title": "CNX Backups",
+  "tags": ["backup", "borg", "cnx"],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "version": 1,
+  "refresh": "1m",
+  "time": { "from": "now-7d", "to": "now" },
+  "templating": { "list": [] },
+  "annotations": { "list": [] },
+  "panels": [
+    {
+      "type": "row",
+      "title": "Backups",
+      "id": 1,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }
+    },
+    {
+      "type": "stat",
+      "title": "Backup health",
+      "description": "1 if any borgbackup job is in the failed state, 0 otherwise. A successful run leaves the oneshot unit inactive (still OK); only a real failure shows FAILED. Derived from the node_exporter systemd collector on the backup client (ns1).",
+      "id": 2,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 5, "w": 8, "x": 0, "y": 1 },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "thresholds" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "green", "value": null }]
+          },
+          "noValue": "no data",
+          "mappings": [
+            {
+              "type": "value",
+              "options": {
+                "0": { "text": "OK", "color": "green", "index": 0 },
+                "1": { "text": "FAILED", "color": "red", "index": 1 }
+              }
+            }
+          ]
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "auto",
+        "orientation": "auto"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "max(node_systemd_unit_state{name=~\"borgbackup-job-.+\\\\.service\",state=\"failed\"})",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "type": "stat",
+      "title": "Last backup run",
+      "description": "When the most recent backup timer last fired (the daily borgbackup job). 'No data' before the first run.",
+      "id": 3,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 5, "w": 8, "x": 8, "y": 1 },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "dateTimeFromNow",
+          "color": { "mode": "fixed", "fixedColor": "text" },
+          "noValue": "never"
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "none",
+        "graphMode": "none",
+        "textMode": "auto",
+        "orientation": "auto"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "max(node_systemd_timer_last_trigger_seconds{name=~\"borgbackup-job-.+\\\\.timer\"}) * 1000",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "type": "stat",
+      "title": "Time since last backup",
+      "description": "Age of the most recent backup. Backups run daily, so anything past ~25h means a run was missed. Red over 25h.",
+      "id": 4,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 5, "w": 8, "x": 16, "y": 1 },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s",
+          "color": { "mode": "thresholds" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "red", "value": 90000 }
+            ]
+          },
+          "noValue": "never"
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "auto",
+        "orientation": "auto"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "time() - max(node_systemd_timer_last_trigger_seconds{name=~\"borgbackup-job-.+\\\\.timer\"})",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "type": "table",
+      "title": "Backup jobs (current state)",
+      "description": "Every borgbackup job and the systemd unit state it is currently in, per client. 'inactive' is the normal resting state of a oneshot job between runs.",
+      "id": 5,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
+      "options": { "showHeader": true },
+      "fieldConfig": {
+        "defaults": { "custom": { "align": "auto" } },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "node_systemd_unit_state{name=~\"borgbackup-job-.+\\\\.service\"} == 1",
+          "format": "table",
+          "instant": true
+        }
+      ],
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true,
+              "Value": true,
+              "__name__": true,
+              "job": true,
+              "type": true
+            }
+          }
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Failed state over time",
+      "description": "1 while a backup job is in the failed state. A spike here is a backup that did not complete and was not retried before the next scrape.",
+      "id": 6,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
+      "fieldConfig": {
+        "defaults": { "unit": "short", "min": 0, "max": 1 },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "node_systemd_unit_state{name=~\"borgbackup-job-.+\\\\.service\",state=\"failed\"}",
+          "legendFormat": "{{instance}} {{name}}"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Time since last backup (history)",
+      "description": "Age of the latest backup over time. The sawtooth should reset to near zero once a day; a steady climb without a reset means backups stopped running.",
+      "id": 7,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 14 },
+      "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "time() - node_systemd_timer_last_trigger_seconds{name=~\"borgbackup-job-.+\\\\.timer\"}",
+          "legendFormat": "{{instance}} {{name}}"
+        }
+      ]
+    }
+  ]
+}

modules/monitoring/dashboards/dns.json

@@ -0,0 +1,326 @@
+{
+  "uid": "cnx-dns",
+  "title": "CNX DNS",
+  "tags": ["dns", "knot", "cnx"],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "version": 3,
+  "refresh": "30s",
+  "time": { "from": "now-6h", "to": "now" },
+  "templating": { "list": [] },
+  "annotations": { "list": [] },
+  "panels": [
+    {
+      "type": "row",
+      "title": "Alerts",
+      "id": 11,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }
+    },
+    {
+      "type": "table",
+      "title": "Active alerts",
+      "description": "Firing vmalert alerts (the ALERTS series vmalert writes back to VictoriaMetrics). An empty table means all clear.",
+      "id": 12,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 7, "w": 24, "x": 0, "y": 1 },
+      "options": { "showHeader": true },
+      "fieldConfig": {
+        "defaults": { "custom": { "align": "auto" } },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "ALERTS{alertstate=\"firing\"}",
+          "format": "table",
+          "instant": true
+        }
+      ],
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true,
+              "Value": true,
+              "__name__": true,
+              "alertstate": true
+            }
+          }
+        }
+      ]
+    },
+    {
+      "type": "row",
+      "title": "DNS / Zones",
+      "id": 1,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 8 }
+    },
+    {
+      "type": "table",
+      "title": "Zone SOA serial (per nameserver)",
+      "description": "ns1 and ns2 should report the same serial per zone. A divergence here is the secondary-out-of-sync condition.",
+      "id": 2,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 },
+      "options": { "showHeader": true },
+      "fieldConfig": {
+        "defaults": { "custom": { "align": "auto" } },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "knot_zone_serial",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "{{zone}} @ {{instance}}"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Seconds until zone expiry",
+      "description": "On secondaries this counts down between successful transfers; a steady decline toward zero means transfers are failing.",
+      "id": 3,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 9 },
+      "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "knot_zone_status_expiration",
+          "legendFormat": "{{zone}} @ {{instance}}"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Query rate by nameserver",
+      "id": 4,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 17 },
+      "fieldConfig": { "defaults": { "unit": "qps" }, "overrides": [] },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "sum by (instance) (rate(knot_stats_request_protocol_total[5m]))",
+          "legendFormat": "{{instance}}"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Response codes",
+      "id": 5,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 17 },
+      "fieldConfig": { "defaults": { "unit": "qps" }, "overrides": [] },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "sum by (type) (rate(knot_stats_response_code_total[5m]))",
+          "legendFormat": "{{type}}"
+        }
+      ]
+    },
+    {
+      "type": "row",
+      "title": "Hosts",
+      "id": 6,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 25 }
+    },
+    {
+      "type": "timeseries",
+      "title": "CPU busy %",
+      "id": 7,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 26 },
+      "fieldConfig": {
+        "defaults": { "unit": "percent", "min": 0, "max": 100 },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "100 - (avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)",
+          "legendFormat": "{{instance}}"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Memory used %",
+      "id": 8,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 26 },
+      "fieldConfig": {
+        "defaults": { "unit": "percent", "min": 0, "max": 100 },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "100 * (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)",
+          "legendFormat": "{{instance}}"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Root filesystem used %",
+      "id": 9,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 34 },
+      "fieldConfig": {
+        "defaults": { "unit": "percent", "min": 0, "max": 100 },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "100 * (1 - node_filesystem_avail_bytes{mountpoint=\"/\",fstype!=\"tmpfs\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!=\"tmpfs\"})",
+          "legendFormat": "{{instance}}"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Load average (1m)",
+      "id": 10,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 34 },
+      "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "node_load1",
+          "legendFormat": "{{instance}}"
+        }
+      ]
+    },
+    {
+      "type": "row",
+      "title": "DNS probes (outside-in)",
+      "id": 20,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 42 }
+    },
+    {
+      "type": "table",
+      "title": "Probe status (per zone / server)",
+      "description": "blackbox_exporter on control queries each nameserver's public address (v4 + v6) for every zone: an SOA query (zone served) and a DNSKEY query (still signed). UP = the resolver's-eye view is healthy.",
+      "id": 21,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 43 },
+      "options": { "showHeader": true },
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "align": "auto",
+            "cellOptions": { "type": "color-background" }
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "red", "value": null },
+              { "color": "green", "value": 1 }
+            ]
+          },
+          "mappings": [
+            {
+              "type": "value",
+              "options": {
+                "0": { "text": "DOWN", "index": 0 },
+                "1": { "text": "UP", "index": 1 }
+              }
+            }
+          ]
+        },
+        "overrides": [
+          {
+            "matcher": { "id": "byName", "options": "zone" },
+            "properties": [
+              { "id": "custom.cellOptions", "value": { "type": "auto" } }
+            ]
+          },
+          {
+            "matcher": { "id": "byName", "options": "query" },
+            "properties": [
+              { "id": "custom.cellOptions", "value": { "type": "auto" } }
+            ]
+          },
+          {
+            "matcher": { "id": "byName", "options": "instance" },
+            "properties": [
+              { "id": "custom.cellOptions", "value": { "type": "auto" } }
+            ]
+          }
+        ]
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "probe_success",
+          "format": "table",
+          "instant": true
+        }
+      ],
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": { "Time": true, "__name__": true, "job": true },
+            "renameByName": { "Value": "status" }
+          }
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Probe success (1 = ok)",
+      "description": "0 means the probe failed: the zone is not being served or not signed from that public address. Sustained failures fire DNSResolutionProbeFailed / DNSSECProbeFailed.",
+      "id": 22,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 43 },
+      "fieldConfig": {
+        "defaults": { "unit": "short", "min": 0, "max": 1 },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "probe_success",
+          "legendFormat": "{{zone}} {{query}} @ {{instance}}"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "DNS probe latency",
+      "description": "Total round-trip time of each blackbox DNS probe. A climbing trend points at a slow or overloaded nameserver before it starts failing outright.",
+      "id": 23,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 51 },
+      "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "probe_duration_seconds",
+          "legendFormat": "{{zone}} {{query}} @ {{instance}}"
+        }
+      ]
+    }
+  ]
+}

modules/monitoring/dashboards/uptime.json

@@ -0,0 +1,194 @@
+{
+  "uid": "cnx-uptime",
+  "title": "CNX Uptime",
+  "tags": ["uptime", "availability", "cnx"],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "version": 1,
+  "refresh": "30s",
+  "time": { "from": "now-24h", "to": "now" },
+  "templating": { "list": [] },
+  "annotations": { "list": [] },
+  "panels": [
+    {
+      "type": "row",
+      "title": "Uptime",
+      "id": 1,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }
+    },
+    {
+      "type": "stat",
+      "title": "Host status",
+      "description": "Whether VictoriaMetrics is currently able to scrape each host's node_exporter. UP means the host (and its mesh path) is reachable; DOWN means the scrape failed. One tile per machine.",
+      "id": 2,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 6, "w": 12, "x": 0, "y": 1 },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "thresholds" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "green", "value": null }]
+          },
+          "noValue": "no data",
+          "mappings": [
+            {
+              "type": "value",
+              "options": {
+                "0": { "text": "DOWN", "color": "red", "index": 0 },
+                "1": { "text": "UP", "color": "green", "index": 1 }
+              }
+            }
+          ]
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "value_and_name",
+        "orientation": "auto"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "up{job=\"node\"}",
+          "legendFormat": "{{instance}}",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "type": "stat",
+      "title": "Current uptime",
+      "description": "Time since each host last booted (now - node_boot_time_seconds). A value that drops back to near zero means the host rebooted.",
+      "id": 3,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 6, "w": 12, "x": 12, "y": 1 },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "dtdurations",
+          "color": { "mode": "fixed", "fixedColor": "text" },
+          "noValue": "no data"
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "none",
+        "graphMode": "none",
+        "textMode": "value_and_name",
+        "orientation": "auto"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "time() - node_boot_time_seconds{job=\"node\"}",
+          "legendFormat": "{{instance}}",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "type": "bargauge",
+      "title": "Availability over window",
+      "description": "Fraction of successful scrapes over the selected time range, per host (avg of up over $__range). 100% means every scrape in the window succeeded; dips reveal flapping or outages. Red below 99%.",
+      "id": 4,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 7 },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "color": { "mode": "thresholds" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "red", "value": null },
+              { "color": "yellow", "value": 99 },
+              { "color": "green", "value": 99.9 }
+            ]
+          },
+          "noValue": "no data"
+        },
+        "overrides": []
+      },
+      "options": {
+        "displayMode": "gradient",
+        "orientation": "horizontal",
+        "showUnfilled": true,
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        }
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "avg_over_time(up{job=\"node\"}[$__range]) * 100",
+          "legendFormat": "{{instance}}",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Uptime over time",
+      "description": "Host uptime across the window. The line should climb steadily; a reset to zero marks a reboot.",
+      "id": 5,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 7 },
+      "fieldConfig": {
+        "defaults": { "unit": "s", "custom": { "fillOpacity": 0 } },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "time() - node_boot_time_seconds{job=\"node\"}",
+          "legendFormat": "{{instance}}"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Up/down history",
+      "description": "1 while a host's node_exporter was scrapeable, 0 while it was not. Gaps to zero are outages or lost mesh connectivity.",
+      "id": 6,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 6, "w": 24, "x": 0, "y": 15 },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "min": 0,
+          "max": 1,
+          "custom": { "fillOpacity": 20, "lineInterpolation": "stepAfter" }
+        },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "up{job=\"node\"}",
+          "legendFormat": "{{instance}}"
+        }
+      ]
+    }
+  ]
+}

modules/monitoring/dashboards/wan.json

@@ -0,0 +1,267 @@
+{
+  "uid": "cnx-wan",
+  "title": "CNX WAN Speed",
+  "tags": ["wan", "speedtest", "gateway", "cnx"],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "version": 1,
+  "refresh": "5m",
+  "time": { "from": "now-7d", "to": "now" },
+  "templating": { "list": [] },
+  "annotations": { "list": [] },
+  "panels": [
+    {
+      "type": "row",
+      "title": "WAN speed (site gateways)",
+      "id": 1,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }
+    },
+    {
+      "type": "stat",
+      "title": "Download (latest)",
+      "description": "Most recent hourly librespeed result per gateway.",
+      "id": 2,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 5, "w": 6, "x": 0, "y": 1 },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bps",
+          "color": { "mode": "fixed", "fixedColor": "green" },
+          "noValue": "no data"
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "value",
+        "graphMode": "area",
+        "textMode": "value_and_name",
+        "orientation": "auto"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "speedtest_download_mbps * 1e6",
+          "legendFormat": "{{instance}}",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "type": "stat",
+      "title": "Upload (latest)",
+      "description": "Most recent hourly librespeed result per gateway.",
+      "id": 3,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 5, "w": 6, "x": 6, "y": 1 },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bps",
+          "color": { "mode": "fixed", "fixedColor": "blue" },
+          "noValue": "no data"
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "value",
+        "graphMode": "area",
+        "textMode": "value_and_name",
+        "orientation": "auto"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "speedtest_upload_mbps * 1e6",
+          "legendFormat": "{{instance}}",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "type": "stat",
+      "title": "Ping (latest)",
+      "description": "Latency to the librespeed test server. Yellow over 50 ms, red over 100 ms.",
+      "id": 4,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 5, "w": 6, "x": 12, "y": 1 },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms",
+          "color": { "mode": "thresholds" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 50 },
+              { "color": "red", "value": 100 }
+            ]
+          },
+          "noValue": "no data"
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "value",
+        "graphMode": "none",
+        "textMode": "value_and_name",
+        "orientation": "auto"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "speedtest_ping_ms",
+          "legendFormat": "{{instance}}",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "type": "stat",
+      "title": "Test status",
+      "description": "Whether the last speed test on each gateway completed. FAILING also fires the WANSpeedTestFailing alert after 3h.",
+      "id": 5,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 5, "w": 6, "x": 18, "y": 1 },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "thresholds" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "green", "value": null }]
+          },
+          "noValue": "no data",
+          "mappings": [
+            {
+              "type": "value",
+              "options": {
+                "0": { "text": "FAILING", "color": "red", "index": 0 },
+                "1": { "text": "OK", "color": "green", "index": 1 }
+              }
+            }
+          ]
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "value_and_name",
+        "orientation": "auto"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "speedtest_success",
+          "legendFormat": "{{instance}}",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Download",
+      "description": "Hourly download rate per gateway, with each link's own 7-day median as reference — the WANSpeedDegraded alert fires when a link stays below half its median for 3h.",
+      "id": 6,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 9, "w": 12, "x": 0, "y": 6 },
+      "fieldConfig": {
+        "defaults": { "unit": "bps", "min": 0 },
+        "overrides": [
+          {
+            "matcher": { "id": "byRegexp", "options": ".*7d median.*" },
+            "properties": [
+              {
+                "id": "custom.lineStyle",
+                "value": { "fill": "dash", "dash": [10, 10] }
+              },
+              { "id": "custom.fillOpacity", "value": 0 }
+            ]
+          }
+        ]
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "speedtest_download_mbps * 1e6",
+          "legendFormat": "{{instance}}"
+        },
+        {
+          "refId": "B",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "quantile_over_time(0.5, speedtest_download_mbps[7d]) * 1e6",
+          "legendFormat": "{{instance}} 7d median"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Upload",
+      "description": "Hourly upload rate per gateway.",
+      "id": 7,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 9, "w": 12, "x": 12, "y": 6 },
+      "fieldConfig": {
+        "defaults": { "unit": "bps", "min": 0 },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "speedtest_upload_mbps * 1e6",
+          "legendFormat": "{{instance}}"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Latency & jitter",
+      "description": "Ping and jitter towards the librespeed test server. A latency climb with stable throughput usually means bufferbloat or an ISP routing change.",
+      "id": 8,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 15 },
+      "fieldConfig": {
+        "defaults": { "unit": "ms", "min": 0 },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "speedtest_ping_ms",
+          "legendFormat": "{{instance}} ping"
+        },
+        {
+          "refId": "B",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "speedtest_jitter_ms",
+          "legendFormat": "{{instance}} jitter"
+        }
+      ]
+    }
+  ]
+}

modules/monitoring/exporters.nix

@@ -0,0 +1,121 @@
+# Per-host observability agents, imported by every machine. Host metrics
+# everywhere; Knot DNS metrics on the nameservers; journald shipped to
+# VictoriaLogs on control. Everything is reachable only over the ZeroTier mesh
+# (see the firewall rule at the bottom); the public side is already closed by the
+# Hetzner cloud firewall.
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  mesh = import ../mesh-hosts.nix {
+    dir = config.clan.core.settings.directory;
+    inherit lib;
+  };
+  knotEnabled = config.services.knot.enable;
+  # node_exporter on every host; knot-exporter only where Knot runs.
+  ports = [ 9100 ] ++ lib.optional knotEnabled 9433;
+in
+{
+  # extraInputRules (the mesh-scoped accept below) needs the nftables firewall
+  # backend. allowedTCP/UDPPorts used elsewhere (53, 9993) translate unchanged.
+  networking.nftables.enable = true;
+
+  # Host metrics: CPU, memory, disk, network, systemd unit state.
+  services.prometheus.exporters.node = {
+    enable = true;
+    # Listen on all interfaces (incl. the v6 mesh). We deliberately do NOT bind
+    # to the ZeroTier ULA: the node module renders --web.listen-address without
+    # IPv6 brackets, and binding a single ULA would also race ZeroTier bring-up
+    # at boot. Reachability is constrained by the firewall rule instead.
+    listenAddress = "";
+    port = 9100;
+    enabledCollectors = [ "systemd" ];
+  };
+
+  # Knot DNS metrics. The exporter reads Knot's control socket, so it runs as the
+  # knot user; mod-stats (below) populates the query/response counters it exports.
+  systemd.services.knot-exporter = lib.mkIf knotEnabled {
+    description = "Prometheus exporter for Knot DNS";
+    after = [ "knot.service" ];
+    wants = [ "knot.service" ];
+    wantedBy = [ "multi-user.target" ];
+    serviceConfig = {
+      ExecStart = lib.concatStringsSep " " [
+        "${pkgs.prometheus-knot-exporter}/bin/knot-exporter"
+        "--web-listen-addr ::"
+        "--web-listen-port 9433"
+        "--knot-library-path ${pkgs.knot-dns.out}/lib/libknot.so"
+        "--knot-socket-path /run/knot/knot.sock"
+      ];
+      User = "knot";
+      Group = "knot";
+      Restart = "on-failure";
+      RestartSec = 5;
+      NoNewPrivileges = true;
+      ProtectSystem = "strict";
+      ProtectHome = true;
+      PrivateTmp = true;
+      ProtectKernelTunables = true;
+      ProtectControlGroups = true;
+      RestrictAddressFamilies = [
+        "AF_INET"
+        "AF_INET6"
+        "AF_UNIX"
+      ];
+    };
+  };
+
+  # mod-stats: per-query/zone counters exposed over the control socket and read
+  # by knot-exporter. Loaded as a global module on the default template so it
+  # applies to every zone. (Merges with the zone/acl/policy settings elsewhere.)
+  services.knot.settings = lib.mkIf knotEnabled {
+    "mod-stats" = [
+      {
+        id = "default";
+        "request-protocol" = true;
+        "server-operation" = true;
+        "response-code" = true;
+        "query-type" = true;
+        "reply-nodata" = true;
+      }
+    ];
+    template = [
+      {
+        id = "default";
+        global-module = [ "mod-stats/default" ];
+      }
+    ];
+  };
+
+  # Ship journald to VictoriaLogs on control (services.victorialogs in
+  # server.nix). control uploads to loopback so its own logs survive a mesh
+  # outage; ns1/ns2 push over the mesh to control's ZeroTier address.
+  services.journald.upload = {
+    enable = true;
+    settings.Upload.URL =
+      let
+        dest =
+          if config.networking.hostName == "control" then
+            "127.0.0.1:9428"
+          else
+            "[${mesh.hosts.control}]:9428";
+      in
+      "http://${dest}/insert/journald";
+  };
+
+  # systemd-journal-upload exits if the sink is unreachable. Upstream already
+  # restarts it (Restart=always/RestartSec=3sec), but the default start-rate limit
+  # (5 tries / 10s) lets it give up permanently — so a transient VictoriaLogs
+  # outage leaves the uploader dead until the next deploy. Disable the limit so it
+  # retries forever and self-heals once the sink returns. (A persistent failure
+  # still surfaces loudly in a deploy, which is what we want.)
+  systemd.services.systemd-journal-upload.startLimitIntervalSec = 0;
+
+  # Scrape ports reachable only from the ZeroTier mesh.
+  networking.firewall.extraInputRules = ''
+    ip6 saddr ${mesh.subnet} tcp dport { ${lib.concatMapStringsSep ", " toString ports} } accept
+  '';
+}

modules/monitoring/parsedmarc.nix

@@ -0,0 +1,125 @@
+# DMARC report analyzer, imported by control only. parsedmarc fetches the
+# aggregate/forensic reports that land in the dmarc@cnx.email mailbox on mx1,
+# parses the XML, and stores results in a local Elasticsearch; the official
+# parsedmarc dashboard + an Elasticsearch datasource are auto-provisioned into
+# the Grafana instance that server.nix already runs on this host.
+#
+# IMAP runs over the ZeroTier mesh, not the public net: we pin mx1.cnx.email to
+# its mesh address in /etc/hosts so TLS still validates against the public
+# Let's Encrypt cert (primary domain mx1.cnx.email) while the bytes stay on the
+# overlay. The mailbox passphrase is the shared mail-dmarc-cred secret; parsedmarc
+# reads it as root in its ExecStartPre, so root-owned (clan default) is fine.
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  mesh = import ../mesh-hosts.nix {
+    dir = config.clan.core.settings.directory;
+    inherit lib;
+  };
+
+  # Grafana 13 (since the externalisation of core data sources) no longer bundles
+  # the Elasticsearch data source, and nixpkgs has no grafanaPlugins entry for it,
+  # so a bare elasticsearch datasource reports "Plugin not registered" and every
+  # DMARC panel fails. Pull the official, Grafana-signed plugin from the catalog
+  # and load it declaratively.
+  elasticsearchPlugin = pkgs.grafanaPlugins.grafanaPlugin {
+    pname = "elasticsearch";
+    version = "12.6.4";
+    zipHash.x86_64-linux = "sha256-xyAUprdWyQM0IJSg/oBVZ0ltFAAffgPrphln9+IKcUY=";
+  };
+in
+{
+  imports = [ ../mail-dmarc-cred.nix ];
+
+  # Elasticsearch 7.x is under the (unfree) Elastic License; allow just this one
+  # package rather than opening allowUnfree globally.
+  nixpkgs.config.allowUnfreePredicate = pkg: lib.getName pkg == "elasticsearch";
+
+  services.grafana.declarativePlugins = [ elasticsearchPlugin ];
+
+  # Keep mx1's IMAP traffic on the mesh while presenting the public cert name.
+  networking.hosts.${mesh.hosts.mx1} = [ "mx1.cnx.email" ];
+
+  services.parsedmarc = {
+    enable = true;
+    provision = {
+      # Local Elasticsearch on 127.0.0.1:9200 (loopback; no firewall change).
+      # datasource + dashboard default to true once ES and Grafana are both on.
+      elasticsearch = true;
+      # GeoIP needs a MaxMind account/license key; skip it (reports still parse,
+      # just without source-IP geolocation).
+      geoIp = false;
+      grafana = {
+        # We provision the two Elasticsearch datasources ourselves (below) so we
+        # can pin their UIDs. Leaving this on would add a second, uid-less pair
+        # and collide on the dmarc-ag/dmarc-fo names.
+        datasource = false;
+        dashboard = true;
+      };
+    };
+    settings = {
+      imap = {
+        host = "mx1.cnx.email";
+        port = 993;
+        ssl = true;
+        user = "dmarc@cnx.email";
+        password = {
+          _secret = config.clan.core.vars.generators.mail-dmarc-cred.files."passphrase".path;
+        };
+      };
+      mailbox = {
+        watch = true; # IMAP IDLE: process reports as they arrive
+        delete = false; # archive processed reports, don't delete
+      };
+      general = {
+        save_aggregate = true;
+        save_forensic = true;
+      };
+    };
+  };
+
+  # The bundled parsedmarc dashboard selects its datasource through template
+  # variables ($datasourceag/$datasourcefo) whose stored value is the string
+  # "dmarc-ag"/"dmarc-fo". Grafana 11+ resolves a datasource template variable
+  # by UID, not by name, so without a matching UID every panel renders
+  # "datasource was not found". Pin the UIDs to those names so it resolves.
+  services.grafana.provision.datasources.settings = {
+    # These two were previously provisioned uid-less, so they exist in Grafana's
+    # DB with random UIDs. Provisioning a *new* UID onto an existing datasource
+    # makes Grafana abort with "data source not found" and crash-loop, so delete
+    # the stale records first; the entries below re-create them with pinned UIDs.
+    deleteDatasources = [
+      {
+        name = "dmarc-ag";
+        orgId = 1;
+      }
+      {
+        name = "dmarc-fo";
+        orgId = 1;
+      }
+    ];
+    datasources =
+      let
+        esVersion = lib.getVersion config.services.elasticsearch.package;
+        es = name: {
+          inherit name;
+          uid = name;
+          type = "elasticsearch";
+          access = "proxy";
+          url = "http://localhost:9200";
+          jsonData = {
+            timeField = "date_range";
+            inherit esVersion;
+          };
+        };
+      in
+      [
+        (es "dmarc-ag")
+        (es "dmarc-fo")
+      ];
+  };
+}

modules/monitoring/server.nix

@@ -0,0 +1,184 @@
+# Monitoring server, imported by control only: VictoriaMetrics (TSDB + scraper)
+# and Grafana. VictoriaMetrics binds loopback (only Grafana, on the same host,
+# reads it). Grafana is reachable over the ZeroTier mesh, scoped by the firewall
+# rule at the bottom; the Hetzner cloud firewall keeps it off the public net.
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  mesh = import ../mesh-hosts.nix {
+    dir = config.clan.core.settings.directory;
+    inherit lib;
+  };
+  probes = import ./blackbox-probes.nix { inherit lib; };
+  vmPort = 8428;
+  logsPort = 9428;
+  grafanaPort = 3000;
+  controlV6 = mesh.hosts.control;
+
+  # A single scrape target with a friendly instance label. IPv6 mesh addresses
+  # must be bracketed for Prometheus-style targets.
+  target = name: addr: port: {
+    targets = [ "${addr}:${toString port}" ];
+    labels.instance = name;
+  };
+  v6 = addr: "[${addr}]";
+
+  adminPasswordFile = config.clan.core.vars.generators.grafana-admin.files."password".path;
+  secretKeyFile = config.clan.core.vars.generators.grafana-secret-key.files."secret-key".path;
+in
+{
+  services.victoriametrics = {
+    enable = true;
+    listenAddress = "127.0.0.1:${toString vmPort}";
+    retentionPeriod = "180d";
+    # The scraper dials IPv4-only by default; our ns1/ns2 targets are mesh ULAs,
+    # so without this VM drops them with "no suitable address found (try -enableTCP6)".
+    extraOptions = [ "-enableTCP6" ];
+    prometheusConfig = {
+      global.scrape_interval = "30s";
+      scrape_configs = [
+        {
+          job_name = "node";
+          static_configs = [
+            # control scrapes its own node_exporter over loopback so host metrics
+            # survive even if the mesh is down; ns1/ns2 are scraped over the mesh.
+            (target "control" "127.0.0.1" 9100)
+            (target "ns1" (v6 mesh.hosts.ns1) 9100)
+            (target "ns2" (v6 mesh.hosts.ns2) 9100)
+            (target "mx1" (v6 mesh.hosts.mx1) 9100)
+            (target "web01" (v6 mesh.hosts.web01) 9100)
+            (target "gw-cnx-1" (v6 mesh.hosts.gw-cnx-1) 9100)
+          ];
+        }
+        {
+          job_name = "blocky";
+          static_configs = [
+            (target "gw-cnx-1" (v6 mesh.hosts.gw-cnx-1) 4000)
+          ];
+        }
+        {
+          job_name = "knot";
+          static_configs = [
+            (target "ns1" (v6 mesh.hosts.ns1) 9433)
+            (target "ns2" (v6 mesh.hosts.ns2) 9433)
+          ];
+        }
+      ]
+      # Outside-in DNS probes via the blackbox exporter (blackbox.nix). The job
+      # list is generated from the same probe definitions the exporter uses.
+      ++ probes.scrapeConfigs;
+    };
+  };
+
+  # Centralized logs: VictoriaLogs ingests journald from all three hosts, each
+  # of which runs systemd-journal-upload against /insert/journald (exporters.nix).
+  # Binds all interfaces because ns1/ns2 push over the mesh; the firewall rule at
+  # the bottom scopes 9428 to the mesh subnet and the Hetzner firewall closes the
+  # public side. Retention is set via extraOptions (no dedicated NixOS option).
+  services.victorialogs = {
+    enable = true;
+    listenAddress = ":${toString logsPort}";
+    # -enableTCP6: like the scraper above, VictoriaLogs is IPv4-only by default
+    # for *listening* too — ":9428" binds 0.0.0.0 only, so ns1/ns2 pushing over
+    # the IPv6 mesh get "connection refused". This makes it bind [::] (dual-stack)
+    # so the mesh can reach it. Retention has no dedicated NixOS option.
+    extraOptions = [
+      "-retentionPeriod=30d"
+      "-enableTCP6"
+    ];
+  };
+
+  # Admin password generated once and stored as a clan secret. Retrieve with:
+  #   clan vars get control grafana-admin/password
+  clan.core.vars.generators.grafana-admin = {
+    files."password" = {
+      secret = true;
+      owner = "grafana";
+      group = "grafana";
+    };
+    runtimeInputs = [ pkgs.openssl ];
+    script = ''
+      openssl rand -base64 24 | tr -d "\n" > "$out"/password
+    '';
+  };
+
+  # NixOS 26.05 removed the built-in default for services.grafana's secret_key
+  # (used to sign/encrypt DB secrets). Mint our own instead of hardcoding the old
+  # upstream default. Safe to generate fresh here: the provisioned datasources
+  # carry no credentials, so nothing in Grafana's DB is encrypted with it.
+  clan.core.vars.generators.grafana-secret-key = {
+    files."secret-key" = {
+      secret = true;
+      owner = "grafana";
+      group = "grafana";
+    };
+    runtimeInputs = [ pkgs.openssl ];
+    script = ''
+      openssl rand -base64 32 | tr -d "\n" > "$out"/secret-key
+    '';
+  };
+
+  services.grafana = {
+    enable = true;
+    # VictoriaLogs datasource plugin so journald is greppable from Grafana,
+    # alongside the metrics datasource.
+    declarativePlugins = [ pkgs.grafanaPlugins.victoriametrics-logs-datasource ];
+    settings = {
+      server = {
+        http_addr = "::";
+        http_port = grafanaPort;
+        root_url = "http://${v6 controlV6}:${toString grafanaPort}/";
+      };
+      security = {
+        admin_user = "admin";
+        admin_password = "$__file{${adminPasswordFile}}";
+        secret_key = "$__file{${secretKeyFile}}";
+      };
+      "auth.anonymous".enabled = false;
+      users.allow_sign_up = false;
+    };
+    provision = {
+      enable = true;
+      datasources.settings = {
+        apiVersion = 1;
+        datasources = [
+          {
+            name = "VictoriaMetrics";
+            type = "prometheus";
+            uid = "victoriametrics";
+            access = "proxy";
+            url = "http://127.0.0.1:${toString vmPort}";
+            isDefault = true;
+          }
+          {
+            name = "VictoriaLogs";
+            type = "victoriametrics-logs-datasource";
+            uid = "victorialogs";
+            access = "proxy";
+            url = "http://127.0.0.1:${toString logsPort}";
+          }
+        ];
+      };
+      dashboards.settings = {
+        apiVersion = 1;
+        providers = [
+          {
+            name = "cnx";
+            options.path = ./dashboards;
+            options.foldersFromFilesStructure = false;
+          }
+        ];
+      };
+    };
+  };
+
+  # Grafana (admin laptops + servers) and VictoriaLogs ingestion (ns1/ns2 push
+  # journald over the mesh) reachable only from the ZeroTier mesh.
+  networking.firewall.extraInputRules = ''
+    ip6 saddr ${mesh.subnet} tcp dport { ${toString grafanaPort}, ${toString logsPort} } accept
+  '';
+}

modules/static-ipv6.nix

@@ -0,0 +1,48 @@
+# Static public IPv6 for hosts where SLAAC/RA doesn't bring up a global
+# address (e.g. Hetzner, which allocates a /64 but expects you to pick one
+# address yourself and route via the link-local gateway fe80::1).
+{ lib, config, ... }:
+let
+  cfg = config.cnx.staticIPv6;
+in
+{
+  options.cnx.staticIPv6 = {
+    enable = lib.mkEnableOption "static public IPv6 on the primary interface";
+
+    address = lib.mkOption {
+      type = lib.types.str;
+      example = "2a01:4f8:c014:b5c5::1";
+      description = "Public IPv6 address (no prefix length).";
+    };
+
+    prefixLength = lib.mkOption {
+      type = lib.types.int;
+      default = 64;
+      description = "Prefix length of the allocated block.";
+    };
+
+    interface = lib.mkOption {
+      type = lib.types.str;
+      default = "enp1s0";
+      description = "Interface to assign the address to.";
+    };
+
+    gateway = lib.mkOption {
+      type = lib.types.str;
+      default = "fe80::1";
+      description = "IPv6 default gateway (Hetzner uses the link-local fe80::1).";
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    networking.interfaces.${cfg.interface}.ipv6.addresses = [
+      {
+        inherit (cfg) address prefixLength;
+      }
+    ];
+    networking.defaultGateway6 = {
+      address = cfg.gateway;
+      interface = cfg.interface;
+    };
+  };
+}

modules/web-proxy.nix

@@ -0,0 +1,76 @@
+# Public reverse proxy with TLS termination for web01. Caddy fronts internal
+# services and forwards to them over the ZeroTier mesh, never the public net.
+# The cert is a single wildcard (*.cnx.network) obtained via ACME DNS-01, so
+# adding a vhost needs no new issuance. Public ports: 443 for the proxy and 80
+# only for Caddy's HTTP->HTTPS redirect (issuance never uses inbound HTTP).
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  mesh = import ./mesh-hosts.nix {
+    dir = config.clan.core.settings.directory;
+    inherit lib;
+  };
+  hosts = import ./hosts.nix;
+  certName = "cnx.network";
+in
+{
+  imports = [ ./dns/acme-web01-secret.nix ];
+
+  # Render the shared acme_web01 TSIG secret into a lego rfc2136 env file. lego
+  # (via security.acme below) uses it to write _acme-challenge.cnx.network TXT
+  # records on ns1, which authorizes the acme_web01 key for exactly that owner.
+  clan.core.vars.generators.dns-acme-web01-rfc2136 = {
+    files."rfc2136.env".secret = true; # root-owned; systemd reads it as root
+    dependencies = [ "dns-acme-web01-secret" ];
+    script = ''
+      printf 'RFC2136_NAMESERVER=${hosts.ns1.ipv4}:53\nRFC2136_TSIG_ALGORITHM=hmac-sha256.\nRFC2136_TSIG_KEY=acme_web01\nRFC2136_TSIG_SECRET=%s\n' \
+        "$(cat "$in"/dns-acme-web01-secret/secret)" > "$out"/rfc2136.env
+    '';
+  };
+
+  security.acme = {
+    acceptTerms = true;
+    defaults.email = "postmaster@cnx.email";
+    # One wildcard cert for every vhost this proxy serves, via DNS-01 (so issuance
+    # never depends on inbound HTTP). Port 80 is open only for Caddy's
+    # HTTP->HTTPS redirect, not for ACME.
+    certs.${certName} = {
+      domain = "*.cnx.network";
+      extraDomainNames = [ "cnx.network" ];
+      dnsProvider = "rfc2136";
+      environmentFile = config.clan.core.vars.generators.dns-acme-web01-rfc2136.files."rfc2136.env".path;
+      # ns1 is the only nameserver that accepts the acme_web01 UPDATE; check
+      # propagation against it directly rather than a public resolver.
+      dnsResolver = "${hosts.ns1.ipv4}:53";
+      # Caddy reads the cert from explicit file paths (tls directive below), so it
+      # won't notice a renewal on its own — reload it whenever the cert changes.
+      reloadServices = [ "caddy.service" ];
+    };
+  };
+
+  # The lego-issued cert is owned group=acme; Caddy needs to read the key.
+  users.users.caddy.extraGroups = [ "acme" ];
+
+  # Reverse proxy. The explicit `tls cert key` points Caddy at the wildcard cert
+  # and disables its automatic ACME, so no extra issuance happens. Backends are
+  # dialed over the mesh by their ZeroTier address (mesh.hosts.<name>).
+  services.caddy = {
+    enable = true;
+    virtualHosts."grafana.cnx.network".extraConfig = ''
+      tls /var/lib/acme/${certName}/cert.pem /var/lib/acme/${certName}/key.pem
+      reverse_proxy http://[${mesh.hosts.control}]:3000
+    '';
+  };
+
+  # 443 serves the proxy; 80 only carries Caddy's automatic HTTP->HTTPS redirect
+  # (the Hetzner cloud firewall also scopes these in
+  # modules/hetzner-firewall-rules.nix). Admin still rides the mesh.
+  networking.firewall.allowedTCPPorts = [
+    80
+    443
+  ];
+}

sops/machines/gw-cnx-1/key.json

@@ -0,0 +1,6 @@
+[
+  {
+    "publickey": "age1g8a6fqhqg8q8qu4jyuhxc7hf00t5qvra55chyfm0qk45wlmvc9qs2cft8t",
+    "type": "age"
+  }
+]

sops/machines/mx1/key.json

@@ -0,0 +1,6 @@
+[
+  {
+    "publickey": "age1l5hw95p5h4sthrgn0usms9yfkwwmcvv34tjgrtv9s4e6x39chacshgxavs",
+    "type": "age"
+  }
+]

sops/machines/web01/key.json

@@ -0,0 +1,6 @@
+[
+  {
+    "publickey": "age1yey6gxgsyl4tj6ek0tve2pckt6qersqspk66ukkzum8mrr6zppqsj4jn3m",
+    "type": "age"
+  }
+]

sops/secrets/control-age.key/groups/admins

@@ -0,0 +1 @@
+../../../groups/admins

sops/secrets/control-age.key/secret

@@ -3,12 +3,17 @@
 	"sops": {
 		"age": [
 			{
-				"recipient": "age1yubikey1qtc8v5yqy3g5sh4trwwdp7elmavvkvkvzc4tfdnv2g8wyd8y5lc064mpv34",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IEZXQ2lTZyBBMXZ4QWtr\nQVRaeHJnSlNyMmdMOVN6MmRMTm9Fc3FEbURsY1BLamhqS0J0SAphb1orQjJtNEwy\ncE16ejVRS0c3RXdjdzdUMUh0bFNTTGs5YW5WdS8xQ3FRCi0tLSBHWnlOOGZVVXMx\nQnBKMXd6MktTOTU3TmpRSEgrTGFuM2o4NGVlZlVmZjRjCsBsIrVfs218rztmmAtJ\n0iFU7ZcoUsEixjUogQ7xuoBppiRcY3jWz5CtlvpaMheXRv1DqrHrwK2i1kLHwhSO\n/s0=\n-----END AGE ENCRYPTED FILE-----\n"
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBNRWlKNFVSOThSWlI3LzAz\ndnQxbnNGSlhoVmZrQmpIVEJieStlUzlseDJJCmdGbldCSU1oTGszZmtzSWhJaWta\nVDhoQU8vVXB0bG1VUkxoWDlyR0I1a0EKLS0tIDFzdklCTTBjeXBoT3FqVW1Pbkxl\nYVZ5Vm55OFp4YWwzS3dVNXRoZmpBT1EK0pNbk20gwPe5Q1bZR3XRHMWz+x+8FzRa\niq5tLcGkgfMO6WTzj8vtB7g0Q3sFgmPXTqb8yMzqAkZd4z9FKJGt6A==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBcFBHaFpS\nbjJiL0ZnOXlzMVVNdjhvUGxmaEJhTklSR2ZaTnhPWXFCaC8vbgpLdE04aW16dEls\neGw3VllEYVAzbGJ5TVZoOXhkR3dCUkREdUkxMU13clVzCi0tLSA3eW93QnFsNW1r\nK0ZiWEtrWFVqeTJyTk1maGVTMzdBTkJUdk5WN2N3WlprCp5oak+Bh/g4+ikE1LR5\n7kYrWQlzjC0Z+F5GegEtRjGCPnolWllnPUH4dGGjfuyAxy9qfi1J6Bi6m8uzo5rY\n+zs=\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa"
 			}
 		],
 		"lastmodified": "2026-06-14T05:20:21Z",
 		"mac": "ENC[AES256_GCM,data:dnvYlFK81wOynOkj0rfiOpGSjoxMMFs5BIlOc7tt3tnpxRRPu5IsHVIyRM6/spU3Ajwv9WMgbp+RncCorlkFUNZxZ2Xc11UGsDDCW7ddV5CHLNdXGpeAbHnIjehJTJehjitbt/Fan2ac9bnzQlIV1bjydhJ0G1A9cDWMQs6g5VU=,iv:D/YRqIFpMHQ0zJXA/92l3VHgUbhzOW7D4oHJQi0+B/Q=,tag:GjcLVcZ+gyQ9v4yaZeHnTg==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
 		"version": "3.12.1"
 	}
 }

sops/secrets/gw-cnx-1-age.key/groups/admins

@@ -0,0 +1 @@
+../../../groups/admins

sops/secrets/gw-cnx-1-age.key/secret

@@ -0,0 +1,19 @@
+{
+	"data": "ENC[AES256_GCM,data:bsb98446kjvnDAVWNIqvH64uD6XyfEpohPFZs38jDaxxTp1wqJmFdeDW+p3SGld8Q+4h6rq7rFtTUJnQyslvgsPfYN5tBhJuTbI=,iv:pPUf3RgHqS8LlRIBdZ7jmzxp5VmlP68nrX8TCVP8aGU=,tag:SseZa3Sf5LuMXWN8CNSbxw==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBLRkppdUoza2l0SnFpME5n\nM0xBRVk4c2lhaHMvazFNaGNTdkJDcnJEQkVrCjhFUlZMUitnZ20zQlFEcm5WQmVG\nWmJVeUh6S09MS0xpQUR4aDF1V1dCZFkKLS0tICtOMEJ6QmRRR3BkaU5sZHdzOElO\nSnV4K0dHMHhDOURPTEpucXFuNEUwaU0Kx4xAaKTrw1bKJPQVCHcByTcapFhslwrV\nqssEBnYWXqFgI3vh3CatBXuZY1Lm3eTKhDDQ2ZyPKgc/l0gXyFwH2g==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBMHFKQTNG\nazdEcThReFZGVVBtVFBDY1luQk1UTkl0a0p1bzQ0cUxBZWVyYwo4aU5qaEs0czJB\nUzZFcG9uTUZ1OUFScW9ZeHljbDBEbjkzZ08vS2F1Qjd3Ci0tLSBHTXhVWjNsWHhK\nbWtrMWw3Nm8wSU9MNXZwcWxVZXRJS1dTK0hycnFoVlRJChemDXVb7LgM7OeUUeqa\nTl0IuDHmIIx/lUYnJWGZmWQHz2znk679j36VQQNWcK6eZlKUIqYmBj3fMu+OYir2\nMLI=\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa"
+			}
+		],
+		"lastmodified": "2026-07-28T09:03:13Z",
+		"mac": "ENC[AES256_GCM,data:MKIyykYwtQqpUnXqZQKzKwoRhSon2PH0wYiTDTvogaVREHfpEPs/mdOjQBUHtbr1QGqWNh4AxdBTKfpjXTlncHAEtcIdVH3qPccVxsXqfqRmZPDFzcHZAIw+VSvDsKwD3anixiXDeHmeLawwbyb1rnS0P8KquXbmjvirwGUIQLU=,iv:+hdLXMJMCby/6Q6JGxp4p95GgPKbdA/OTAZloyylr08=,tag:ZZ4dKjhjpE2xqcoNf4qeoA==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.13.1"
+	}
+}

sops/secrets/gw-cnx-1-age.key/users/berwn

@@ -0,0 +1 @@
+../../../users/berwn

sops/secrets/mx1-age.key/groups/admins

@@ -0,0 +1 @@
+../../../groups/admins

sops/secrets/mx1-age.key/secret

@@ -0,0 +1,18 @@
+{
+	"data": "ENC[AES256_GCM,data:lAgbSvyxBl0/NG8rHweKEEqYyDsHa+SrPZnxVubB2x5H3cqydhTCs8NMxJp+RKTPW2nhfmB/lrKgLHFfFMJfMg5jZ2BvZtR1IzA=,iv:1PgMbra/ec7yiCm7K5yo+1lCLJ89ryfP3SQML7uYv4k=,tag:lmaG+dZPCLHtQ59uFeSkVg==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBELzcrdUc0cW9pSXhGbzRY\naTd5aWgzcFpLQ3MyVElaZU5JK2Z4ZGRtMXpJCk14bmZLRlY4Z3pDeUxPM203anVY\nbE1aUEdDZ3NkcGxrcCtWS1ZmRWIrcmMKLS0tIDVTOUJhenR0WVhZSWNWT2VtRFpv\nc1h6Nk95Z3Y4eFQ1NUdmWWJiaWdDdGMKB3whh/RgAePTJnGmeDJ/WFv4NI42vA5O\nB0F6jmSDNa5beP8Um2DjWdPENkJJjv9yv38b7hP8BLDe9Ba4WBNfBA==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBby9pdmFZ\nOElJVCtUQmJWVVljQi9KcHE2a2FaeU5HeUhyUm05Zm0wZDZKVwptY2hHRFJKMi9I\nN3NDY3ZjYVloTjF6R0l4RllDS1dpNzA1TDJDc0hGQnFvCi0tLSB1akhUWk9RU0Rt\naDV5MHdHcExlSzZWbSs0S1A3NGhES0V1U0pEaUd5WE04ChtGuEq0HnRiVTDwhJnO\nIWMhwCYaewHk+k0a0Z9qCwqqKxhfiGS6kg/YTKHTNhQ0bxIA8yqgQoaE8Hl5Zhzs\nKqk=\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2026-06-18T04:33:03Z",
+		"mac": "ENC[AES256_GCM,data:Iffy/eS80qOJLdaGOIxti1otdxSLHA1TV06R8xq4zI1qFklJX8OWRy9CAYwAcTURS88gd5c2VGHsBI8yHOBj7LKiLJiu5/xRgD106hotBOJUy90UZ6MMbM2wwHomhQg5r5kNpRsJaPFaLFzm5YJN9gLUU1kxO4Nnt2L3bxZTogQ=,iv:cqbFD3ZIJnaZU7j7l9Qt1QqiUSQtclV7MGZWEZRZQoY=,tag:F0ZZQSQKYtBI65uutjNxow==,type:str]",
+		"version": "3.12.1"
+	}
+}

sops/secrets/mx1-age.key/users/berwn

@@ -0,0 +1 @@
+../../../users/berwn

sops/secrets/ns1-age.key/groups/admins

@@ -0,0 +1 @@
+../../../groups/admins

sops/secrets/ns1-age.key/secret

@@ -3,12 +3,17 @@
 	"sops": {
 		"age": [
 			{
-				"recipient": "age1yubikey1qtc8v5yqy3g5sh4trwwdp7elmavvkvkvzc4tfdnv2g8wyd8y5lc064mpv34",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IEZXQ2lTZyBBOW5zQ01V\nTUsrL2FvVVdvOGwvLzI4TGpqSmdRM2pBWndjMUd6K1BlVFY2SAorT2lTZHh0OS8y\nR0hqRER2Z00vYVFRS0lHaG5CL1BjSEw1MVdrYmFZU3VZCi0tLSBvRjlQendYR1V3\nSjdjcVRIbWIyZ05wNURXQjRnelAzMm9hTkc1Tm0yK0IwClY/Iz/DcGOu5pqq44gH\nx3CP0XY2db0PMkY0jlk2v9Mp32sshiBpmaMuFqz28JH5gwTbI2r0nXwteu4GM0EH\nN5Q=\n-----END AGE ENCRYPTED FILE-----\n"
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBcU01TXl0\nRmU5ZlZOK3NZZE8vNktOWXBkRnZtZHlUbnBHYlVQbDdzZlRDbApFeFI0OE1KdndQ\nZWczZGRYZXBONTc2aG4vZTJEZ1hCdGljNFJBcmF0UGpRCi0tLSBGMGJVWUtJMWFP\nQUZwWVZTWUFKZnVJUTJSMHlRMGRNcExPSEpXTFpLSkVNCj1WEuRyATcxp0GYeoqC\nACpQpcqgIBVD1dYB0pPo+I6yDXvx+JbwVdgGl+2Cxj40HQOfyvJecoOO3efOZrB3\nUZU=\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBRb0ZvUmN2RWhtQkpXMXJr\nY1o2MXZsQ0dXNFYxa2RLUGlJWnN3Z25zbnp3CmtLbkYvdHR2MUZIY2U1MzhyU1VD\ndm5MTmMveEJLOHZWWnVWdVpDcTFsTmMKLS0tIFEwYzZYeU5uc29BTEVxN2YvN0Zw\nd0VSeTNhQk1pNVdaZVdTVk1oNjFwQWcKwEbHyT354Cbam9wuiIhapre+nbqtMr9h\nbOjF53LR9RZ9/mbpTNDLOkq2QW2aM4tKkRfdUgP1XgboilzcTjba8A==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06"
 			}
 		],
 		"lastmodified": "2026-06-14T05:20:24Z",
 		"mac": "ENC[AES256_GCM,data:+9s1F7y7mE5LCmIT23W2gCwRLagSkxu1wvNqyxt9fGG4Uxj1LOspq9I+spNdiaRo5EfYowNbQLsuq/LD8VMy9RCz9/YZVG41Hskv1MGz2TrNZ9IeXS+VYIGCkxeWNERvKDDy+o6cPcU39JR59lqR0OE1pdEl3tOsLE+9XDNRBHg=,iv:Hsh/iaObBUjTBQiGNubA/Z/b4guQFhE9wln0yLAT3nA=,tag:E7mjMixGNipsdNin2dQAcQ==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
 		"version": "3.12.1"
 	}
 }

sops/secrets/ns2-age.key/groups/admins

@@ -0,0 +1 @@
+../../../groups/admins

sops/secrets/ns2-age.key/secret

@@ -3,12 +3,17 @@
 	"sops": {
 		"age": [
 			{
-				"recipient": "age1yubikey1qtc8v5yqy3g5sh4trwwdp7elmavvkvkvzc4tfdnv2g8wyd8y5lc064mpv34",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IEZXQ2lTZyBBb2U1Mnc4\nOWxRcngvREJYSXdNdjJpTHZnQkdqZks2RWtUODREaURhdzFubgprTDd0cmxRYXdn\nTHNYblpUcHpBQmkyR0xGTm12VTNTR0lZWlBRMjQ3MlRBCi0tLSBhZ0VHcUVCYzAw\nSkc1MkhKRGNtVm96SlhQQXZwblV0YjdJUzE1VVEyMEQ0CmkQWSOEJOFrDLFagXyd\n2cIneWoyVtUtHZqFkuc0me4h0aA4PZ/JZKA9JnA2zbl34CRhzW8lmwchLJCFWarU\n+V0=\n-----END AGE ENCRYPTED FILE-----\n"
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBaTVGQWRC\nZUJocGx5Zjd0c1JpUEdHbGczNUh1RjRqSERRNHNSbnRndWFUMAoxbFBEU0Ezajhr\nSUlIRStWUUdVTDJMQWxmSkw0c0FNYnFxNEJIdnExTmx3Ci0tLSBMRGRPLzlXTEN3\ncHZCMGF3WDZTcFhTd2srL3Vpc2dRdExrVzFJcWhjQVJrCrA1y2SapkZoppI3z4LD\nH71D/WxIcTHp3fIzICNvgxWHn3+vkDDZcpcgH+gRiPCDqrgDp0NZcZiDlz6zyHGr\nmaM=\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB0Rmp1S29hVE5LNkZpLzRV\nMGZJL242YmdHNkFXR2d0TmlVWXViTlJFNGpnClBSdEhaRlNWbVNrb05tTHh0Q3Rk\naVFNdUd1clNVckREajVoRmZCenFEaVkKLS0tIExOaHd1WWxZRFM1UFk4K0E0cWpE\nK2duUE1KeEtmeVorRnc3MFdDZXFGZjAKKw4ykWu+HzBN3WAj/gx0mMYk0RiyRR50\nYv7hg1B/+x4lDCTjAJIWrrBEjLJBZTZRnECmTMTlJ39uOhxeUOizCA==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06"
 			}
 		],
 		"lastmodified": "2026-06-14T05:20:27Z",
 		"mac": "ENC[AES256_GCM,data:0EX6VnkasTfYUvJuu7ojg8+T+XjMvMmHtaS8iqzDADaLi0W1/PfwxoAFduSjNVWpZI/Qc+mBmkRSqqVOWsOesyQhDbPRGh6TKBar0IYRb63g/trxri5niAHeMxiMEv/XBYkmRWSViThKz+TgFBEKrnok0AddppxgQ7dlXQhGETc=,iv:hxcmEWg0pmVmnHMzmrjwZcokFBE6WQudkFrtw57GyU8=,tag:d0RhkrwjMtNi3iexA/sYBA==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
 		"version": "3.12.1"
 	}
 }

sops/secrets/web01-age.key/groups/admins

@@ -0,0 +1 @@
+../../../groups/admins

sops/secrets/web01-age.key/secret

@@ -0,0 +1,18 @@
+{
+	"data": "ENC[AES256_GCM,data:u+O5nWbFlvp6SGyHJggfkCLCT0ZuDy89e/VGGQNdt3yYzgdNmnrtd+2q+Ft3MtoOSSCLvStriGQzfLhcqEgqGgt3PqfIzCO1IG4=,iv:GyrZ1XUiOZe1I1Z/HebTy2NM2tfDHxIH5zGVk7HD+xQ=,tag:js6fbXWajVZSxt0hmnnA5g==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBWenArV1pZMURzNElTeGlC\nRSttUFRwM2E5d2htVDlFcTkwVDIrV2JxQjE0CmE0NUZ4UERHdHpqYk9uK01GOTFQ\ndW5UNlRrY0hvU2pNQmpSanh5RG5YbVkKLS0tIFdYQmkva1NORG80U3dDSWszZTlO\nOXViWUxMTVR1NE00cjdpVXpVS1J5YU0KHBkeKAJZDc+R1GLKwDYLyQBlEW7tPnMh\nf3tsUvtD0flqPAXNeDgyOmKufP7U6oDy/OriFC9+zYQbWyEEc6CZHg==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBL05xT1BJ\nSldCSkFhaFErWDNMeXMvRksvekkwNUZLZFVtSmUydVRrNG9JTwpFaHNsVThZN2gw\nMlNxUXhnN2xYNFluK1hadGxzMFZuaWR3cVFRYW9mWndVCi0tLSBUUmRMSnJCeTRM\nUVdKY3hzWVRkQkFuQ3FaRDVZSEp1b2N1ajg5RnlhT3VRCjX/vWj0We88ATiz808w\nz60RL0BvDGJ6m1BNmqAdtfCCClH33YXQBGrKT2E5elvOTl0iOCrT7HPjzXxJZXuw\nkKg=\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2026-06-20T19:26:29Z",
+		"mac": "ENC[AES256_GCM,data:uD+L3op6ZPtvmLreJ7S4GE6bQuItV87w1LTMFvjI1Kb5+Z0sXlL0TpYO8WLx8X0yaL3HddwlmBKYQGp/OlRPqZMFDbQuK25oeKB54jfm6YDn66rrMQJl1FOw68fLJaHYNjNelEg/bj2WG7YpfZBoWO67MW6F+44Rg4XF85w/1x0=,iv:X9TmEL5JkcC9waLumWpgpBwp3YWLMslZi++dv2HZ0mk=,tag:i7c1CxT1Xv/+T/jY9E0cdg==,type:str]",
+		"version": "3.12.1"
+	}
+}

sops/secrets/web01-age.key/users/berwn

@@ -0,0 +1 @@
+../../../users/berwn

sops/users/berwn/key.json

@@ -1,6 +1,6 @@
 [
   {
-    "publickey": "age1yubikey1qtc8v5yqy3g5sh4trwwdp7elmavvkvkvzc4tfdnv2g8wyd8y5lc064mpv34",
+    "publickey": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa",
     "type": "age"
   }
 ]

vars/per-machine/control/emergency-access/password/groups/admins

@@ -0,0 +1 @@
+../../../../../../sops/groups/admins

vars/per-machine/control/emergency-access/password/secret

@@ -3,12 +3,17 @@
 	"sops": {
 		"age": [
 			{
-				"recipient": "age1yubikey1qtc8v5yqy3g5sh4trwwdp7elmavvkvkvzc4tfdnv2g8wyd8y5lc064mpv34",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IEZXQ2lTZyBBclYvY0RS\nMFpHOEpQU0kreWVlTmhqRkxzZGp4UnEvdGcybFRKSE4yYnZXNQpSSGlOaWRWcHFn\nNTk4S2J0dVdvSXk5RzJEaVZJY1VYa1hMcWdleGpRdFdRCi0tLSB0TUJoK3UxdVll\nWUxnaUFqUEExVFE2VllLanJSM2hJSHpxTlpFUVZqRXJJCuAtDqWR4cbc3snmpF/E\nXZn5hBsdyXaxRR1t5ZUCrKBl3zUBq6h/o+tUR105q3QWdm8WY2itIbc8ZEZ7hqi8\nCtc=\n-----END AGE ENCRYPTED FILE-----\n"
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBcnhlSmZv\neVlFR2JWNjR4UWpTeUY0ZnRtR0IyU2tHc2I3TXFnelpyRGR2QwpRL285aTgvNExj\ncWFyV1ZLbGZoOGVMUERldWYrbjNSUjRkTU1oZ20rTU9FCi0tLSBqdFJlNEJmMVYx\nbGYwWDZ0UlI2WEg2ekdDckRRK1hFNlM1STVIZmtzQ2JNCidXOYx35lyp24WGXydR\nEUHxxlCZuOhUZ6fjKY6p2/mKNQPqt41hq+tvf7PLb7xsJLUVJARnQskqo7NCH3ma\nsV8=\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBGdEt1MTFDSTVKWE9ObTAw\neVUyZzlLUW5WYm1VZE5waWI4czJUOXMxR1RBCjJYT09teGpWK3Fvcml2Rzd6eDhU\nMTNhVDRRaFZnYjJpUUVicytRZlN6d00KLS0tIEMxc2dtbUVsTmI5KzNTTVVrSnBu\nQ2lvTGJQcWNhVHdFck1zaWtXY2RHNXMKHG464L9M0OuL6+uWL6CZa1T6b0S5//fJ\ns4QVIOHeXkSo1l1sU1rMoK1Hx+lipoNCYZiTlB6dhw6WH2B6lXCKiw==\n-----END AGE ENCRYPTED FILE-----\n"
 			}
 		],
 		"lastmodified": "2026-06-14T08:00:24Z",
 		"mac": "ENC[AES256_GCM,data:xbnHfzQnGPW6uwH5e0j47OUuwaaaVh7mOuBJGKtGbPEallPA8kAv4CxKQIwfOlVh2VjcAUoVa9sMq/1+cknbN8ubANQ5b8eBsL+dfz+9hcwtR5hrXxN85DsBgw1PVDLdxtFszqvzuQRjEWColWoUzL46ZGlqZ7luoxbVF1nPSak=,iv:xKHnp9UJDOGLt5YrYnGxFvLOrN9Qaumt8wp2WVYijMQ=,tag:/l2XipPDBiWYhHTU+vsC/Q==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
 		"version": "3.12.1"
 	}
 }

vars/per-machine/control/grafana-admin/password/groups/admins

@@ -0,0 +1 @@
+../../../../../../sops/groups/admins

vars/per-machine/control/grafana-admin/password/secret

@@ -0,0 +1,23 @@
+{
+	"data": "ENC[AES256_GCM,data:olwE8Xo5y0hn2jDOgH/9A3iJ0O33ukUnjSjbF19oaY0=,iv:ZptLhMDSCHUSZAlBry+ILcuDOlEYZsGvgDBAxLzL9ms=,tag:nK4Ak54Hzz7epsc0Xrc33g==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1env5y2eey0p3dggs8tydwmtyhtwqylpg7spuququ4vvax6arzp2qjnm4tx",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBzZEpWbm5rWFZkQ2FhSS85\nSm14N0NKNVZJcmNZR1BFWW5ILzZiSS9MMGxrCjc3dGRCQkVwRVFoUWozejFqTTZu\nSHRtZUkxOU1OOFFWMlB1ZHZFUU9lMTQKLS0tIDhXUkx6ZmlLb0tYMERralVTQS8r\nYkdzZmVUV1hCamFVRlZJajFCdzBpeTAKfRbQwWPyDrCrqvfjsUFtRvCc1rEPAakA\n4HkbLC7vTrFqizqTKx+9n0zQCLRAQwEdM8HLb/vsBG0NsVRKOANO/g==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBclJ1eW1u\nb2xNQlZNRllOL2JPSEZLTEpPU1NDeWprWW1Cb1F4RWVoalIycQpWZGc0RG9BTDhV\neVlBc2dhaTFrdXMwT3RqRzBLUDAwTXNjQnFKc0RsLzNRCi0tLSBOam5QNmFlUzNm\nMEZKNm1nQlM5eWxTUnhWOXR2L0VWK2FLRndmT2hVd2lnCieOpQefykg9hfb+Tm5K\nDpgw/1XTw4GlyIzjp3sZAZX7dSLPPtZix290pVX68rxedQFmHWds8c4+UmLhZAe7\nlSI=\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA4dzlDcTc1OE5VajJONUtp\nMUY1ZVRoVmpLVDhpSzBnTVpoOTAvbWloN1F3CjRxb0xhZWVlN1dMK0dqZmJJdngr\nRDRyWlo0aXRJM2RHUmM4L20rQmhrVEkKLS0tIDBKRWZzNlg5NUZBZkRlQjNlSnVq\nVlNHc1E4LzdTdWVmMHZyZEdTdHBEZEkKTcUjUOuEBlIi5IW/JcdCuDxPIWRSdIQM\n/VBQT8p2x7n9dLPFmE4GkOaOsogussAnKTltFCopsDa8uymCFjAzAA==\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2026-06-17T03:17:45Z",
+		"mac": "ENC[AES256_GCM,data:sc6zUWctq3TL7TNzsV7joAc2I6+BkjUHN64e/SL3YVLICkS5Bz5rMFew1WEPxVWA/DSNCri1JxltmcdmpvYTNtIkcs6P0Gs+Mr19aW7HLyvUZ9FHBCx29lvj0RAG2Xx2g2iBCLY4tMYRaaBt3JcvxIgAhfo3+O31bYEF2/m1IRg=,iv:zsWgdszzm0ReFOu0VJOCu2RlJ184EnvMw6Winb9c32Q=,tag:6pr704vsIBo8OrEgyhIskA==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.12.1"
+	}
+}

vars/per-machine/control/grafana-secret-key/secret-key/groups/admins

@@ -0,0 +1 @@
+../../../../../../sops/groups/admins