mob next [ci-skip] [ci skip] [skip ci]

lastFile:machines/gw-cnx-1/configuration.nix

.gitignore

@@ -1,2 +1,2 @@
-result
+result*
 .direnv/

TODO.md

@@ -0,0 +1,69 @@
+# Infra roadmap
+
+Prioritized backlog for the cnx-network clan. See `docs/` for how the current
+pieces work.
+
+## 1. Alerting (done — pending deploy)
+
+Rules evaluated by vmalert against VictoriaMetrics on control, declared in
+`modules/monitoring/alerts.nix`:
+
+- [x] SOA serial divergence between ns1 and ns2 (secondary out of sync)
+- [x] Zone-expiry countdown on the secondary approaching zero (transfers failing)
+- [x] Any scrape target down (`up == 0`)
+- [x] Root filesystem nearly full
+
+Delivery stays minimal for now (`notifier.blackhole`): vmalert remote-writes
+alert state back to VM, so firing alerts show up as the `ALERTS` series in
+Grafana. Wiring a real notifier (Matrix) is a later step — drop `blackhole` and
+set `settings."notifier.url"` to an Alertmanager.
+
+## 2. Backups of critical state (DNSSEC done — pending vars + deploy)
+
+clan `borgbackup` instance in `clan.nix`: control is the server (repos under
+`/var/lib/borgbackup/<client>`), ns1 the client. ns1 declares
+`clan.core.state.knot.folders = [ "/var/lib/knot" ]`, so the Knot KASP keystore
+is backed up nightly (01:00) over the mesh with repokey encryption — control
+never holds plaintext. ns1 maps the `control` machine name to its mesh IP via
+`networking.hosts` so the `borg@control` repo resolves.
+
+Before deploy: `clan vars generate ns1` (YubiKey) to mint the borgbackup ssh
+keypair + repokey; control won't evaluate until ns1's public key exists. Then
+deploy ns1 and control.
+
+- [x] DNSSEC key material on ns1 (KSK/ZSK in Knot's KASP store) — losing it forces
+      an emergency DS rollover at the registrar
+- [ ] VictoriaMetrics TSDB on control (optional, retention is 180d) — deferred;
+      regenerable over time and control is the backup server, so this needs a
+      second client→server pair (e.g. control→ns2) rather than the same topology
+
+## 3. Blackbox DNS probing (done — pending deploy)
+
+`blackbox_exporter` on control (loopback `:9115`), probing each nameserver's
+public v4+v6 address for every zone: an SOA query (zone served?) and a DNSKEY
+query (still signed?). Blackbox has no DO-bit option, so signing is checked by
+asking for DNSKEY directly and asserting the RRset is present. Probe defs live
+in `modules/monitoring/blackbox-probes.nix`, shared by the exporter
+(`blackbox.nix`) and the VM scrape jobs (`server.nix`). Verified live against
+ns1/ns2: SOA + DNSKEY succeed on both servers over v4 and v6.
+
+- [x] `blackbox_exporter` on control doing real DNS + DNSSEC-validation queries
+      against ns1/ns2 — catches outside-in resolution failures the Knot stats miss
+- [x] paired with alerts (`DNSResolutionProbeFailed` / `DNSSECProbeFailed` in
+      `alerts.nix`) and a "DNS probes (outside-in)" row on the CNX DNS dashboard
+
+## 4. Third secondary off Hetzner (resilience)
+
+- [ ] A secondary nameserver on a different provider/network so a single-provider
+      outage doesn't take all authoritative DNS down (architectural — new machine)
+
+## 5. Centralized logs (done — pending deploy)
+
+VictoriaLogs on control (`:9428`, 30d retention, mesh-scoped) in
+`modules/monitoring/server.nix`, plus a VictoriaLogs Grafana datasource. All
+three hosts ship journald with systemd's own `services.journald.upload` to the
+`/insert/journald` endpoint (`modules/monitoring/exporters.nix`) — no extra
+agent. control uploads over loopback; ns1/ns2 over the mesh.
+
+- [x] VictoriaLogs on control to grep journald across all three hosts, pairing
+      with the existing VictoriaMetrics setup

clan.nix

@@ -1,28 +1,153 @@
 {
+  inputs,
+  self,
+  libNet,
+  lib,
+  ...
+}:
+let
+  hosts = import ./modules/hosts.nix;
+
+  # Single source of the machine list: inventory AND the per-machine baseline
+  # below are both derived from it, so no machine can be added without the
+  # baseline (e.g. admins group encryption for all its generated secrets).
+  fleet = {
+    control = { };
+    ns1 = { };
+    ns2 = { };
+    mx1 = { };
+    web01 = { };
+    # Site gateways (Topton 1U routers): dynamic PPPoE WAN, so they are NOT in
+    # modules/hosts.nix / the `internet` instance — clan reaches them over the
+    # zerotier mesh (or Tor) instead.
+    gw-cnx-1 = { };
+  };
+in
+{
+  clan = {
     # Ensure this is unique among all clans you want to use.
     meta.name = "cnx-network-clan";
     meta.domain = "cnx-network.internal";
 
-  inventory.machines = {
-    control = { };
-    ns1 = { };
-    ns2 = { };
-  };
+    specialArgs = { inherit inputs self; };
+
+    # Customize nixpkgs
+    # pkgsForSystem =
+    #   system:
+    #   import nixpkgs {
+    #     inherit system;
+    #     config = {
+    #       allowUnfree = true;
+    #     };
+    #     overlays = [];
+    #   };
+    secrets.age.plugins = [
+      "age-plugin-yubikey"
+      "age-plugin-fido2-hmac"
+    ];
+
+    inventory.machines = fleet;
 
     inventory.instances = {
 
-    admin = {
-      roles.default.tags.all = { };
-      roles.default.settings.allowedKeys = {
+      # Admin SSH keys + root password, split per the clan-core migration off
+      # the deprecated `admin` service (sshd handles keys, users the password).
+      sshd = {
+        roles.server.tags.all = { };
+        roles.server.settings.authorizedKeys = {
           "berwn" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIENAjhGQGraQoAjJzsomKP8GAmQPeGL1rNRNHgRcLqtT";
           "kurogeek" =
             "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEcZ/p1Ofa9liwIzPWzNtONhJ7+FUWd2lCz33r81t8+w kurogeek@kurogeek";
         };
       };
 
+      gw-router = {
+        module = {
+          name = "gw-router";
+          input = "self";
+        };
+        roles.default.machines."gw-cnx-1" =
+          let
+            genVlan = id: length: rec {
+              inherit id;
+              prefixLength = length;
+              subnet = libNet.net.cidr.make length "10.0.${lib.toString id}.0";
+              address = libNet.net.cidr.host 1 subnet;
+              dhcp.pool.from = libNet.net.cidr.host 100 subnet;
+              dhcp.pool.to = libNet.net.cidr.host 200 subnet;
+            };
+          in
+          {
+            settings = {
+              wan = {
+                interface = "enp1s0";
+                vlanId = null;
+              };
+
+              vlans = {
+                mgmt = lib.recursiveUpdate (genVlan 10 24) {
+                  dhcp.fixedIPs.storinator01 = {
+                    hwAddress = "7c:c2:55:e0:d6:40";
+                    ipAddress = "10.1.10.53";
+                  };
+                };
+                lan = genVlan 20 24;
+                iot = genVlan 40 24;
+                voip = genVlan 50 24;
+                dmz = genVlan 60 24;
+                unit1 = genVlan 110 24;
+                unit2 = genVlan 120 24;
+                unit3 = genVlan 130 24;
+                unit4 = genVlan 140 24;
+                unit5 = lib.recursiveUpdate (genVlan 150 24) {
+                  dhcp.fixedIPs.newt = {
+                    hwAddress = "7c:d3:0a:21:58:0b";
+                    ipAddress = "10.1.150.22";
+                  };
+                };
+              };
+
+              trunkPorts = [ "enp2s0" ];
+
+              accessPorts = {
+                enp4s0.vlanId = 10;
+              };
+
+              upLinkPorts = [ "enp3s0" ];
+
+              enableOmada = true;
+
+              speedtest = {
+                enable = true;
+              };
+
+            };
+          };
+      };
+
+      root-user = {
+        module = {
+          name = "users";
+          input = "clan-core";
+        };
+        roles.default.tags.all = { };
+        roles.default.settings = {
+          user = "root";
+          prompt = false; # auto-generate, like the old admin service
+          share = false; # per-machine password, not fleet-wide
+        };
+      };
+
       zerotier = {
         roles.controller.machines."control" = { };
         roles.peer.tags.all = { };
+        # External members admitted by ZeroTier node id (stable per device).
+        # Inventory machines are auto-accepted; this is only for peers outside the
+        # clan. Node id comes from `zerotier-cli info` on the joining device.
+        roles.controller.settings.allowedIds = [
+          "8802c8d7e0" # alex-nixos
+          "2bd36db8cc" # kurogeek-thinkpad
+        ];
       };
 
       tor = {
@@ -30,21 +155,35 @@
       };
 
       # Direct SSH to public IPs — clan's priority-1 connection path, with the
-    # ZeroTier mesh and Tor kept as automatic fallbacks. Raw IPs (not the
-    # ns1/ns2 DNS names) so reaching these hosts never depends on their own
-    # DNS being up.
-    internet = {
-      roles.default.machines.control.settings.host = "77.42.68.181";
-      roles.default.machines.ns1.settings.host = "46.224.170.206";
-      roles.default.machines.ns2.settings.host = "157.180.70.82";
-    };
+      # ZeroTier mesh and Tor kept as automatic fallbacks. Raw IPs (from
+      # modules/hosts.nix, not the ns1/ns2 DNS names) so reaching these hosts never
+      # depends on their own DNS being up.
+      internet.roles.default.machines = builtins.mapAttrs (_: h: {
+        settings.host = h.ipv4;
+      }) hosts;
 
       # Recovery root password for console access when a machine fails to boot.
       emergency-access = {
         roles.default.tags.nixos = { };
       };
+
+      # Encrypted, deduplicating backups. control hosts the repos; ns1 is the
+      # only client, backing up its declared clan.core.state (the Knot DNSSEC
+      # keystore) over the mesh. Repo lives at /var/lib/borgbackup/ns1 on control.
+      # Cross-host so an ns1 loss is recoverable; repokey encryption means control
+      # never holds plaintext. Run `clan vars generate ns1` (YubiKey) before deploy.
+      borgbackup = {
+        roles.server.machines.control = { };
+        roles.client.machines.ns1 = { };
+      };
     };
 
-  machines = {
+    # Fleet-wide baseline applied to every machine. Secrets minted by
+    # `clan vars generate` are encrypted for the admins group from the very
+    # first run — generating before this took effect is what forced the
+    # re-encryption dance (`clan vars fix`) on gw-cnx-1.
+    machines = builtins.mapAttrs (_: _: {
+      clan.core.sops.defaultGroups = [ "admins" ];
+    }) fleet;
   };
 }

docs/book.toml

@@ -0,0 +1,12 @@
+[book]
+title = "CNX Infra Runbook"
+description = "Operational docs for the cnx-network clan: DNS, ZeroTier mesh, monitoring."
+authors = ["B4L"]
+src = "src"
+language = "en"
+
+[output.html]
+default-theme = "navy"
+preferred-dark-theme = "navy"
+git-repository-url = "https://git.b4l.co.th/B4L/cnx-network-clan"
+edit-url-template = "https://git.b4l.co.th/B4L/cnx-network-clan/_edit/main/docs/{path}"

docs/src/SUMMARY.md

@@ -0,0 +1,9 @@
+# Summary
+
+- [Overview](./overview.md)
+- [ZeroTier mesh](./mesh.md)
+- [DNS](./dns.md)
+- [Mail](./mail.md)
+- [Site gateways](./gateways.md)
+- [Monitoring](./monitoring.md)
+- [Backups](./backups.md)

docs/src/backups.md

@@ -0,0 +1,61 @@
+# Backups
+
+Encrypted, deduplicating backups via clan's `borgbackup` service, declared in
+`clan.nix`. The only critical, non-regenerable state is the **Knot DNSSEC
+keystore** on `ns1` (the KSK/ZSK private keys under `/var/lib/knot`); losing it
+forces an emergency DS rollover at the registrar.
+
+## Topology
+
+- **control** is the borgbackup **server** — it hosts the repos under
+  `/var/lib/borgbackup/<client>` (so `ns1`'s repo is `/var/lib/borgbackup/ns1`).
+- **ns1** is the **client**. It backs up everything it declares as clan state
+  (`clan.core.state.knot.folders = [ "/var/lib/knot" ]`) once a day at 01:00,
+  over the ZeroTier mesh.
+
+The backup is cross-host so that losing `ns1` is recoverable, and stays
+self-contained (no third-party storage). Encryption is `repokey` with a
+generated passphrase, so `control` only ever stores ciphertext.
+
+Mesh peers have no name resolution, so `ns1` maps the `control` machine name to
+its ZeroTier address via `networking.hosts`; that is how the `borg@control` repo
+URL resolves.
+
+## Secrets
+
+The borgbackup ssh keypair and repokey passphrase are clan vars, generated once
+(needs the YubiKey). `control` will not evaluate until `ns1`'s public key
+exists, so generate before the first deploy:
+
+```
+clan vars generate ns1
+clan machines update ns1
+clan machines update control
+```
+
+## Operating
+
+Backups are driven by systemd on `ns1` (`borgbackup-job-control.timer`).
+
+```
+# trigger a backup now (on ns1)
+borgbackup-create
+
+# list archives (on ns1)
+borgbackup-list
+
+# restore selected folders from an archive (on ns1)
+NAME='<archive-name>' FOLDERS=/var/lib/knot borgbackup-restore
+```
+
+Retention is pruned automatically: all archives from the last day, then 7 daily
+and 4 weekly.
+
+## Monitoring
+
+The **CNX Backups** Grafana dashboard
+(`modules/monitoring/dashboards/backups.json`) tracks job health, time since the
+last successful run, and per-job state — all from the node_exporter systemd
+collector on the client. There is no dedicated borg metrics exporter; the unit
+state and the timer's last-trigger timestamp are enough to catch a backup that
+stops running or fails.

docs/src/dns.md

@@ -0,0 +1,95 @@
+# DNS
+
+Authoritative DNS for three zones, served by Knot:
+
+- `cnx.network`
+- `buildfor.life`
+- `cnx.email`
+
+Add a zone in `modules/dns/domains.nix` **and** drop a matching `<domain>.zone`
+file in `modules/dns/zones/`.
+
+## Primary / secondary
+
+- **`ns1` = primary (master).** Loads each zone from its file, signs it, and
+  notifies `ns2`. Config in `machines/ns1/configuration.nix`.
+- **`ns2` = secondary (slave).** Pulls every zone from `ns1` (AXFR/IXFR) and
+  accepts its NOTIFY. Config in `machines/ns2/configuration.nix`.
+
+Zone transfers run **over the ZeroTier mesh**, authenticated with a shared TSIG
+key (`dns-tsig`, a clan var copied to both machines).
+
+## Serial handling
+
+`ns1` uses `zonefile-load = difference-no-serial` with `serial-policy = unixtime`:
+edit records without touching the SOA serial — Knot diffs the file, assigns a
+strictly-monotonic unixtime serial, signs, and transfers. `journal-content = all`
+holds the live signed zone (required by `difference-no-serial`).
+
+## DNSSEC
+
+Automatic signing on `ns1` only, policy `cnx`: ECDSA P-256/SHA-256. The ZSK
+auto-rolls; the KSK is kept stable, so the DS at the registrar only changes on a
+manual KSK rollover.
+
+### Registrar DS records
+
+Knot manages all key material itself on `ns1` (the only signer); the KSK/ZSK
+private keys live in the KASP keystore under `/var/lib/knot` (backed up nightly —
+see [Backups](./backups.md)). You never touch the private keys directly.
+
+What a registrar needs is the **DS record** for a zone's KSK, which anchors the
+zone into the parent's chain of trust. Generate it on `ns1` — the `keymgr` wrapper
+is already pointed at Knot's config, and it runs as the `knot` user that owns the
+keystore:
+
+```
+sudo -u knot keymgr <zone> ds
+```
+
+e.g. `sudo -u knot keymgr cnx.email ds`. Paste the printed DS record (key tag,
+algorithm 13, digest type, digest) into the registrar's DNSSEC form for that
+domain. Repeat per signed zone (`cnx.network`, `buildfor.life`, `cnx.email`) at
+whichever registrar holds each delegation. After submitting, confirm the parent
+publishes it with `dig +short DS <zone>`.
+
+The ZSK rolls automatically and needs no registrar action; only a **KSK rollover**
+requires re-submitting the DS.
+
+> **Pending (manual):** submit DS records for `buildfor.life` and `cnx.email`
+> once they're at a DNSSEC-capable registrar.
+
+## ACME DNS-01
+
+Certificates are issued by `_acme-challenge` TXT updates that `ns1` accepts over
+TSIG, signs, and transfers to `ns2` (which never needs these keys). Each consumer
+gets its **own** key, scoped by an ACL to exactly the owner names it needs and
+attached only to the zone it lives in — so a leaked key can write nothing but its
+own challenges.
+
+- **`acme_ddns`** (`acl_acme`) — the general key, scoped to `TXT` at or under
+  `_acme-challenge.<zone>` and attached to every zone. Client config:
+  ```
+  clan vars get ns1 dns-acme-tsig/acme.conf
+  ```
+- **`acme_mx1`** (`acl_acme_mx1`) — held only by `mx1`, scoped to
+  `_acme-challenge.{mx1,mta-sts,mail}` and attached only to `cnx.email` (the mail
+  cert plus its MTA-STS and client-alias SANs). Secret shared via the
+  `dns-acme-mx1-secret` generator.
+- **`acme_web01`** (`acl_acme_web01`) — held only by `web01`, scoped to
+  `_acme-challenge` and attached only to `cnx.network` (where the wildcard
+  `*.cnx.network` challenge lands, at the apex). Secret shared via the
+  `dns-acme-web01-secret` generator.
+
+## Runbook: stale secondary
+
+If `ns2` serves stale records while SOA serials match (e.g. after a manual zone
+edit that didn't bump the serial as expected), force a fresh transfer on `ns2`:
+
+```
+knotc zone-retransfer <zone>
+```
+
+Watch the **CNX DNS** Grafana dashboard: the per-nameserver SOA serial table
+should agree across `ns1`/`ns2`, and "seconds until zone expiry" on the secondary
+should reset on each successful transfer rather than counting toward zero.

docs/src/gateways.md

@@ -0,0 +1,145 @@
+# Site gateways
+
+NixOS routers (OPNsense replacements) on Topton 1U fanless boxes — Intel N300,
+4x i226-V 2.5G ports (some units add 2x 10G SFP+), 16GB DDR5, 256GB mSATA.
+Four sites are planned; the first is `gw-cnx-1` (Chiang Mai). Everything lives
+in the reusable `modules/router/` module (`cnx.router.*` options); a site's
+machine config only sets port names, VLANs, and feature flags.
+
+Naming: `gw-<city>-<n>`, e.g. `gw-cnx-1`.
+
+## What each gateway runs
+
+| Function     | Implementation                                                                                                                                                                                |
+| ------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| WAN          | PPPoE (`pppd`), per-site ISP credentials via clan vars prompts; `wan.vlanId` when the ISP tags the session (AIS: 10); `wan.macAddress` to clone the old router's MAC if the ISP has it pinned |
+| LAN          | VLAN-filtering bridge `br0` over the trunk ports (networkd); `accessPorts` pin a port untagged to one VLAN — convention: the last copper port is an untagged `mgmt` recovery port             |
+| Firewall/NAT | nftables: default-deny WAN, no inter-VLAN, MSS clamp, v4 NAT                                                                                                                                  |
+| DHCP         | Kea, one subnet per VLAN                                                                                                                                                                      |
+| DNS          | Blocky (blocklist resolver), metrics on :4000 scraped by control                                                                                                                              |
+| IPv6         | DHCPv6-PD on ppp0, /64 per VLAN via SLAAC                                                                                                                                                     |
+| Bans         | CrowdSec + nftables bouncer (sshd log parsing)                                                                                                                                                |
+| Omada        | Optional per site: TP-Link Omada controller as a podman container                                                                                                                             |
+| Proxy        | Optional: Caddy reverse proxy for internal services under `*.<site><n>.cnx.network` with a real Let's Encrypt wildcard (DNS-01 against ns1)                                                   |
+| Diagnostics  | iperf3 server on 5201, reachable from every VLAN and the mesh (`iperf3 -c <router-vlan-addr>`); CLI toolkit: tcpdump, mtr, ethtool, conntrack, kdig, iftop, librespeed-cli                    |
+| Speed test   | Hourly librespeed run (`speedtest.timer`) → `speedtest_*` metrics via node_exporter; vmalert flags download < 50% of the link's own 7-day median (`WANSpeedDegraded`)                         |
+| Management   | ZeroTier mesh: SSH, node_exporter, journald upload — like the fleet                                                                                                                           |
+| Storage      | Single-disk ZFS (zstd, `/var` its own dataset for snapshots)                                                                                                                                  |
+
+## Addressing plan
+
+Each site owns `10.<siteId>.0.0/16`. Defaults per VLAN: subnet
+`10.<siteId>.<vlanId>.0/24`, router at `.1`, DHCP pool `.100–.199`
+(`.2–.99` static/infra, `.200–.254` reserved).
+
+VLAN id convention (fleet-wide): **10 = mgmt**, **20 = lan** (both mandatory at
+every site), 30 = guest, 40 = iot (reserved). Sites add their own beyond that.
+
+Larger subnets (e.g. public-WiFi guest networks that outgrow a /24) are carved
+from the **upper half** `10.<siteId>.128.0/17` and set explicitly on the VLAN.
+The lower half stays reserved for /24s indexed by VLAN id. High-churn VLANs
+should also shorten `dhcp.leaseTime` (default 86400 s) so the pool recycles.
+First user: `gw-cnx-2` (site 2) runs the public WiFi — guest VLAN 30 at
+`10.2.128.0/22`, pool `10.2.128.100 – 10.2.131.250`, `dhcp.leaseTime = 3600`.
+
+| Site | siteId | mgmt           | lan            | site-specific VLANs                                             |
+| ---- | ------ | -------------- | -------------- | --------------------------------------------------------------- |
+| cnx  | 1      | `10.1.10.0/24` | `10.1.20.0/24` | iot 40, voip 50, dmz 60, unit1–5 110/120/130/140/150 (all /24s) |
+
+Static DHCP leases are declared per VLAN via `dhcp.reservations` (attribute
+name = hostname, plus `hwAddress`/`ipAddress`); park them outside the pool,
+in the `.2–.99` infra range or `.200–.254`.
+
+Trust model: mgmt → everything; other VLANs → router DNS/DHCP + internet only
+(no inter-VLAN); WAN → nothing inbound; mesh → SSH, metrics, Omada UI.
+
+## Adding a site
+
+1. Copy `machines/gw-cnx-1/` to `machines/gw-<city>-<n>/`; set `site`,
+   `siteId` (next free number), port names, VLANs, and `omada.enable`.
+   Fill the real disk id in `disko.nix` (from the installer:
+   `ls -l /dev/disk/by-id/`). Keep the `mgmt`/`lan` VLANs.
+2. Add the machine to `inventory.machines` in `clan.nix` and to the machine
+   list in `modules/mesh-hosts.nix`. Do **not** add it to `modules/hosts.nix`
+   (dynamic PPPoE IP; clan connects over the mesh).
+3. Add its node_exporter and Blocky scrape targets in
+   `modules/monitoring/server.nix`.
+4. If the site uses the internal proxy (`cnx.router.proxy.enable`), wire ns1:
+   import `(import ../../modules/dns/acme-gw-secret.nix "gw-<city>-<n>")` in
+   **both** the gateway's and ns1's configuration, and on ns1 add the
+   `dns-acme-gw-<city>-<n>-knot` generator, its `keyFiles` entry, and an
+   `acl_acme_gw_<city>_<n>` scoped to `_acme-challenge.<city><n>` on the
+   `cnx.network` zone (copy the `gw-cnx-1` blocks). Then
+   `clan vars generate ns1` and redeploy ns1.
+5. `git add` the new machine directory — flake evaluation only sees
+   git-tracked files, so an untracked `machines/gw-…/` is silently ignored.
+6. `clan vars generate gw-<city>-<n>` — prompts for the site's PPPoE
+   credentials, mints the ZeroTier identity etc. (`nix flake check` fails
+   until this has run, because mesh-hosts reads the ZeroTier IP var.)
+7. Boot the box from a NixOS installer USB on the local network, then:
+   `clan machines install gw-<city>-<n> --target-host root@<lan-ip>`
+8. Check `facter.json` for the real NIC names, fix `wan.interface` /
+   `trunkPorts` if the enumeration differs, and
+   `clan machines update gw-<city>-<n>` (rides the mesh from then on).
+9. Add a row to the site table above and to the machines table in
+   [Overview](./overview.md).
+
+## Omada controller
+
+Sites with TP-Link Omada APs/switches set `cnx.router.omada.enable = true`.
+The controller runs as a podman container (`mbentley/omada-controller` — there
+is no nixpkgs package) with host networking on the mgmt VLAN, where adoption
+broadcasts (UDP 29810) live. UI: `https://<mgmt-addr>:8043` from mgmt, or the
+gateway's mesh address from an admin machine. Controller state is under
+`/var/lib/omada`, declared as clan state (`clan.core.state.omada`); wiring it
+into the borgbackup instance is still a follow-up.
+
+## Internal reverse proxy
+
+`cnx.router.proxy.enable` puts Caddy on the gateway, terminating TLS for
+`<service>.<site><n>.cnx.network` (e.g. `https://omada.cnx1.cnx.network`) and
+forwarding to internal backends:
+
+```nix
+cnx.router.proxy = {
+  enable = true;
+  services.omada = {
+    backend = "https://127.0.0.1:8043";
+    insecureSkipVerify = true; # Omada's cert is self-signed
+  };
+};
+```
+
+- **Cert**: one real Let's Encrypt wildcard `*.<site><n>.cnx.network` per
+  gateway, issued via ACME DNS-01 (lego/RFC2136) against ns1 — works behind
+  PPPoE with zero inbound reachability, and browsers trust it without a CA
+  install. Each gateway has its own TSIG key (`acme_gw_<city>_<n>`) that ns1
+  scopes to `_acme-challenge.<site><n>` TXT records only.
+- **Resolution**: the names exist only internally — Blocky answers
+  `*.<site><n>.cnx.network` with the router's `lan` address; the public
+  `cnx.network` zone never carries them.
+- **Access**: `proxy.allowVlans` (default `mgmt` + `lan`) get 443 (and 80 for
+  the HTTP→HTTPS redirect). Not exposed to WAN, guest VLANs, or the mesh.
+
+## Runbook
+
+- **PPPoE down**: `systemctl status pppd-wan`, `journalctl -u pppd-wan` on the
+  gateway (over the mesh). ISP credentials live in the `pppoe-credentials`
+  vars generator; re-enter with `clan vars generate gw-<city>-<n> --regenerate`.
+- **A VLAN gets no leases**: `systemctl status kea-dhcp4-server`; check the
+  port's `bridgeVLANs` tagging and that the switch trunk carries the VLAN.
+- **DNS/blocklist issues**: Blocky metrics are in VictoriaMetrics (job
+  `blocky`); `journalctl -u blocky` on the gateway.
+- **Slow internet reported / `WANSpeedDegraded` firing**: check the
+  `speedtest_download_mbps` / `speedtest_ping_ms` trend in VictoriaMetrics,
+  then `librespeed-cli` and `mtr <isp-gw>` on the gateway to separate ISP
+  degradation from local saturation (`iftop` on ppp0 shows who's eating it).
+- **Banned yourself**: `cscli decisions list` / `cscli decisions delete --ip <ip>`
+  on the gateway. The ZeroTier mesh `/88` is whitelisted at the parser stage
+  (`cnx/mesh-whitelist`), so admin access over the mesh can never be banned.
+
+## Follow-ups
+
+- Borgbackup client for Omada + Kea lease state.
+- Suricata (IDS-only) if CPU headroom allows — deliberately skipped for now.
+- Remaining three sites.

docs/src/mail.md

@@ -0,0 +1,90 @@
+# Mail
+
+A single mail server, `mx1`, runs the [Simple NixOS Mailserver](https://nixos-mailserver.readthedocs.io/)
+stack (Postfix + Dovecot + Rspamd + OpenDKIM) for the `cnx.email` domain. All of
+it is declared in `modules/mail.nix`, imported by `machines/mx1/configuration.nix`.
+
+Mailboxes are **virtual** (not system users): each address is a login account
+whose password is a four-word passphrase auto-generated by a clan vars generator
+(e.g. `otter-lantern-cobalt-driftwood-42`). The generator stores both the
+passphrase (handed to the user) and its sha-512 hash (consumed by SNM); plaintext
+never lands in the repo.
+
+The mail cert is obtained by `mx1` itself via ACME DNS-01 against `ns1` (key
+`acme_mx1`), and carries `mx1.cnx.email` plus the `mta-sts.cnx.email` and
+`mail.cnx.email` SANs. See [DNS → ACME DNS-01](./dns.md#acme-dns-01).
+
+## Add a mailbox
+
+1. Append the address to `accounts` in `modules/mail.nix`:
+   ```nix
+   accounts = [
+     "postmaster@cnx.email"
+     "alice@cnx.email"      # new
+   ];
+   ```
+2. Mint its generated password (user-run, YubiKey touch):
+   ```
+   clan vars generate mx1
+   ```
+3. Redeploy: `clan machines update mx1`.
+4. Hand the passphrase to the user. The generator name replaces `@`→`-at-` and
+   `.`→`-`:
+   ```
+   clan vars get mx1 mail-passwd-alice-at-cnx-email/passphrase
+   ```
+
+The DMARC report inbox (`dmarc@cnx.email`) is special: its password comes from
+the **shared** `mail-dmarc-cred` generator (not the per-machine set) so parsedmarc
+on `control` can read the same passphrase over the mesh.
+
+## Add a domain
+
+Adding a second mail domain is more than a config line — it needs its own DNS
+records and DKIM key. Steps:
+
+1. **Serve the zone** (if not already): add it in `modules/dns/domains.nix` and
+   drop a `<domain>.zone` file — see [DNS](./dns.md).
+2. **Declare the domain** in `modules/mail.nix`:
+   ```nix
+   mailserver.domains = [ "cnx.email" "newdomain.tld" ];
+   ```
+   Add at least a `postmaster@newdomain.tld` account (RFC 5321 requires it) the
+   same way as above.
+3. **Publish the mail DNS records** in the new zone, mirroring `cnx.email`'s
+   `; ---- Mail ----` block:
+   - `MX 10 mx1.cnx.email.` — point at the existing MX; do **not** stand up a new
+     hostname unless you really want a separate server.
+   - SPF: `@ TXT "v=spf1 mx -all"`.
+   - DKIM: SNM generates a key per domain on first start under
+     `mailserver.dkimKeyDirectory` (selector `mail`). After deploy, read the
+     public key off `mx1` and paste it as `mail._domainkey TXT`. The private keys
+     are declared as clan state and backed up.
+   - DMARC: `_dmarc TXT "v=DMARC1; p=quarantine; rua=mailto:dmarc@cnx.email; ..."`
+     — reports can keep flowing to the existing `dmarc@cnx.email` inbox.
+4. **DANE / MTA-STS are tied to `mx1.cnx.email`**, the MX hostname — they do **not**
+   need duplicating per sending-domain. A new domain that uses `mx1.cnx.email` as
+   its MX inherits the existing TLSA and MTA-STS policy. Only add new TLSA/MTA-STS
+   records if you introduce a new MX hostname.
+5. `clan vars generate mx1` (if you added accounts), then
+   `clan machines update mx1`.
+
+## DKIM, DANE, MTA-STS reminders
+
+These three are easy to get subtly wrong; the live records for `cnx.email` and
+the exact commands to regenerate them are in `modules/dns/zones/cnx.email.zone`:
+
+- **DKIM** key rotation = regenerate under `dkimKeyDirectory`, then republish the
+  `mail._domainkey TXT`.
+- **DANE TLSA** (`_25._tcp.mx1 TLSA 3 1 1 …`) is the SHA-256 of `mx1`'s cert
+  **public key**. It stays valid across renewals because lego runs with
+  `--reuse-key`; only recompute it if the key changes. The `openssl` one-liner is
+  in the zone file.
+- **MTA-STS**: bump the `_mta-sts TXT` id **every time** the policy in
+  `modules/mail.nix` changes, or senders keep the cached policy.
+
+## DMARC reporting
+
+`mx1` delivers aggregate/forensic DMARC reports to `dmarc@cnx.email`; parsedmarc
+on `control` polls that mailbox over the mesh and feeds the results into Grafana.
+See [Monitoring](./monitoring.md).

docs/src/mesh.md

@@ -0,0 +1,37 @@
+# ZeroTier mesh
+
+A private IPv6 overlay that every machine (and admin laptops) shares. DNS zone
+transfers and metrics scraping ride this mesh, never the public net.
+
+- **Controller:** `control` (the `zerotier` instance in `clan.nix`).
+- **Peers:** every machine (`roles.peer.tags.all`).
+- **Prefix:** `fd06:1bad:ece2:92ad:ba99:9300::/88` (RFC 4193: `fd` + network id + `0x9993`).
+
+## The mesh map
+
+`modules/mesh-hosts.nix` does **not** hardcode addresses. It reads each machine's
+IP from the public clan vars that clan-core's zerotier service writes. As of
+clan-core 26.05 these are shared, instance-scoped generators
+(`vars/shared/zerotier-ip-<machine>-zerotier/ip/value`), and the `/88` subnet is
+derived from the instance network id
+(`vars/shared/zerotier-network-zerotier/network-id/value`). Regenerate or re-key a
+node and the map follows automatically.
+
+Consumers: `modules/dns/authoritative.nix` (transfer ACLs), `modules/monitoring/*`
+(scrape targets and firewall scoping).
+
+## Admitting external members
+
+Inventory machines are auto-accepted. External devices (admin laptops) are listed
+in `clan.nix` under the controller's `allowedIds` — clan-core 26.05 admits members
+by their 10-char ZeroTier node id directly, so no IP derivation is needed:
+
+```nix
+roles.controller.settings.allowedIds = [
+  "8802c8d7e0" # alex-nixos
+  "2bd36db8cc" # kurogeek-thinkpad
+];
+```
+
+A device's 10-char node id comes from `zerotier-cli info` on that device. After
+editing, deploy `control`; the controller admits the new member on its next run.

docs/src/monitoring.md

@@ -0,0 +1,122 @@
+# Monitoring
+
+Metrics and logs live on `control` over the ZeroTier mesh; the Grafana dashboards
+are also published publicly through `web01` (see [Dashboards](#dashboards)).
+
+## Collection
+
+- **node_exporter** (`:9100`) on every machine — CPU, memory, disk, systemd units.
+  Binds all interfaces; the scrape ports are firewall-scoped to the mesh subnet
+  (`modules/monitoring/exporters.nix`).
+- **knot-exporter** (`:9433`) on `ns1`/`ns2` only — reads Knot's control socket,
+  fed by the `mod-stats` module (query/response counters per zone).
+- **blackbox_exporter** (`127.0.0.1:9115`) on `control` only — outside-in DNS
+  probes. For every zone it queries each nameserver's **public** address (v4 and
+  v6) for SOA (is the zone served?) and DNSKEY (is it still signed?). This is the
+  resolver's-eye view that the Knot stats can't see. Probe definitions are shared
+  between the exporter and the scrape jobs in `modules/monitoring/blackbox-probes.nix`.
+
+## Storage & scraping
+
+**VictoriaMetrics** on `control`, bound to `127.0.0.1:8428`, 180-day retention
+(`modules/monitoring/server.nix`). It scrapes `control` over loopback and
+`ns1`/`ns2`/`mx1`/`web01` over the mesh.
+
+> The scraper dials IPv4-only by default, so mesh (IPv6) targets need
+> `extraOptions = [ "-enableTCP6" ]`. Without it, ns1/ns2 are dropped with
+> "no suitable address found". Check live target health on `control`:
+>
+> ```
+> curl -s http://127.0.0.1:8428/api/v1/targets | jq '.data.activeTargets[] | {i:.labels.instance, h:.health, e:.lastError}'
+> ```
+
+## Dashboards
+
+**Grafana** on `control` (`:3000`), anonymous access disabled. Reachable directly
+over the mesh, and publicly at `https://grafana.cnx.network` via `web01`'s reverse
+proxy (TLS termination — see [Overview](./overview.md)). The admin password is a
+clan var:
+
+```
+clan vars get control grafana-admin/password
+```
+
+Dashboards are provisioned from `modules/monitoring/dashboards/` (any JSON file
+there is picked up):
+
+- **CNX DNS** (`dns.json`) — firing alerts, per-nameserver SOA serials, zone
+  expiry countdowns, query/response rates, host CPU/memory/disk/load, and the
+  outside-in DNS probes.
+- **CNX Backups** (`backups.json`) — borgbackup job health, time since the last
+  run, and per-job state. See [Backups](./backups.md).
+- **CNX Uptime** (`uptime.json`) — per-host up/down status, current uptime,
+  availability over the selected window, and up/down history. Label-driven, so
+  every scraped host appears automatically.
+- **CNX WAN Speed** (`wan.json`) — per-gateway WAN speed from the hourly
+  librespeed runs: latest download/upload/ping, throughput history with each
+  link's 7-day median as reference, latency/jitter, and test status. See
+  [Site gateways](./gateways.md).
+- **parsedmarc** — DMARC aggregate/forensic report viewer. Auto-provisioned by
+  the `parsedmarc` module (not from `dashboards/`); reads its own Elasticsearch
+  datasource, not VictoriaMetrics. See [DMARC reports](#dmarc-reports) below.
+
+## DMARC reports
+
+The `cnx.email` DMARC record (`rua`/`ruf`) points at the `dmarc@cnx.email`
+mailbox on `mx1`. **parsedmarc** on `control` (`modules/monitoring/parsedmarc.nix`)
+polls that mailbox over IMAPS, parses the XML reports, and stores them in a local
+**Elasticsearch** (`127.0.0.1:9200`, loopback-only); Grafana renders them via the
+parsedmarc dashboard + two Elasticsearch datasources (`dmarc-ag`/`dmarc-fo`).
+
+> Two Grafana-13 gotchas, both handled in `modules/monitoring/parsedmarc.nix`:
+>
+> 1. **The Elasticsearch data source plugin is no longer bundled.** Grafana 13
+>    externalised its core data sources and nixpkgs ships no `grafanaPlugins`
+>    entry for Elasticsearch, so a bare ES datasource fails with "Plugin not
+>    registered" and every panel errors (and Explore silently falls back to the
+>    default datasource). We pull the official, Grafana-signed `elasticsearch`
+>    plugin from the catalog via `services.grafana.declarativePlugins`.
+> 2. **Datasource variables now resolve by UID, not name.** The bundled dashboard
+>    binds its datasource through template variables whose stored value is the
+>    datasource _name_ (`dmarc-ag`/`dmarc-fo`); Grafana 11+ matches that against
+>    the datasource **UID**. So we provision the two datasources ourselves with
+>    their **UID pinned to their name** (and leave the module's own
+>    `provision.grafana.datasource` off to avoid a duplicate, uid-less pair).
+>    `deleteDatasources` clears any stale uid-less records left by an earlier
+>    deploy so the pinned-UID versions can be re-created.
+
+The IMAP fetch rides the **mesh**, not the public net: `control` pins
+`mx1.cnx.email` to mx1's mesh address in `/etc/hosts`, so TLS still validates
+against the public cert while the bytes stay on the overlay. The mailbox
+passphrase is the shared `mail-dmarc-cred` clan var (so both mx1's mailserver and
+control's parsedmarc see the same value):
+
+```
+clan vars get mx1 mail-dmarc-cred/passphrase
+```
+
+## Logs
+
+**VictoriaLogs** on `control` (`:9428`), 30-day retention
+(`modules/monitoring/server.nix`). All three hosts ship journald to it via
+systemd's own `services.journald.upload` → the `/insert/journald` endpoint
+(`modules/monitoring/exporters.nix`); no extra agent. `control` uploads over
+loopback so its logs survive a mesh outage, the other hosts push over the mesh,
+and 9428 is firewall-scoped to the mesh like everything else.
+
+> Same IPv4-only default as the scraper: VictoriaLogs binds `0.0.0.0:9428` for a
+> bare `:9428`, so mesh (IPv6) pushes from ns1/ns2 are refused until you pass
+> `extraOptions = [ "-enableTCP6" ]` (binds `[::]`). Verify the bind on `control`:
+>
+> ```
+> ss -tlnp | grep 9428   # want [::]:9428, not 0.0.0.0:9428
+> ```
+
+Query logs from Grafana via the provisioned **VictoriaLogs** datasource (Explore
+view, LogsQL), or directly in the built-in UI at `http://[control]:9428/select/vmui`.
+Logs are tagged with `_HOSTNAME` and `_SYSTEMD_UNIT`, so to follow one service
+across hosts:
+
+```
+_SYSTEMD_UNIT:"knot.service"
+```

docs/src/overview.md

@@ -0,0 +1,37 @@
+# Overview
+
+This is the operational runbook for the **cnx-network** clan. Everything here is
+managed declaratively from the [clan repo](https://git.b4l.co.th/B4L/cnx-network-clan);
+this book is built from `docs/` and served on `control` over the ZeroTier mesh.
+
+## Machines
+
+| Machine    | Role                                   | Public IPv4      | Public IPv6             |
+| ---------- | -------------------------------------- | ---------------- | ----------------------- |
+| `control`  | ZeroTier controller, monitoring, docs  | `77.42.68.181`   | `2a01:4f9:c013:e6d0::1` |
+| `ns1`      | Knot DNS **primary** (master)          | `46.224.170.206` | `2a01:4f8:c014:b5c5::1` |
+| `ns2`      | Knot DNS **secondary** (slave)         | `157.180.70.82`  | `2a01:4f9:c014:6d87::1` |
+| `mx1`      | Mail server (**MX** for cnx.email)     | `5.223.65.38`    | `2a01:4ff:2f0:1963::1`  |
+| `web01`    | Public reverse proxy (TLS termination) | `5.223.55.246`   | `2a01:4ff:2f0:2d8f::1`  |
+| `gw-cnx-1` | Site gateway Chiang Mai (router)       | dynamic (PPPoE)  | —                       |
+
+## Access
+
+- Admin SSH and all internal services ride the **ZeroTier mesh**, not the public
+  net. Public SSH (22) is intentionally closed at the Hetzner cloud firewall.
+- clan reaches machines by their public IPs first (the `internet` instance), with
+  the mesh and Tor as automatic fallbacks.
+
+## Host baseline
+
+Every machine sets a fixed-offset `time.timeZone` and runs **chrony** for time
+sync (`services.chrony.enable`). chrony is the single NTP implementation across
+the fleet — do **not** use `systemd-timesyncd`; the two are mutually exclusive
+and we standardise on chrony everywhere. New hosts must set both the timezone and
+chrony as part of their baseline config.
+
+## Editing these docs
+
+Commit-to-edit: change the markdown under `docs/src/`, commit, and redeploy
+`control`. There is no in-browser editor by design — the docs are versioned and
+reviewed alongside the config that they describe.

flake.lock

@@ -1,5 +1,21 @@
 {
   "nodes": {
+    "blobs": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1604995301,
+        "narHash": "sha256-wcLzgLec6SGJA8fx1OEN1yV/Py5b+U5iyYpksUY/yLw=",
+        "owner": "simple-nixos-mailserver",
+        "repo": "blobs",
+        "rev": "2cccdf1ca48316f2cfd1c9a0017e8de5a7156265",
+        "type": "gitlab"
+      },
+      "original": {
+        "owner": "simple-nixos-mailserver",
+        "repo": "blobs",
+        "type": "gitlab"
+      }
+    },
     "clan-core": {
       "inputs": {
         "data-mesher": "data-mesher",
@@ -7,22 +23,21 @@
         "flake-parts": "flake-parts",
         "nix-darwin": "nix-darwin",
         "nix-select": "nix-select",
-        "nixos-facter-modules": "nixos-facter-modules",
         "nixpkgs": "nixpkgs",
         "sops-nix": "sops-nix",
         "systems": "systems",
         "treefmt-nix": "treefmt-nix"
       },
       "locked": {
-        "lastModified": 1779116933,
-        "narHash": "sha256-BoSQT4vbuWbGcDbXp4nqBd1I9X8r+dLSPtTbc5yJd28=",
-        "rev": "10d86d5d8291fe5162a5535f433a4d50efc4b2bc",
+        "lastModified": 1783497933,
+        "narHash": "sha256-TxmwEews6URFPqOWEHNychtXbFDgLZjbOfEXtvtOm6U=",
+        "rev": "3dc0221ca09033599fe98055e9bbc81bdf32732a",
         "type": "tarball",
-        "url": "https://git.clan.lol/api/v1/repos/clan/clan-core/archive/10d86d5d8291fe5162a5535f433a4d50efc4b2bc.tar.gz"
+        "url": "https://git.clan.lol/api/v1/repos/clan/clan-core/archive/3dc0221ca09033599fe98055e9bbc81bdf32732a.tar.gz"
       },
       "original": {
         "type": "tarball",
-        "url": "https://git.clan.lol/clan/clan-core/archive/25.11.tar.gz"
+        "url": "https://git.clan.lol/clan/clan-core/archive/26.05.tar.gz"
       }
     },
     "data-mesher": {
@@ -41,11 +56,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1767582502,
-        "narHash": "sha256-WVcYGWcAlWzVt38OaTC5i5Q3QkIKJKZsJ7LcQZVVxeE=",
-        "rev": "31f2e3ecf207fd2760e3cebf7c2cf3cb7170ea3d",
+        "lastModified": 1778718524,
+        "narHash": "sha256-pXLoI6Ax0EnUK6r34UM1vibVC7CfTu6j72R2692ZzPs=",
+        "rev": "12c552ad547d87254f33f33bddd1a2cdbeac754d",
         "type": "tarball",
-        "url": "https://git.clan.lol/api/v1/repos/clan/data-mesher/archive/31f2e3ecf207fd2760e3cebf7c2cf3cb7170ea3d.tar.gz"
+        "url": "https://git.clan.lol/api/v1/repos/clan/data-mesher/archive/12c552ad547d87254f33f33bddd1a2cdbeac754d.tar.gz"
       },
       "original": {
         "type": "tarball",
@@ -60,11 +75,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1766150702,
-        "narHash": "sha256-P0kM+5o+DKnB6raXgFEk3azw8Wqg5FL6wyl9jD+G5a4=",
+        "lastModified": 1780290312,
+        "narHash": "sha256-eTAlX0CwgB84Ts3GaBd944A3DRXVMzgA0EqroZBISUo=",
         "owner": "nix-community",
         "repo": "disko",
-        "rev": "916506443ecd0d0b4a0f4cf9d40a3c22ce39b378",
+        "rev": "115e5211780054d8a890b41f0b7734cafad54dfe",
         "type": "github"
       },
       "original": {
@@ -73,6 +88,22 @@
         "type": "github"
       }
     },
+    "flake-compat": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1767039857,
+        "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=",
+        "owner": "NixOS",
+        "repo": "flake-compat",
+        "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
     "flake-parts": {
       "inputs": {
         "nixpkgs-lib": [
@@ -81,11 +112,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1777988971,
-        "narHash": "sha256-qIoWPDs+0/8JecyYgE3gpKQxW/4bLW/gp45vow9ioCQ=",
+        "lastModified": 1778716662,
+        "narHash": "sha256-m1Yf0wZ8j1OHjTc2UwHwyQRSnNeSgLJOd7q5Y45hzi4=",
         "owner": "hercules-ci",
         "repo": "flake-parts",
-        "rev": "0678d8986be1661af6bb555f3489f2fdfc31f6ff",
+        "rev": "f7c1a2d347e4c52d5fb8d10cb4d94b5884e546fb",
         "type": "github"
       },
       "original": {
@@ -94,6 +125,67 @@
         "type": "github"
       }
     },
+    "git-hooks": {
+      "inputs": {
+        "flake-compat": [
+          "nixos-mailserver",
+          "flake-compat"
+        ],
+        "gitignore": "gitignore",
+        "nixpkgs": [
+          "nixos-mailserver",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1778507602,
+        "narHash": "sha256-kTwur1wV+01SdqskVMSo6JMEpg71ps3HpbFY2GsflKs=",
+        "owner": "cachix",
+        "repo": "git-hooks.nix",
+        "rev": "61ab0e80d9c7ab14c256b5b453d8b3fb0189ba0a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "git-hooks.nix",
+        "type": "github"
+      }
+    },
+    "gitignore": {
+      "inputs": {
+        "nixpkgs": [
+          "nixos-mailserver",
+          "git-hooks",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1709087332,
+        "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=",
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "rev": "637db329424fd7e46cf4185293b9cc8c88c95394",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "type": "github"
+      }
+    },
+    "lib-net": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1596309860,
+        "narHash": "sha256-izAzepR/6cDvnRfaa2ceSolMLMwqzQB5x9q62aR5J2g=",
+        "type": "tarball",
+        "url": "https://gist.github.com/duairc/5c9bb3c922e5d501a1edb9e7b3b845ba/archive/3885f7cd9ed0a746a9d675da6f265d41e9fd6704.tar.gz"
+      },
+      "original": {
+        "type": "tarball",
+        "url": "https://gist.github.com/duairc/5c9bb3c922e5d501a1edb9e7b3b845ba/archive/3885f7cd9ed0a746a9d675da6f265d41e9fd6704.tar.gz"
+      }
+    },
     "nix-darwin": {
       "inputs": {
         "nixpkgs": [
@@ -102,20 +194,37 @@
         ]
       },
       "locked": {
-        "lastModified": 1772129556,
-        "narHash": "sha256-Utk0zd8STPsUJPyjabhzPc5BpPodLTXrwkpXBHYnpeg=",
+        "lastModified": 1779036909,
+        "narHash": "sha256-zXcwYQGCT6pzinK+1dBB2ekTVtfxGZAapb3Evdcu4fY=",
         "owner": "nix-darwin",
         "repo": "nix-darwin",
-        "rev": "ebec37af18215214173c98cf6356d0aca24a2585",
+        "rev": "56c666e108467d87d13508936aade6d567f2a501",
         "type": "github"
       },
       "original": {
         "owner": "nix-darwin",
-        "ref": "nix-darwin-25.11",
         "repo": "nix-darwin",
         "type": "github"
       }
     },
+    "nix-lib-net": {
+      "inputs": {
+        "lib-net": "lib-net"
+      },
+      "locked": {
+        "lastModified": 1732312010,
+        "narHash": "sha256-kQuiPSq3Iz/t5RC+QqwHfvj4hKQesm/WGffcQlhhCVE=",
+        "owner": "reo101",
+        "repo": "nix-lib-net",
+        "rev": "0f3412ccb56d783b974341f5f83abfaed7aa8b5b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "reo101",
+        "repo": "nix-lib-net",
+        "type": "github"
+      }
+    },
     "nix-select": {
       "locked": {
         "lastModified": 1763303120,
@@ -129,40 +238,52 @@
         "url": "https://git.clan.lol/clan/nix-select/archive/main.tar.gz"
       }
     },
-    "nixos-facter-modules": {
+    "nixos-mailserver": {
+      "inputs": {
+        "blobs": "blobs",
+        "flake-compat": "flake-compat",
+        "git-hooks": "git-hooks",
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
       "locked": {
-        "lastModified": 1773858690,
-        "narHash": "sha256-oW0/lC0oRG5H5LaK6Rmh9L1wmkn9TbenM4bXwnIEDKA=",
-        "owner": "nix-community",
-        "repo": "nixos-facter-modules",
-        "rev": "139dcef4dfc97009629c445806f197883351ab4a",
-        "type": "github"
+        "lastModified": 1785198005,
+        "narHash": "sha256-falBPi+PJtMx0vwII8L24wjDGXNZMwRiVXsw8WTXLEo=",
+        "owner": "simple-nixos-mailserver",
+        "repo": "nixos-mailserver",
+        "rev": "d357b9f048c5532ec81b0e0034c0b8463d5ddd46",
+        "type": "gitlab"
       },
       "original": {
-        "owner": "nix-community",
-        "repo": "nixos-facter-modules",
-        "type": "github"
+        "owner": "simple-nixos-mailserver",
+        "ref": "nixos-26.05",
+        "repo": "nixos-mailserver",
+        "type": "gitlab"
       }
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1778003029,
-        "narHash": "sha256-q/nkKLDtHIyLjZpKhWk3cSK5IYsFqtMd6UtXF3ddjgA=",
-        "owner": "nixos",
-        "repo": "nixpkgs",
-        "rev": "0c88e1f2bdb93d5999019e99cb0e61e1fe2af4c5",
-        "type": "github"
+        "lastModified": 1780453794,
+        "narHash": "sha256-hhAl/iKiurXPn7rdzDgiSuRB8tqOB6f0buWkh8Y9mkY=",
+        "rev": "6b316287bae2ee04c9b93c8c858d930fd07d7338",
+        "type": "tarball",
+        "url": "https://releases.nixos.org/nixos/26.05/nixos-26.05.1183.6b316287bae2/nixexprs.tar.xz"
       },
       "original": {
-        "owner": "nixos",
-        "ref": "nixos-25.11",
-        "repo": "nixpkgs",
-        "type": "github"
+        "type": "tarball",
+        "url": "https://nixos.org/channels/nixos-26.05/nixexprs.tar.xz"
       }
     },
     "root": {
       "inputs": {
         "clan-core": "clan-core",
+        "flake-parts": [
+          "clan-core",
+          "flake-parts"
+        ],
+        "nix-lib-net": "nix-lib-net",
+        "nixos-mailserver": "nixos-mailserver",
         "nixpkgs": [
           "clan-core",
           "nixpkgs"
@@ -178,11 +299,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1777944972,
-        "narHash": "sha256-VfGRo1qTBKOe3s2gOv8LSoA6Fk19PvBlwQ1ECN0Evn8=",
+        "lastModified": 1780547341,
+        "narHash": "sha256-Gq8KNx5A7hBB3uGJaj6eQfLDIz5YdLu92gqBcvHvoUo=",
         "owner": "Mic92",
         "repo": "sops-nix",
-        "rev": "c591bf665727040c6cc5cb409079acb22dcce33c",
+        "rev": "9ed65852b6257fbeae4355bc24ecfea307ca759a",
         "type": "github"
       },
       "original": {
@@ -193,15 +314,16 @@
     },
     "systems": {
       "locked": {
-        "lastModified": 1681028828,
-        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "lastModified": 1774449309,
+        "narHash": "sha256-brhZ8DmuGtzkCYHJg4HEd602amKm89Y9ytsFZ5uWD1w=",
         "owner": "nix-systems",
         "repo": "default",
-        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "rev": "c29398b59d2048c4ab79345812849c9bd15e9150",
         "type": "github"
       },
       "original": {
         "owner": "nix-systems",
+        "ref": "future-26.11",
         "repo": "default",
         "type": "github"
       }
@@ -213,26 +335,6 @@
           "nixpkgs"
         ]
       },
-      "locked": {
-        "lastModified": 1768158989,
-        "narHash": "sha256-67vyT1+xClLldnumAzCTBvU0jLZ1YBcf4vANRWP3+Ak=",
-        "owner": "numtide",
-        "repo": "treefmt-nix",
-        "rev": "e96d59dff5c0d7fddb9d113ba108f03c3ef99eca",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "treefmt-nix",
-        "type": "github"
-      }
-    },
-    "treefmt-nix_2": {
-      "inputs": {
-        "nixpkgs": [
-          "nixpkgs"
-        ]
-      },
       "locked": {
         "lastModified": 1780220602,
         "narHash": "sha256-eynAfOmbmxJnkp7YewvCEbShNnnYJ9gLLqkzsYtBPeM=",
@@ -246,6 +348,26 @@
         "repo": "treefmt-nix",
         "type": "github"
       }
+    },
+    "treefmt-nix_2": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1784369104,
+        "narHash": "sha256-47cxbcZODibHv3rELFQ9vZly0vUNkND/atn/U7HLeb0=",
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "rev": "df3c0640565d04a0261253cdd89fce78ec50168a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "type": "github"
+      }
     }
   },
   "root": "root",

flake.nix

@@ -1,68 +1,81 @@
 {
-  inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/25.11.tar.gz";
+  inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/26.05.tar.gz";
   inputs.nixpkgs.follows = "clan-core/nixpkgs";
   inputs.treefmt-nix.url = "github:numtide/treefmt-nix";
   inputs.treefmt-nix.inputs.nixpkgs.follows = "nixpkgs";
+  # Simple NixOS Mailserver, pinned to the branch matching clan-core's nixpkgs.
+  inputs.nixos-mailserver.url = "gitlab:simple-nixos-mailserver/nixos-mailserver/nixos-26.05";
+  inputs.nixos-mailserver.inputs.nixpkgs.follows = "nixpkgs";
+
+  inputs.flake-parts.follows = "clan-core/flake-parts";
+  inputs.nix-lib-net.url = "github:reo101/nix-lib-net";
 
   outputs =
-    {
+    inputs@{
       self,
       clan-core,
-      nixpkgs,
+      flake-parts,
       ...
-    }@inputs:
+    }:
     let
-      # Usage see: https://docs.clan.lol
-      clan = clan-core.lib.clan {
-        inherit self;
-        imports = [ ./clan.nix ];
-        specialArgs = { inherit inputs; };
-
-        # Customize nixpkgs
-        # pkgsForSystem =
-        #   system:
-        #   import nixpkgs {
-        #     inherit system;
-        #     config = {
-        #       allowUnfree = true;
-        #     };
-        #     overlays = [];
-        #   };
-        secrets.age.plugins = [
-          "age-plugin-yubikey"
-          "age-plugin-fido2-hmac"
-        ];
-      };
-
+    in
+    flake-parts.lib.mkFlake { inherit inputs; } {
       systems = [
         "x86_64-linux"
         "aarch64-linux"
         "aarch64-darwin"
         "x86_64-darwin"
       ];
-      forAllSystems = nixpkgs.lib.genAttrs systems;
-      pkgsFor = system: clan-core.inputs.nixpkgs.legacyPackages.${system};
-      treefmtFor = system: inputs.treefmt-nix.lib.evalModule (pkgsFor system) ./fmt.nix;
+
+      imports = [
+        inputs.clan-core.flakeModules.default
+
+        ./clan.nix
+        ./modules/clan/flake-module.nix
+      ];
+
+      _module.args.libNet = inputs.nixpkgs.lib.extend inputs.nix-lib-net.overlays.raw;
+
+      perSystem =
+        { system, ... }:
+        let
+          pkgs = clan-core.inputs.nixpkgs.legacyPackages.${system};
+          treefmtEval = inputs.treefmt-nix.lib.evalModule pkgs ./fmt.nix;
+          clanCli = clan-core.packages.${system}.clan-cli;
+          # `clan machines update a b c` normally runs machines in parallel,
+          # which interleaves their output and buries the YubiKey PIN prompts.
+          # This wrapper (first in PATH) runs them one at a time instead; any
+          # flags fall through to the real CLI untouched.
+          clanSequential = pkgs.writeShellScriptBin "clan" ''
+            if [ "$#" -gt 3 ] && [ "$1" = machines ] && [ "$2" = update ]; then
+              shift 2
+              for arg in "$@"; do
+                case "$arg" in
+                  -*) exec ${clanCli}/bin/clan machines update "$@" ;;
+                esac
+              done
+              for machine in "$@"; do
+                ${clanCli}/bin/clan machines update "$machine" || exit
+              done
+              exit 0
+            fi
+            exec ${clanCli}/bin/clan "$@"
+          '';
         in
         {
-      inherit (clan.config) nixosConfigurations nixosModules clanInternals;
-      clan = clan.config;
-
           # `nix fmt` and the `nix flake check` formatting gate.
-      formatter = forAllSystems (system: (treefmtFor system).config.build.wrapper);
-      checks = forAllSystems (system: {
-        formatting = (treefmtFor system).config.build.check self;
-      });
+          formatter = treefmtEval.config.build.wrapper;
+          checks.formatting = treefmtEval.config.build.check self;
 
           # Add the Clan cli tool to the dev shell.
           # Use "nix develop" to enter the dev shell.
-      devShells = forAllSystems (system: {
-        default = (pkgsFor system).mkShell {
+          devShells.default = pkgs.mkShell {
             packages = [
-            clan-core.packages.${system}.clan-cli
-            (treefmtFor system).config.build.wrapper
+              clanSequential
+              clanCli
+              treefmtEval.config.build.wrapper
             ];
           };
-      });
+        };
     };
 }

fmt.nix

@@ -24,6 +24,7 @@
 
       # No formatter, or reformatting would corrupt them.
       "*.zone" # Knot zone files
+      "docs/book.toml" # mdBook config; no TOML formatter enabled
       "flake.lock"
       ".envrc"
       ".gitignore"

inventory.json

@@ -8,6 +8,15 @@
     },
     "ns2": {
       "installedAt": 1781418857
+    },
+    "mx1": {
+      "installedAt": 1781757322
+    },
+    "web01": {
+      "installedAt": 1781983723
+    },
+    "gw-cnx-1": {
+      "installedAt": 1785494267
     }
   }
 }

machines/control/configuration.nix

@@ -1,21 +1,29 @@
+{ config, ... }:
+let
+  hosts = import ../../modules/hosts.nix;
+in
 {
   imports = [
     ../../modules/hetzner-firewall.nix
     ../../modules/static-ipv6.nix
     ../../modules/monitoring/exporters.nix
     ../../modules/monitoring/server.nix
+    ../../modules/monitoring/blackbox.nix
+    ../../modules/monitoring/alerts.nix
+    ../../modules/monitoring/parsedmarc.nix
+    ../../modules/docs.nix
   ];
 
   clan.core.sops.defaultGroups = [ "admins" ];
 
-  # Public IPv6; SLAAC doesn't bring it up here.
+  # Public IPv6 (from modules/hosts.nix); SLAAC doesn't bring it up here.
   cnx.staticIPv6 = {
     enable = true;
-    address = "2a01:4f9:c013:e6d0::1";
+    address = hosts.${config.networking.hostName}.ipv6;
   };
 
   time.timeZone = "Etc/GMT-3"; # UTC+3 (fixed offset, no DST)
-  services.timesyncd.enable = true;
+  services.chrony.enable = true;
 
   # Public Hetzner Cloud firewalls, synced from this config on every deploy.
   # Rules live in their own data file; see that file for the no-public-SSH note.

machines/gw-cnx-1/configuration.nix

@@ -0,0 +1,91 @@
+# Site gateway Chiang Mai (site 1): Topton 1U, Intel N300, 4x i226-V 2.5G.
+# Port roles below use the expected igc names — verify against facter.json
+# after the first install and adjust if the box enumerates differently.
+{ config, lib, ... }:
+{
+  imports = [
+    # ../../modules/router
+    ../../modules/monitoring/exporters.nix
+    (import ../../modules/dns/acme-gw-secret.nix "gw-cnx-1")
+  ];
+
+  clan.core.sops.defaultGroups = [ "admins" ];
+
+  # Until the install generates facter.json (which normally provides this).
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+
+  # ZFS (disko.nix) needs a stable machine-unique hostId; derive it from the
+  # hostname so every gateway gets one for free when copied for a new site.
+  networking.hostId = builtins.substring 0 8 (
+    builtins.hashString "sha256" config.networking.hostName
+  );
+
+  # cnx.router = {
+  #   enable = true;
+  #   site = "cnx";
+  #   siteId = 1;
+  #   wan.interface = "enp1s0";
+  #   wan.vlanId = null; # this ISP runs PPPoE untagged on the port
+  #   trunkPorts = [
+  #     "enp2s0"
+  #     # "enp3s0" # STAGING: serves as the uplink below until cutover
+  #   ];
+  #   # Dedicated on-site recovery port: untagged mgmt, always available even
+  #   # if the switch config is broken.
+  #   accessPorts.enp4s0 = "mgmt";
+  #   # Replaces the newedge.house OPNsense box; renumbered to the fleet
+  #   # convention (10.1.<vlanId>.0/24, router .1, pool .100-.199). The old
+  #   # untagged LAN becomes tagged mgmt — infra switch ports get PVID 10.
+  #   vlans = {
+  #     mgmt = {
+  #       id = 10; # 10.1.10.0/24 — servers, APs, switches, Omada, admin
+  #       dhcp.reservations.storinator01 = {
+  #         hwAddress = "7c:c2:55:e0:d6:40";
+  #         ipAddress = "10.1.10.53";
+  #       };
+  #     };
+  #     lan.id = 20; # 10.1.20.0/24 — trusted clients
+  #     iot.id = 40; # 10.1.40.0/24
+  #     voip.id = 50; # 10.1.50.0/24
+  #     dmz.id = 60; # 10.1.60.0/24
+  #     unit1.id = 110; # 10.1.110.0/24
+  #     unit2.id = 120; # 10.1.120.0/24
+  #     unit3.id = 130; # 10.1.130.0/24
+  #     unit4.id = 140; # 10.1.140.0/24
+  #     unit5 = {
+  #       id = 150; # 10.1.150.0/24
+  #       dhcp.reservations.newt = {
+  #         hwAddress = "7c:d3:0a:21:58:0b";
+  #         ipAddress = "10.1.150.22";
+  #       };
+  #     };
+  #   };
+  #   # This site runs the Omada controller for its APs/switches.
+  #   omada.enable = true;
+  #
+  #   # Internal reverse proxy: real wildcard cert *.cnx1.cnx.network; Blocky
+  #   # resolves the names to the router's LAN address for mgmt+lan clients.
+  #   proxy = {
+  #     enable = true;
+  #     services.omada = {
+  #       # Omada's UI is HTTPS with a self-signed cert on the host network.
+  #       backend = "https://127.0.0.1:8043";
+  #       insecureSkipVerify = true;
+  #     };
+  #   };
+  # };
+
+  # STAGING (remove at cutover, and restore enp3s0 to trunkPorts): DHCP-client
+  # uplink into the existing OPNsense LAN so the box has internet + mesh while
+  # it runs alongside the old router. Default-deny firewall on this interface
+  # (it's in no VLAN zone); PPPoE simply retries until the WAN port is cabled.
+  # Do NOT connect the trunk ports to the production switch while staging —
+  # Kea on tag 10 would fight the OPNsense LAN DHCP in one broadcast domain.
+  # systemd.network.networks."05-staging" = {
+  #   matchConfig.Name = "enp3s0";
+  #   networkConfig.DHCP = "ipv4";
+  # };
+
+  time.timeZone = "Etc/GMT-7"; # UTC+7 (Thailand, fixed offset, no DST)
+  services.chrony.enable = true;
+}

machines/gw-cnx-1/disko.nix

@@ -0,0 +1,97 @@
+{ ... }:
+let
+  hashDisk = disk: "os-${builtins.substring 0 5 (builtins.hashString "sha256" disk)}";
+  os = "/dev/disk/by-id/ata-TS256GMSA230S_J927900030";
+in
+{
+
+  boot.loader = {
+    systemd-boot = {
+      enable = true;
+    };
+    efi = {
+      canTouchEfiVariables = true;
+    };
+  };
+
+  boot.zfs.forceImportRoot = true;
+
+  disko.devices = {
+    disk = {
+      "os-${hashDisk os}" = {
+        type = "disk";
+        device = os;
+        content = {
+          type = "gpt";
+          partitions = {
+            ESP = {
+              size = "1G";
+              type = "EF00";
+              content = {
+                type = "filesystem";
+                format = "vfat";
+                mountpoint = "/boot";
+                mountOptions = [ "nofail" ];
+              };
+            };
+            system = {
+              size = "100%";
+              content = {
+                type = "zfs";
+                pool = "zroot";
+              };
+            };
+            swap = {
+              size = "16G";
+              content = {
+                type = "swap";
+              };
+            };
+          };
+        };
+      };
+    };
+    zpool = {
+      zroot = {
+        type = "zpool";
+        rootFsOptions = {
+          mountpoint = "none";
+          compression = "lz4";
+          acltype = "posixacl";
+          xattr = "sa";
+          "com.sun:auto-snapshot" = "true";
+        };
+        options.ashift = "12";
+        datasets = {
+          "root" = {
+            type = "zfs_fs";
+            options.mountpoint = "none";
+          };
+          "root/nixos" = {
+            type = "zfs_fs";
+            options.mountpoint = "/";
+            mountpoint = "/";
+          };
+          "root/home" = {
+            type = "zfs_fs";
+            options.mountpoint = "/home";
+            mountpoint = "/home";
+          };
+          "root/tmp" = {
+            type = "zfs_fs";
+            mountpoint = "/tmp";
+            options = {
+              mountpoint = "/tmp";
+              sync = "disabled";
+            };
+          };
+          "root/var" = {
+            type = "zfs_fs";
+            options.mountpoint = "/var";
+            mountpoint = "/var";
+          };
+        };
+      };
+    };
+  };
+}

machines/mx1/configuration.nix

@@ -0,0 +1,23 @@
+{ config, inputs, ... }:
+let
+  hosts = import ../../modules/hosts.nix;
+in
+{
+  imports = [
+    inputs.nixos-mailserver.nixosModules.default
+    ../../modules/mail.nix
+    ../../modules/static-ipv6.nix
+    ../../modules/monitoring/exporters.nix
+  ];
+
+  clan.core.sops.defaultGroups = [ "admins" ];
+
+  # Public IPv6 (from modules/hosts.nix); SLAAC doesn't bring it up here.
+  cnx.staticIPv6 = {
+    enable = true;
+    address = hosts.${config.networking.hostName}.ipv6;
+  };
+
+  time.timeZone = "Etc/GMT-8"; # UTC+8 (Singapore, fixed offset, no DST)
+  services.chrony.enable = true;
+}

machines/mx1/disko.nix

@@ -0,0 +1,50 @@
+# ---
+# schema = "single-disk"
+# [placeholders]
+# mainDisk = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_117494657"
+# ---
+# This file was automatically generated!
+# CHANGING this configuration requires wiping and reinstalling the machine
+{
+
+  boot.loader.grub.efiSupport = true;
+  boot.loader.grub.efiInstallAsRemovable = true;
+  boot.loader.grub.enable = true;
+  disko.devices = {
+    disk = {
+      main = {
+        name = "main-5a0919ffeb6044a39b7d44bba8895ff2";
+        device = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_117494657";
+        type = "disk";
+        content = {
+          type = "gpt";
+          partitions = {
+            "boot" = {
+              size = "1M";
+              type = "EF02"; # for grub MBR
+              priority = 1;
+            };
+            ESP = {
+              type = "EF00";
+              size = "500M";
+              content = {
+                type = "filesystem";
+                format = "vfat";
+                mountpoint = "/boot";
+                mountOptions = [ "umask=0077" ];
+              };
+            };
+            root = {
+              size = "100%";
+              content = {
+                type = "filesystem";
+                format = "ext4";
+                mountpoint = "/";
+              };
+            };
+          };
+        };
+      };
+    };
+  };
+}

machines/ns1/configuration.nix

@@ -1,30 +1,49 @@
-{ config, pkgs, ... }:
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
 let
   domains = import ../../modules/dns/domains.nix;
+  mesh = import ../../modules/mesh-hosts.nix { inherit config lib; };
+  hosts = import ../../modules/hosts.nix;
 in
 {
   imports = [
     ../../modules/dns/authoritative.nix
+    ../../modules/dns/acme-mx1-secret.nix
+    ../../modules/dns/acme-web01-secret.nix
+    (import ../../modules/dns/acme-gw-secret.nix "gw-cnx-1")
     ../../modules/static-ipv6.nix
     ../../modules/monitoring/exporters.nix
   ];
 
   clan.core.sops.defaultGroups = [ "admins" ];
 
-  # Public IPv6 (matches the ns1 AAAA glue); SLAAC doesn't bring it up here.
+  # Knot's state dir holds the non-regenerable DNSSEC key material (KSK/ZSK
+  # private keys in the KASP keystore). Declaring it as clan state makes the
+  # borgbackup client back it up; losing it forces an emergency DS rollover at
+  # the registrar. mode 0700 owned by knot, but borg runs as root so it reads it.
+  clan.core.state.knot.folders = [ "/var/lib/knot" ];
+
+  # The borgbackup repo is addressed as `borg@control`; mesh peers have no name
+  # resolution, so map the control machine name to its ZeroTier mesh address.
+  networking.hosts.${mesh.hosts.control} = [ "control" ];
+
+  # Public IPv6 (from modules/hosts.nix; matches the ns1 AAAA glue); SLAAC
+  # doesn't bring it up here.
   cnx.staticIPv6 = {
     enable = true;
-    address = "2a01:4f8:c014:b5c5::1";
+    address = hosts.${config.networking.hostName}.ipv6;
   };
 
   time.timeZone = "Etc/GMT-1"; # UTC+1 (fixed offset, no DST)
-  services.timesyncd.enable = true;
+  services.chrony.enable = true;
 
-  # ACME DNS-01 (RFC 2136): a dedicated TSIG key, scoped to ns1 only, that an
-  # external ACME client uses to write _acme-challenge TXT records. acl_acme
-  # (referenced by each zone below) limits the key to TXT updates at or under
-  # _acme-challenge.<zone>; Knot then signs the record and transfers it to ns2,
-  # which never needs this key. Retrieve the secret for the client with:
+  # ACME DNS-01 (RFC 2136), general key. A dedicated TSIG key scoped by acl_acme
+  # (referenced by every zone below) to TXT updates at or under _acme-challenge.
+  # Retrieve the client config with:
   #   clan vars get ns1 dns-acme-tsig/acme.conf
   clan.core.vars.generators.dns-acme-tsig = {
     files."acme.conf" = {
@@ -38,8 +57,66 @@ in
     '';
   };
 
+  # ACME DNS-01, dedicated mx1 key. A *separate* TSIG key (acme_mx1) that only
+  # mx1 holds, rendered from the shared secret (generator dns-acme-mx1-secret,
+  # imported above). acl_acme_mx1 scopes it to TXT updates at exactly
+  # _acme-challenge.{mx1,mta-sts,mail} (the mail cert and its MTA-STS + client-
+  # alias SANs), and it is attached only to the cnx.email zone below — so this
+  # credential can write nothing but mx1's own cert challenges.
+  clan.core.vars.generators.dns-acme-mx1-knot = {
+    files."acme.conf" = {
+      secret = true;
+      owner = "knot";
+      group = "knot";
+    };
+    dependencies = [ "dns-acme-mx1-secret" ];
+    script = ''
+      printf 'key:\n  - id: acme_mx1\n    algorithm: hmac-sha256\n    secret: %s\n' \
+        "$(cat "$in"/dns-acme-mx1-secret/secret)" > "$out"/acme.conf
+    '';
+  };
+
+  # ACME DNS-01, dedicated web01 key. A *separate* TSIG key (acme_web01) that only
+  # web01 holds, rendered from the shared secret (generator dns-acme-web01-secret,
+  # imported above). acl_acme_web01 scopes it to TXT updates at _acme-challenge on
+  # the cnx.network zone — the owner the wildcard *.cnx.network challenge uses — so
+  # this credential can write nothing but web01's own cert challenges.
+  clan.core.vars.generators.dns-acme-web01-knot = {
+    files."acme.conf" = {
+      secret = true;
+      owner = "knot";
+      group = "knot";
+    };
+    dependencies = [ "dns-acme-web01-secret" ];
+    script = ''
+      printf 'key:\n  - id: acme_web01\n    algorithm: hmac-sha256\n    secret: %s\n' \
+        "$(cat "$in"/dns-acme-web01-secret/secret)" > "$out"/acme.conf
+    '';
+  };
+
+  # ACME DNS-01, dedicated gateway keys. Same pattern as web01: each gateway
+  # holds its own TSIG key (acme_gw_<site>_<n>), rendered from the shared
+  # secret generator imported above. acl_acme_gw_<site>_<n> scopes it to TXT
+  # updates at _acme-challenge.<site><n> on cnx.network — the owner its internal
+  # wildcard *.<site><n>.cnx.network challenge uses — and nothing else.
+  clan.core.vars.generators.dns-acme-gw-cnx-1-knot = {
+    files."acme.conf" = {
+      secret = true;
+      owner = "knot";
+      group = "knot";
+    };
+    dependencies = [ "dns-acme-gw-cnx-1-secret" ];
+    script = ''
+      printf 'key:\n  - id: acme_gw_cnx_1\n    algorithm: hmac-sha256\n    secret: %s\n' \
+        "$(cat "$in"/dns-acme-gw-cnx-1-secret/secret)" > "$out"/acme.conf
+    '';
+  };
+
   services.knot.keyFiles = [
     config.clan.core.vars.generators.dns-acme-tsig.files."acme.conf".path
+    config.clan.core.vars.generators.dns-acme-mx1-knot.files."acme.conf".path
+    config.clan.core.vars.generators.dns-acme-web01-knot.files."acme.conf".path
+    config.clan.core.vars.generators.dns-acme-gw-cnx-1-knot.files."acme.conf".path
   ];
 
   services.knot.settings.acl = [
@@ -52,6 +129,41 @@ in
       "update-owner-match" = "sub-or-equal";
       "update-owner-name" = [ "_acme-challenge" ];
     }
+    {
+      id = "acl_acme_mx1";
+      key = "acme_mx1";
+      action = [ "update" ];
+      "update-type" = [ "TXT" ];
+      "update-owner" = "name";
+      "update-owner-match" = "sub-or-equal";
+      "update-owner-name" = [
+        "_acme-challenge.mx1"
+        "_acme-challenge.mta-sts"
+        "_acme-challenge.mail"
+      ];
+    }
+    {
+      id = "acl_acme_web01";
+      key = "acme_web01";
+      action = [ "update" ];
+      "update-type" = [ "TXT" ];
+      "update-owner" = "name";
+      "update-owner-match" = "sub-or-equal";
+      # Wildcard *.cnx.network places its challenge at _acme-challenge.cnx.network,
+      # i.e. _acme-challenge at the cnx.network apex (where this acl is attached).
+      "update-owner-name" = [ "_acme-challenge" ];
+    }
+    {
+      id = "acl_acme_gw_cnx_1";
+      key = "acme_gw_cnx_1";
+      action = [ "update" ];
+      "update-type" = [ "TXT" ];
+      "update-owner" = "name";
+      "update-owner-match" = "sub-or-equal";
+      # The internal wildcard *.cnx1.cnx.network places its challenge at
+      # _acme-challenge.cnx1.cnx.network.
+      "update-owner-name" = [ "_acme-challenge.cnx1" ];
+    }
   ];
 
   # Automatic DNSSEC signing policy (primary only). ECDSA P-256/SHA-256 with
@@ -80,9 +192,16 @@ in
     "dnssec-signing" = true;
     "dnssec-policy" = "cnx";
     notify = [ "ns2" ];
+    # ns2 transfers; acme_ddns does general DNS-01 updates. The dedicated
+    # acme_mx1 key is attached only to cnx.email, so it can't touch other zones.
     acl = [
       "acl_ns2"
       "acl_acme"
-    ]; # ns2 transfers; acme_ddns key does DNS-01 updates
+    ]
+    ++ lib.optionals (d == "cnx.email") [ "acl_acme_mx1" ]
+    ++ lib.optionals (d == "cnx.network") [
+      "acl_acme_web01"
+      "acl_acme_gw_cnx_1"
+    ];
   }) domains;
 }

machines/ns2/configuration.nix

@@ -1,6 +1,7 @@
-{ ... }:
+{ config, ... }:
 let
   domains = import ../../modules/dns/domains.nix;
+  hosts = import ../../modules/hosts.nix;
 in
 {
   imports = [
@@ -11,14 +12,15 @@ in
 
   clan.core.sops.defaultGroups = [ "admins" ];
 
-  # Public IPv6 (matches the ns2 AAAA glue); SLAAC doesn't bring it up here.
+  # Public IPv6 (from modules/hosts.nix; matches the ns2 AAAA glue); SLAAC
+  # doesn't bring it up here.
   cnx.staticIPv6 = {
     enable = true;
-    address = "2a01:4f9:c014:6d87::1";
+    address = hosts.${config.networking.hostName}.ipv6;
   };
 
   time.timeZone = "Etc/GMT-3"; # UTC+3 (fixed offset, no DST)
-  services.timesyncd.enable = true;
+  services.chrony.enable = true;
 
   # ns2 = secondary (slave): pulls every zone from ns1 and accepts its NOTIFY.
   services.knot.settings.zone = map (d: {

machines/web01/configuration.nix

@@ -0,0 +1,22 @@
+{ config, ... }:
+let
+  hosts = import ../../modules/hosts.nix;
+in
+{
+  imports = [
+    ../../modules/static-ipv6.nix
+    ../../modules/monitoring/exporters.nix
+    ../../modules/web-proxy.nix
+  ];
+
+  clan.core.sops.defaultGroups = [ "admins" ];
+
+  # Public IPv6 (from modules/hosts.nix); SLAAC doesn't bring it up here.
+  cnx.staticIPv6 = {
+    enable = true;
+    address = hosts.${config.networking.hostName}.ipv6;
+  };
+
+  time.timeZone = "Etc/GMT-8"; # UTC+8 (Singapore, fixed offset, no DST)
+  services.chrony.enable = true;
+}

machines/web01/disko.nix

@@ -0,0 +1,50 @@
+# ---
+# schema = "single-disk"
+# [placeholders]
+# mainDisk = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_108706511"
+# ---
+# This file was automatically generated!
+# CHANGING this configuration requires wiping and reinstalling the machine
+{
+
+  boot.loader.grub.efiSupport = true;
+  boot.loader.grub.efiInstallAsRemovable = true;
+  boot.loader.grub.enable = true;
+  disko.devices = {
+    disk = {
+      main = {
+        name = "main-ddd46ebf135244608078712d6ec76691";
+        device = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_108706511";
+        type = "disk";
+        content = {
+          type = "gpt";
+          partitions = {
+            "boot" = {
+              size = "1M";
+              type = "EF02"; # for grub MBR
+              priority = 1;
+            };
+            ESP = {
+              type = "EF00";
+              size = "500M";
+              content = {
+                type = "filesystem";
+                format = "vfat";
+                mountpoint = "/boot";
+                mountOptions = [ "umask=0077" ];
+              };
+            };
+            root = {
+              size = "100%";
+              content = {
+                type = "filesystem";
+                format = "ext4";
+                mountpoint = "/";
+              };
+            };
+          };
+        };
+      };
+    };
+  };
+}

modules/clan/flake-module.nix

@@ -0,0 +1,26 @@
+{
+  inputs,
+  ...
+}:
+{
+  imports =
+    let
+      # Get all subdirectories in the current directory
+      dirContents = builtins.readDir ./.;
+
+      # Filter to include only directories that have a flake-module.nix file
+      # and exclude special directories like 'result'
+      validModuleDirs = builtins.filter (
+        name:
+        name != "result"
+        && dirContents.${name} == "directory"
+        && builtins.pathExists (./. + "/${name}/flake-module.nix")
+      ) (builtins.attrNames dirContents);
+
+      # Create import paths for each valid directory
+      imports = (map (name: ./. + "/${name}/flake-module.nix") validModuleDirs) ++ [
+        inputs.clan-core.flakeModules.testModule
+      ];
+    in
+    imports;
+}

modules/clan/gw-router/crowdsec.nix

@@ -0,0 +1,59 @@
+{
+  roles.default.perInstance = { ... }: {
+    nixosModule =
+      {
+        config,
+        ...
+      }:
+      let
+
+        dir = config.clan.core.settings.directory;
+        instance = "zerotier";
+
+        networkId = builtins.readFile "${dir}/vars/shared/zerotier-network-${instance}/network-id/value";
+        full = "fd" + networkId + "9993";
+        hextet = i: builtins.substring (i * 4) 4 full;
+        subnetZtier = "${hextet 0}:${hextet 1}:${hextet 2}:${hextet 3}:${hextet 4}:${builtins.substring 20 2 full}00::/88";
+      in
+      {
+
+        services.crowdsec = {
+          enable = true;
+          autoUpdateService = true;
+          hub.collections = [
+            "crowdsecurity/linux"
+            "crowdsecurity/sshd"
+          ];
+          localConfig = {
+            acquisitions = [
+              {
+                source = "journalctl";
+                journalctl_filter = [ "_SYSTEMD_UNIT=sshd.service" ];
+                labels.type = "syslog";
+              }
+            ];
+            # Never ban the ZeroTier mesh — it is the only admin path to these
+            # boxes (no public SSH), so a false positive would lock us out.
+            # Parser-stage whitelist: mesh events are dropped before any scenario.
+            parsers.s02Enrich = [
+              {
+                name = "cnx/mesh-whitelist";
+                description = "Whitelist the ZeroTier management mesh";
+                whitelist = {
+                  reason = "ZeroTier mesh is the admin path";
+                  cidr = [ subnetZtier ];
+                };
+              }
+            ];
+          };
+        };
+
+        services.crowdsec-firewall-bouncer = {
+          enable = true;
+          registerBouncer.enable = true;
+          settings.mode = "nftables";
+        };
+
+      };
+  };
+}

modules/clan/gw-router/default.nix

@@ -0,0 +1,371 @@
+{ ... }:
+{
+  _class = "clan.service";
+  manifest.name = "gw-router";
+  manifest.description = "A gateway router service to configure most of a router features";
+  manifest.readme = "A gateway router service to configure most of a router features";
+  manifest.categories = [ "System" ];
+
+  roles.default = {
+    description = "Site gateway router role";
+
+    interface =
+      { lib, config, ... }:
+      let
+        vlanModule =
+          { ... }:
+          {
+            options = {
+              id = lib.mkOption {
+                type = lib.types.ints.between 1 4094;
+                description = "802.1Q VLAN id.";
+              };
+              address = lib.mkOption {
+                type = lib.types.str;
+                example = "10.0.10.1";
+                description = "Router address on this VLAN.";
+              };
+              prefixLength = lib.mkOption {
+                type = lib.types.ints.between 8 30;
+                default = 24;
+              };
+              subnet = lib.mkOption {
+                type = lib.types.str;
+                example = "10.0.10.0/24";
+                description = "The VLAN's network in CIDR form (must contain `address`).";
+              };
+              dhcp = {
+                enable = lib.mkOption {
+                  type = lib.types.bool;
+                  default = true;
+                };
+                pool = {
+                  from = lib.mkOption {
+                    type = lib.types.str;
+                    example = "10.0.10.100";
+                  };
+                  to = lib.mkOption {
+                    type = lib.types.str;
+                    example = "10.0.10.199";
+                  };
+                };
+                leaseTime = lib.mkOption {
+                  type = lib.types.ints.positive;
+                  default = 86400;
+                  description = ''
+                    Lease validity in seconds. Lower it for high-churn networks,
+                    e.g. public-WiFi guest VLANs (3600-7200), so the pool recycles.
+                  '';
+                };
+                fixedIPs = lib.mkOption {
+                  type = lib.types.attrsOf (
+                    lib.types.submodule {
+                      options = {
+                        hwAddress = lib.mkOption {
+                          type = lib.types.str;
+                          example = "aa:bb:cc:dd:ee:ff";
+                          description = "Client MAC address.";
+                        };
+                        ipAddress = lib.mkOption {
+                          type = lib.types.str;
+                          description = "Fixed address handed to this client (inside the VLAN's subnet, outside the pool).";
+                        };
+                      };
+                    }
+                  );
+                  default = { };
+                  description = "Static DHCP leases; the attribute name becomes the client's hostname.";
+                };
+              };
+              allowedWAN = lib.mkOption {
+                type = lib.types.bool;
+                default = true;
+                description = "Whether clients on this VLAN may reach the internet.";
+              };
+            };
+          };
+      in
+      {
+        options = {
+          wan = {
+            interface = lib.mkOption {
+              type = lib.types.str;
+              description = "Physical WAN port the PPPoE session runs on.";
+            };
+
+            vlanId = lib.mkOption {
+              type = lib.types.nullOr (lib.types.ints.between 1 4094);
+              default = null;
+              description = ''
+                802.1Q tag the ISP requires for the PPPoE session (AIS Thailand: 10);
+                null for untagged PPPoE directly on the port. Unrelated to the LAN
+                VLANs — this tag exists only on the WAN port.
+              '';
+            };
+
+            macAddress = lib.mkOption {
+              type = lib.types.nullOr lib.types.str;
+              default = null;
+              example = "aa:bb:cc:dd:ee:ff";
+              description = ''
+                Spoofed MAC for the WAN port, e.g. to keep the MAC the ISP has
+                pinned (cloned from the old router). null keeps the hardware MAC.
+              '';
+            };
+          };
+
+          enableOmada = lib.mkOption {
+            type = lib.types.bool;
+            default = false;
+            description = "TP-Link Omada SDN controller (podman container)";
+          };
+
+          speedtest = {
+            enable = {
+              type = lib.types.bool;
+              default = false;
+              description = "Periodic WAN speed test so ISP degradation shows up as a trend. A timer runs librespeed-cli and writes the results as Prometheus metrics into node_exporter's textfile collector";
+            };
+            interval = lib.mkOption {
+              type = lib.types.str;
+              default = "hourly";
+              description = "systemd OnCalendar spec for the WAN speed test.";
+            };
+          };
+
+          proxy = {
+
+            enable = lib.mkEnableOption "internal reverse proxy (Caddy, wildcard cert via DNS-01)";
+
+            services = lib.mkOption {
+              type = lib.types.attrsOf (
+                lib.types.submodule {
+
+                  options = {
+                    backend = lib.mkOption {
+                      type = lib.types.str;
+                      example = "https://127.0.0.1:8043";
+                      description = "URL Caddy forwards to (internal/mesh address).";
+                    };
+                    insecureSkipVerify = lib.mkOption {
+                      type = lib.types.bool;
+                      default = false;
+                      description = "Skip TLS verification towards the backend.";
+                    };
+                  };
+
+                }
+              );
+              default = { };
+              description = "Proxied services";
+            };
+
+            allowVlans = lib.mkOption {
+              type = lib.types.listOf lib.types.str;
+              default = [
+                "mgmt"
+                "lan"
+              ];
+              description = "VLANs whose clients may reach the proxy (443, plus 80 for the redirect).";
+            };
+          };
+
+          vlans = lib.mkOption {
+            type = lib.types.attrsOf (lib.types.submodule vlanModule);
+            description = "VLANs setup for this router";
+          };
+
+          trunkPorts = lib.mkOption {
+            type = lib.types.listOf lib.types.str;
+            description = "LAN ports carrying all VLANs tagged (incl. any 10G SFP+ ports).";
+          };
+
+          accessPorts = lib.mkOption {
+            type = lib.types.attrsOf (
+              lib.types.submodule {
+                options = {
+                  vlanId = lib.mkOption {
+                    type = lib.types.int;
+                    description = "Untagged traffic in from the device gets tagged VLANs inside the bridge, and VLANs traffic going back out to the device gets untagged, so the device itself never has to know VLANs exist.";
+                  };
+                };
+              }
+            );
+            description = ''
+              Ports acting as untagged access ports on a single VLAN (port name ->
+              VLAN id). Frames are untagged on the wire; the bridge tags them with
+              the VLAN's PVID. Use for an always-available on-site mgmt port.
+            '';
+          };
+
+          upLinkPorts = lib.mkOption {
+            type = lib.types.listOf lib.types.str;
+            description = "DHCP-client uplink into the existing router LAN so this machine has internet";
+          };
+        };
+      };
+
+    perInstance =
+      { settings, ... }:
+      {
+        nixosModule =
+          {
+            pkgs,
+            lib,
+            ...
+          }:
+          let
+            vlanIf = name: "vlan-${name}";
+          in
+          {
+
+            networking.useNetworkd = true;
+            networking.useDHCP = false;
+            systemd.network.enable = true;
+
+            systemd.network.netdevs = {
+              "20-br0" = {
+                netdevConfig = {
+                  Name = "br0";
+                  Kind = "bridge";
+                };
+                bridgeConfig = {
+                  VLANFiltering = true;
+                };
+              };
+            }
+            // lib.optionalAttrs (settings.wan.vlanId != null) {
+              "15-wan-lan" = {
+                netdevConfig = {
+                  Name = "wan-vlan";
+                  Kind = "vlan";
+                };
+                vlanConfig.Id = settings.wan.vlanId;
+              };
+            }
+            // lib.mapAttrs' (
+              name: vlan:
+              lib.nameValuePair "30-${vlanIf name}" {
+                netdevConfig = {
+                  Name = vlanIf name;
+                  Kind = "vlan";
+                };
+                vlanConfig.Id = vlan.id;
+              }
+            ) settings.vlans;
+
+            systemd.network.networks =
+              let
+                allVLANs = lib.mapAttrsToList (_: vlan: { VLAN = vlan.id; }) settings.vlans;
+              in
+              {
+                "10-wan" = {
+                  matchConfig.Name = settings.wan.interface;
+                  networkConfig.LinkLocalAddressing = "no";
+                  vlan = lib.optional (settings.wan.vlanId != null) "wan-wlan";
+                  linkConfig = {
+                    RequiredForOnline = "carrier";
+                  }
+                  // lib.optionalAttrs (settings.wan.macAddress != null) {
+                    MACAddress = settings.wan.macAddress;
+                  };
+                };
+              }
+
+              // lib.optionalAttrs (settings.wan.vlanId != null) {
+                "15-wan-lan" = {
+                  matchConfig.Name = "wan-vlan";
+                  networkConfig.LinkLocalAddressing = "no";
+                  linkConfig.RequiredForOnline = "no";
+                };
+              }
+
+              // {
+                "20-br0" = {
+                  matchConfig.Name = "br0";
+                  networkConfig.LinkLocalAddressing = "no";
+                  vlan = lib.mapAttrsToList (name: _: vlanIf name) settings.vlans;
+                  bridgeVLANs = allVLANs;
+                  linkConfig.RequiredForOnline = "no";
+                };
+              }
+
+              // lib.listToAttrs (
+                map (port: {
+                  name = "25-trunk-${port}";
+                  value = {
+                    matchConfig.Name = port;
+                    networkConfig.Bridge = "br0";
+                    bridgeVLANs = allVLANs;
+                    linkConfig.RequiredForOnline = "no";
+                  };
+                }) settings.trunkPorts
+              )
+
+              // lib.mapAttrs' (
+                iface: port:
+                lib.nameValuePair "25-access-${iface}" {
+                  matchConfig.Name = iface;
+                  networkConfig.Bridge = "br0";
+                  bridgeVLANs = [
+                    {
+                      VLAN = port.vlanId;
+                      PVID = port.vlanId;
+                      EgressUntagged = port.vlanId;
+                    }
+                  ];
+                  linkConfig.RequiredForOnline = "no";
+                }
+              ) settings.accessPorts
+
+              // lib.mapAttrs' (
+                name: vlan:
+                lib.nameValuePair "40-${vlanIf name}" {
+                  matchConfig.Name = vlanIf name;
+                  address = [ "${vlan.address}/${toString vlan.prefixLength}" ];
+                  networkConfig = {
+                    IPv6AcceptRA = false;
+                    IPv6SendRA = true;
+                    DHCPPrefixDelegation = true;
+                  };
+                  dhcpPrefixDelegationConfig.SubnetId = "auto";
+                  linkConfig.RequiredForOnline = "no";
+                }
+              ) settings.vlans
+
+              // lib.listToAttrs (
+                map (port: {
+                  name = "15-uplink-${port}";
+                  value = {
+                    matchConfig.Name = "${port}";
+                    networkConfig.DHCP = "ipv4";
+                  };
+                }) settings.upLinkPorts
+              );
+
+            environment.systemPackages = with pkgs; [
+              tcpdump
+              mtr
+              ethtool
+              conntrack-tools
+              knot-dns
+              iftop
+              librespeed-cli
+            ];
+
+          };
+      };
+  };
+
+  imports = [
+    ./crowdsec.nix
+    ./dns-dhcp.nix
+    ./firewall.nix
+    ./iperf.nix
+    ./ipv6.nix
+    ./omada.nix
+    ./pppoe.nix
+    ./proxy.nix
+    ./speedtest.nix
+  ];
+}

modules/clan/gw-router/dns-dhcp.nix

@@ -0,0 +1,81 @@
+{
+  roles.default.perInstance = { settings, ... }: {
+    nixosModule =
+      {
+
+        lib,
+        ...
+      }:
+      let
+        dhcpVlans = lib.filterAttrs (_: vlan: vlan.dhcp.enable) settings.vlans;
+      in
+      {
+        services.kea.dhcp4 = {
+          enable = true;
+          settings = {
+            interfaces-config.interfaces = lib.mapAttrsToList (name: _: "vlan-${name}") dhcpVlans;
+            lease-database = {
+              type = "memfile";
+              persist = true;
+              name = "/var/lib/kea/dhcp4.leases";
+            };
+            valid-lifetime = 86400;
+            subnet4 = lib.mapAttrsToList (name: vlan: {
+              id = vlan.id;
+              subnet = vlan.subnet;
+              interface = "vlan-${name}";
+              valid-lifetime = vlan.dhcp.leaseTime;
+              pools = [ { pool = "${vlan.dhcp.pool.from} - ${vlan.dhcp.pool.to}"; } ];
+              reservations = lib.mapAttrsToList (host: res: {
+                hostname = host;
+                hw-address = res.hwAddress;
+                ip-address = res.ipAddress;
+              }) vlan.dhcp.fixedIPs;
+              option-data = [
+                {
+                  name = "routers";
+                  data = vlan.address;
+                }
+                {
+                  name = "domain-name-servers";
+                  data = vlan.address;
+                }
+              ];
+            }) dhcpVlans;
+          };
+        };
+
+        services.blocky = {
+          enable = true;
+          settings = {
+            ports = {
+              dns = 53;
+              http = 4000;
+            };
+            upstreams.groups.default = [
+              "9.9.9.9"
+              "149.112.112.112"
+              "2620:fe::fe"
+            ];
+            blocking = {
+              denylists.ads = [
+                "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts"
+              ];
+              clientGroupsBlock.default = [ "ads" ];
+            };
+            caching = {
+              minTime = "5m";
+              prefetching = true;
+            };
+            prometheus.enable = true;
+          };
+        };
+
+        networking.nameservers = [
+          "9.9.9.9"
+          "1.1.1.1"
+        ];
+
+      };
+  };
+}

modules/clan/gw-router/firewall.nix

@@ -0,0 +1,64 @@
+{
+  roles.default.perInstance = { settings, ... }: {
+    nixosModule =
+      {
+        config,
+        lib,
+        ...
+      }:
+      let
+
+        dir = config.clan.core.settings.directory;
+        instance = "zerotier";
+
+        networkId = builtins.readFile "${dir}/vars/shared/zerotier-network-${instance}/network-id/value";
+        full = "fd" + networkId + "9993";
+        hextet = i: builtins.substring (i * 4) 4 full;
+        subnetZtier = "${hextet 0}:${hextet 1}:${hextet 2}:${hextet 3}:${hextet 4}:${builtins.substring 20 2 full}00::/88";
+
+        vlanIfs = lib.mapAttrsToList (name: _: "vlan-${name}") settings.vlans;
+        wanVlanIfs = lib.mapAttrsToList (name: _: "vlan-${name}") (
+          lib.filterAttrs (_: vlan: vlan.allowedWAN) settings.vlans
+        );
+        nonMgmtIfs = lib.filter (i: i != "vlan-mgmt") vlanIfs;
+        ifSet = ifs: "{ ${lib.concatStringsSep ", " (map (i: "\"${i}\"") ifs)} }";
+      in
+      {
+        networking.nftables.enable = true;
+
+        services.openssh.openFirewall = false;
+
+        networking.firewall = {
+          enable = true;
+          filterForward = true;
+          trustedInterfaces = [ "vlan-mgmt" ];
+
+          interfaces = lib.genAttrs nonMgmtIfs (_: {
+            allowedTCPPorts = [ 53 ];
+            allowedUDPPorts = [
+              53
+              67
+            ];
+          });
+
+          extraInputRules = ''
+            ip6 saddr ${subnetZtier} tcp dport 22 accept comment "admin ssh over the mesh"
+            ip6 saddr ${subnetZtier} tcp dport 4000 accept comment "blocky metrics scrape from control"
+          '';
+
+          extraForwardRules = ''
+            tcp flags syn tcp option maxseg size set rt mtu comment "MSS clamp for PPPoE mtu 1492"
+            iifname "vlan-mgmt" accept comment "mgmt reaches all VLANs and the WAN"
+            iifname ${ifSet wanVlanIfs} oifname "ppp0" accept comment "LAN to internet"
+          '';
+        };
+
+        networking.nat = {
+          enable = true;
+          externalInterface = "ppp0";
+          internalInterfaces = vlanIfs;
+        };
+
+      };
+  };
+}

modules/clan/gw-router/flake-module.nix

@@ -0,0 +1,19 @@
+{ self, inputs, ... }:
+let
+  module = ./default.nix;
+in
+{
+  clan.modules = {
+    gw-router = module;
+  };
+  # perSystem =
+  #   { ... }:
+  #   {
+  #     clan.nixosTests.service-headplane = {
+  #       imports = [ ./tests/vm/default.nix ];
+  #       _module.args = { inherit self inputs; };
+  #
+  #       clan.modules."@clan/headplane" = module;
+  #     };
+  #   };
+}

modules/clan/gw-router/iperf.nix

@@ -0,0 +1,34 @@
+{
+  roles.default.perInstance = { settings, ... }: {
+    nixosModule =
+      {
+        config,
+        lib,
+        ...
+      }:
+      let
+        vlanIfs = lib.mapAttrsToList (name: _: "vlan-${name}") settings.vlans;
+
+        dir = config.clan.core.settings.directory;
+        instance = "zerotier";
+
+        networkId = builtins.readFile "${dir}/vars/shared/zerotier-network-${instance}/network-id/value";
+        full = "fd" + networkId + "9993";
+        hextet = i: builtins.substring (i * 4) 4 full;
+        subnetZtier = "${hextet 0}:${hextet 1}:${hextet 2}:${hextet 3}:${hextet 4}:${builtins.substring 20 2 full}00::/88";
+      in
+      {
+        services.iperf3.enable = true;
+
+        networking.firewall.interfaces = lib.genAttrs vlanIfs (_: {
+          allowedTCPPorts = [ 5201 ];
+          allowedUDPPorts = [ 5201 ];
+        });
+
+        networking.firewall.extraInputRules = ''
+          ip6 saddr ${subnetZtier} tcp dport 5201 accept comment "iperf3 over the mesh"
+          ip6 saddr ${subnetZtier} udp dport 5201 accept comment "iperf3 over the mesh"
+        '';
+      };
+  };
+}

modules/clan/gw-router/ipv6.nix

@@ -0,0 +1,29 @@
+{
+  roles.default.perInstance = { ... }: {
+    nixosModule =
+      {
+        lib,
+        ...
+      }:
+      {
+
+        systemd.network.networks."45-ppp0" = {
+          matchConfig.Name = "ppp0";
+          networkConfig = {
+            DHCP = "ipv6";
+            # pppd owns the v4 address/route on this link; don't let networkd
+            # tear them down.
+            KeepConfiguration = "static";
+            # Default v6 route comes from the ISP's RA when they send one.
+            IPv6AcceptRA = true;
+          };
+          # Many PPPoE ISPs never send an RA with the M flag; solicit regardless.
+          dhcpV6Config.WithoutRA = "solicit";
+          linkConfig.RequiredForOnline = "no";
+        };
+
+        boot.kernel.sysctl."net.ipv6.conf.all.forwarding" = lib.mkDefault 1;
+
+      };
+  };
+}

modules/clan/gw-router/omada.nix

@@ -0,0 +1,43 @@
+{
+  roles.default.perInstance = { settings, ... }: {
+    nixosModule =
+      {
+        config,
+        lib,
+        ...
+      }:
+      let
+        dir = config.clan.core.settings.directory;
+        instance = "zerotier";
+
+        networkId = builtins.readFile "${dir}/vars/shared/zerotier-network-${instance}/network-id/value";
+        full = "fd" + networkId + "9993";
+        hextet = i: builtins.substring (i * 4) 4 full;
+        subnetZtier = "${hextet 0}:${hextet 1}:${hextet 2}:${hextet 3}:${hextet 4}:${builtins.substring 20 2 full}00::/88";
+      in
+      {
+        config = lib.mkIf (settings.enableOmada) {
+          virtualisation.podman.enable = true;
+          virtualisation.oci-containers = {
+            backend = "podman";
+            containers.omada = {
+              image = "docker.io/mbentley/omada-controller:5.15";
+              extraOptions = [ "--network=host" ];
+              environment.TZ = config.time.timeZone;
+              volumes = [
+                "/var/lib/omada/data:/opt/tplink/EAPController/data"
+                "/var/lib/omada/logs:/opt/tplink/EAPController/logs"
+              ];
+            };
+          };
+
+          networking.firewall.extraInputRules = ''
+            ip6 saddr ${subnetZtier} tcp dport 8043 accept comment "omada ui over the mesh"
+          '';
+
+          clan.core.state.omada.folders = [ "/var/lib/omada" ];
+
+        };
+      };
+  };
+}

modules/clan/gw-router/pppoe.nix

@@ -0,0 +1,58 @@
+{
+  roles.default.perInstance = { settings, ... }: {
+    nixosModule =
+      { config, ... }:
+      let
+        creds = config.clan.core.vars.generators.gw-router;
+
+        pppInterface = if settings.wan.vlanId == null then settings.wan.interface else "wan-vlan";
+      in
+      {
+        clan.core.vars.generators.gw-router = {
+          prompts.pppoe-username = {
+            description = "PPPoE username (from the ISP)";
+            type = "hidden";
+          };
+          prompts.pppoe-password = {
+            description = "PPPoE password (from the ISP)";
+            type = "hidden";
+          };
+          files."pppoe-username".secret = true;
+          files."pppoe-password".secret = true;
+          script = ''
+            user="$(cat "$prompts"/pppoe-username)"
+            pass="$(cat "$prompts"/pppoe-password)"
+            printf 'user "%s"\n' "$user" > "$out"/pppoe-username
+            printf '"%s" * "%s"\n' "$user" "$pass" > "$out"/pppoe-password
+          '';
+        };
+
+        services.pppd = {
+          enable = true;
+          peers.wan = {
+            autostart = true;
+            config = ''
+              plugin pppoe.so ${pppInterface}
+              ifname ppp0
+              file ${creds.files."pppoe-username".path}
+              noipdefault
+              defaultroute
+              noauth
+              hide-password
+              persist
+              maxfail 0
+              holdoff 5
+              lcp-echo-interval 15
+              lcp-echo-failure 3
+              +ipv6
+              mtu 1492
+              mru 1492
+            '';
+          };
+        };
+
+        environment.etc."ppp/chap-secrets".source = creds.files."pppoe-password".path;
+        environment.etc."ppp/pap-secrets".source = creds.files."pppoe-password".path;
+      };
+  };
+}

modules/clan/gw-router/proxy.nix

@@ -0,0 +1,25 @@
+{
+  roles.default.perInstance = { settings, ... }: {
+    nixosModule =
+      {
+        config,
+        lib,
+        ...
+      }:
+      let
+      in
+      {
+
+        config = lib.mkIf (settings.proxy.enable) {
+          assertions = [
+            {
+              assertion = lib.all (v: settings.vlans ? ${v}) settings.proxy.allowVlans;
+              message = "cnx.router.proxy.allowVlans must name VLANs defined in cnx.router.vlans.";
+            }
+          ];
+
+        };
+
+      };
+  };
+}

modules/clan/gw-router/speedtest.nix

@@ -0,0 +1,60 @@
+{
+  roles.default.perInstance = { settings, ... }: {
+    nixosModule =
+      {
+        pkgs,
+        lib,
+        ...
+      }:
+      let
+        textfileDir = "/var/lib/speedtest";
+      in
+      {
+        config = lib.mkIf (settings.speedtest.enable) {
+          services.prometheus.exporters.node.extraFlags = [
+            "--collector.textfile.directory=${textfileDir}"
+          ];
+
+          systemd.services.speedtest = {
+            description = "WAN speed test to Prometheus textfile metrics";
+            after = [ "network-online.target" ];
+            wants = [ "network-online.target" ];
+            path = [
+              pkgs.librespeed-cli
+              pkgs.jq
+            ];
+            serviceConfig = {
+              Type = "oneshot";
+              StateDirectory = "speedtest";
+              # One test at boot would race PPPoE and log a spurious failure.
+              ExecCondition = "${pkgs.iproute2}/bin/ip link show ppp0";
+            };
+            script = ''
+              tmp="${textfileDir}/.speedtest.prom.tmp"
+              if result=$(librespeed-cli --json); then
+                jq -r '.[0]
+                  | "speedtest_download_mbps \(.download)",
+                    "speedtest_upload_mbps \(.upload)",
+                    "speedtest_ping_ms \(.ping)",
+                    "speedtest_jitter_ms \(.jitter)",
+                    "speedtest_success 1"' <<<"$result" > "$tmp"
+              else
+                echo "speedtest_success 0" > "$tmp"
+              fi
+              mv "$tmp" "${textfileDir}/speedtest.prom"
+            '';
+          };
+
+          systemd.timers.speedtest = {
+            wantedBy = [ "timers.target" ];
+            timerConfig = {
+              OnCalendar = settings.speedtest.interval;
+              RandomizedDelaySec = "10m";
+              Persistent = true;
+            };
+          };
+
+        };
+      };
+  };
+}

modules/dns/acme-gw-secret.nix

@@ -0,0 +1,17 @@
+# Shared TSIG secret for a gateway's dedicated ACME key (function: machine
+# name -> module). The acme_gw_<x> key lets that gateway — and only it — write
+# _acme-challenge.<label> TXT records on ns1 to obtain its internal wildcard
+# cert via DNS-01. ns1 scopes it with a matching acl on the cnx.network zone.
+# Import on BOTH ns1 and the gateway machine, applied with the machine name:
+#   (import ../../modules/dns/acme-gw-secret.nix "gw-cnx-1")
+machine:
+{ pkgs, ... }:
+{
+  clan.core.vars.generators."dns-acme-${machine}-secret" = {
+    share = true;
+    files."secret".secret = true;
+    runtimeInputs = [ pkgs.openssl ];
+    # 32 random bytes, base64 — a valid hmac-sha256 TSIG secret.
+    script = ''openssl rand -base64 32 | tr -d '\n' > "$out"/secret'';
+  };
+}

modules/dns/acme-mx1-secret.nix

@@ -0,0 +1,19 @@
+# Shared TSIG secret for the dedicated acme_mx1 key.
+#
+# This key lets mx1 — and only mx1 — write _acme-challenge.mx1.cnx.email TXT
+# records on ns1 to obtain its mail TLS cert via ACME DNS-01. ns1 scopes it with
+# acl_acme_mx1 (attached only to the cnx.email zone) so the credential can touch
+# nothing else. ns1 renders this secret into a Knot key file; mx1 into a lego
+# rfc2136 env file; both must carry the same secret, hence one shared generator
+# with a per-host renderer that depends on it. Imported by ns1 and (via mail.nix)
+# mx1.
+{ pkgs, ... }:
+{
+  clan.core.vars.generators.dns-acme-mx1-secret = {
+    share = true;
+    files."secret".secret = true;
+    runtimeInputs = [ pkgs.openssl ];
+    # 32 random bytes, base64 — a valid hmac-sha256 TSIG secret.
+    script = ''openssl rand -base64 32 | tr -d '\n' > "$out"/secret'';
+  };
+}

modules/dns/acme-web01-secret.nix

@@ -0,0 +1,19 @@
+# Shared TSIG secret for the dedicated acme_web01 key.
+#
+# This key lets web01 — and only web01 — write _acme-challenge.cnx.network TXT
+# records on ns1 to obtain its wildcard (*.cnx.network) TLS cert via ACME DNS-01.
+# ns1 scopes it with acl_acme_web01 (attached only to the cnx.network zone) so the
+# credential can touch nothing else. ns1 renders this secret into a Knot key file;
+# web01 into a lego rfc2136 env file; both must carry the same secret, hence one
+# shared generator with a per-host renderer that depends on it. Imported by ns1
+# and (via web-proxy.nix) web01.
+{ pkgs, ... }:
+{
+  clan.core.vars.generators.dns-acme-web01-secret = {
+    share = true;
+    files."secret".secret = true;
+    runtimeInputs = [ pkgs.openssl ];
+    # 32 random bytes, base64 — a valid hmac-sha256 TSIG secret.
+    script = ''openssl rand -base64 32 | tr -d '\n' > "$out"/secret'';
+  };
+}

modules/dns/authoritative.nix

@@ -1,4 +1,9 @@
-{ config, lib, pkgs, ... }:
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
 let
   # ZeroTier addresses — zone transfers run over the mesh, not the public net.
   mesh = import ../mesh-hosts.nix { inherit config lib; };
@@ -27,7 +32,7 @@ in
 
   # knot binds 0.0.0.0@53 and ::@53, so free port 53 by disabling the
   # systemd-resolved stub listener. Resolution still works via nss-resolve.
-  services.resolved.extraConfig = "DNSStubListener=no";
+  services.resolved.settings.Resolve.DNSStubListener = "no";
 
   services.knot = {
     enable = true;

modules/dns/domains.nix

@@ -4,4 +4,5 @@
   "cnx.network"
   "buildfor.life"
   "cnx.email"
+  "newedge.house"
 ]

modules/dns/zones/cnx.email.zone

@@ -11,8 +11,36 @@ $TTL 3600
 @       IN  NS      ns1.cnx.network.
 @       IN  NS      ns2.cnx.network.
 
-; ---- Mail (fill in once the mail host exists) ----
-;@      IN  MX 10   mail.cnx.email.
-;mail   IN  A       <mail-ipv4>
-;@      IN  TXT     "v=spf1 mx -all"
-;_dmarc IN  TXT     "v=DMARC1; p=quarantine; rua=mailto:postmaster@cnx.email"
+; ---- Mail ----
+mx1     IN  A       5.223.65.38
+mx1     IN  AAAA    2a01:4ff:2f0:1963::1
+; Client-facing alias for IMAP/submission (Thunderbird etc.); the cert carries
+; mail.cnx.email as a SAN. The MX must never point here (CNAMEs are illegal MX
+; targets) — server-to-server delivery and DANE stay on mx1.cnx.email.
+mail    IN  CNAME   mx1.cnx.email.
+@       IN  MX 10   mx1.cnx.email.
+@       IN  TXT     "v=spf1 mx -all"
+; Aggregate (rua) + forensic (ruf) reports go to the dmarc@cnx.email mailbox,
+; which parsedmarc on control polls and feeds into Grafana. fo=1 asks reporters
+; to send a forensic report on any SPF/DKIM failure.
+_dmarc  IN  TXT     "v=DMARC1; p=quarantine; rua=mailto:dmarc@cnx.email; ruf=mailto:dmarc@cnx.email; fo=1"
+
+; ---- DANE / TLSA ----
+; "3 1 1" = DANE-EE, SPKI, SHA-256: the digest of mx1's certificate public key.
+; Valid because the zone is DNSSEC-signed and the lego cert uses --reuse-key, so
+; the key (and thus this digest) is stable across renewals. Compute it AFTER the
+; first issuance and paste the hex below:
+;   ssh mx1 'openssl x509 -in /var/lib/acme/mx1.cnx.email/cert.pem -noout -pubkey \
+;     | openssl pkey -pubin -outform DER | openssl dgst -sha256 -binary | xxd -p -c256'
+_25._tcp.mx1  IN  TLSA  3 1 1  bd9a51f60b6d2dd20f18b3553d2795053ac52f87567a46bc892006bb58506404
+
+; ---- MTA-STS ----
+; Policy host (A/AAAA point at mx1); the _mta-sts TXT id MUST be bumped whenever
+; the policy file in modules/mail.nix changes, or senders keep the cached policy.
+mta-sts   IN  A       5.223.65.38
+mta-sts   IN  AAAA    2a01:4ff:2f0:1963::1
+_mta-sts  IN  TXT     "v=STSv1; id=2026061801"
+mail._domainkey IN TXT ( "v=DKIM1; k=rsa; "
+        "p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAr9QxTs5dLtY76bo156+Tp0GUoE554rMwIooIYa2MMYHNs8zPb0thFmaCKGAINdHKNIq2phXAlk51iBTfdqXjx7gVWSrs+ftykqO3b5hUjgImsgqPWGUTzy5/bUgcDELiD9KKEyKYD3+ebZEw6d0uvBvEsA6a1CPzOsufoCDtyKjByCuQzkCBrK25TUHFolGvEYcZexR0LSF+8hMss"
+        "xyw9NYiPpTXVCWQJnrZZpuOBiX0K2l5CAXVyuT/B5RcBXlAUhBTp3390VEhL0wAZMTOnvtvBYK3NnsTIh96fkh6MfWmre7Fi9hEq//xGf40N5/aomMjJrJdqFZJLZpDotb/XwIDAQAB"
+        )

modules/dns/zones/cnx.network.zone

@@ -25,3 +25,10 @@ control IN  AAAA    fd06:1bad:ece2:92ad:ba99:9306:1bad:ece2
 ;@      IN  A       <web-ipv4>
 ;www    IN  CNAME   cnx.network.
 monitor IN  A       5.223.66.36
+
+; ---- web01 (public reverse proxy / TLS termination) ----
+; Serves a wildcard *.cnx.network TLS cert (ACME DNS-01) and forwards to internal
+; services over the mesh. Add a vhost in modules/web-proxy.nix and a CNAME here.
+web01   IN  A       5.223.55.246
+web01   IN  AAAA    2a01:4ff:2f0:2d8f::1
+grafana IN  CNAME   web01.cnx.network.

modules/dns/zones/newedge.house.zone

@@ -0,0 +1,13 @@
+$ORIGIN newedge.house.
+$TTL 3600
+
+@   IN  SOA ns1.cnx.network. hostmaster.cnx.network. (
+        2026061401  ; serial   (ignored: Knot auto-assigns a dateserial on signing)
+        3600        ; refresh
+        900         ; retry
+        604800      ; expire
+        300 )       ; negative-cache TTL
+
+; Served by the same nameservers (out-of-bailiwick, no glue needed here).
+@       IN  NS      ns1.cnx.network.
+@       IN  NS      ns2.cnx.network.

modules/docs.nix

@@ -0,0 +1,46 @@
+# Infra runbook (mdBook), built at Nix-build time from ./docs and served by Caddy.
+# Reachable only over the ZeroTier mesh (firewall rule below); the public side is
+# already closed by the Hetzner cloud firewall. Imported by control only.
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  mesh = import ./mesh-hosts.nix { inherit config lib; };
+  port = 8080;
+
+  site = pkgs.stdenvNoCC.mkDerivation {
+    name = "cnx-infra-docs";
+    src = ../docs;
+    nativeBuildInputs = [ pkgs.mdbook ];
+    # mdbook writes a state dir under $HOME; the build sandbox has none.
+    buildPhase = ''
+      export HOME=$TMPDIR
+      mdbook build -d $out
+    '';
+    dontInstall = true;
+  };
+in
+{
+  # ":port" makes Caddy serve plain HTTP (no automatic TLS) on all interfaces;
+  # the mesh-scoped firewall rule below is what constrains reachability.
+  services.caddy = {
+    enable = true;
+    virtualHosts.":${toString port}".extraConfig = ''
+      root * ${site}
+      # mdBook doesn't fingerprint asset filenames, and every file in the Nix
+      # store carries an epoch (1970) mtime, so file_server's only validator never
+      # changes across redeploys — conditional requests would 304 forever and pin
+      # browsers to stale docs. no-store sidesteps caching entirely; pages are tiny
+      # and mesh-only, so refetching on each load is free.
+      header Cache-Control "no-store"
+      file_server
+    '';
+  };
+
+  networking.firewall.extraInputRules = ''
+    ip6 saddr ${mesh.subnet} tcp dport ${toString port} accept
+  '';
+}

modules/hetzner-firewall-rules.nix

@@ -24,6 +24,46 @@ let
     description = "ICMP (ping / PMTUD)";
   };
 
+  # Public mail ports for mx1 (MX for cnx.email). 25 is server-to-server
+  # delivery; 587/465 are client submission; 143/993 are IMAP. 443 serves only the
+  # MTA-STS policy (https://mta-sts.cnx.email/.well-known/mta-sts.txt); the cert
+  # itself uses ACME DNS-01 so port 80 stays closed. Admin still rides the mesh.
+  mailPort = port: description: {
+    direction = "in";
+    protocol = "tcp";
+    inherit port;
+    source_ips = world;
+    inherit description;
+  };
+  mailRules = [
+    (mailPort "25" "SMTP (inbound mail)")
+    (mailPort "587" "Submission (STARTTLS)")
+    (mailPort "465" "Submission (implicit TLS)")
+    (mailPort "143" "IMAP (STARTTLS)")
+    (mailPort "993" "IMAP (implicit TLS)")
+    (mailPort "443" "MTA-STS policy (HTTPS)")
+  ];
+
+  # web01 is a public reverse proxy with TLS termination. 443 serves the proxy;
+  # 80 only carries Caddy's HTTP->HTTPS redirect (the cert uses ACME DNS-01, not
+  # HTTP-01). Admin rides the mesh.
+  webRules = [
+    {
+      direction = "in";
+      protocol = "tcp";
+      port = "80";
+      source_ips = world;
+      description = "HTTP (redirect to HTTPS)";
+    }
+    {
+      direction = "in";
+      protocol = "tcp";
+      port = "443";
+      source_ips = world;
+      description = "HTTPS (reverse proxy / TLS termination)";
+    }
+  ];
+
   dnsRules = [
     {
       direction = "in";
@@ -50,4 +90,12 @@ in
   ];
   "clan-ns1" = dnsRules;
   "clan-ns2" = dnsRules;
+  "clan-mx1" = mailRules ++ [
+    zerotier
+    ping
+  ];
+  "clan-web01" = webRules ++ [
+    zerotier
+    ping
+  ];
 }

modules/hosts.nix

@@ -0,0 +1,32 @@
+# Per-host public network facts: single source of truth for each machine's
+# public IPv4 and its static public IPv6. Consumed by clan.nix's `internet`
+# connection hosts (ipv4) and each machine's `cnx.staticIPv6` (ipv6), so an
+# address is written once instead of being duplicated across configs.
+#
+# NOT a driver for the DNS zone files — those stay hand-edited text, so a record
+# here that also appears as A/AAAA glue still needs a matching manual zone edit.
+#
+# ipv6 is the single address to assign from the host's allocated /64 (we take
+# ::1), without prefix length; cnx.staticIPv6 supplies the /64 default.
+{
+  control = {
+    ipv4 = "77.42.68.181";
+    ipv6 = "2a01:4f9:c013:e6d0::1";
+  };
+  ns1 = {
+    ipv4 = "46.224.170.206";
+    ipv6 = "2a01:4f8:c014:b5c5::1";
+  };
+  ns2 = {
+    ipv4 = "157.180.70.82";
+    ipv6 = "2a01:4f9:c014:6d87::1";
+  };
+  mx1 = {
+    ipv4 = "5.223.65.38";
+    ipv6 = "2a01:4ff:2f0:1963::1";
+  };
+  web01 = {
+    ipv4 = "5.223.55.246";
+    ipv6 = "2a01:4ff:2f0:2d8f::1";
+  };
+}

modules/mail-dmarc-cred.nix

@@ -0,0 +1,26 @@
+# Shared credential for the dmarc@cnx.email mailbox.
+#
+# DMARC aggregate/forensic reports are delivered to dmarc@cnx.email on mx1;
+# parsedmarc on control fetches them over IMAPS across the mesh and needs the
+# *plaintext* passphrase, while mx1's mailserver only needs the sha-512 hash.
+# clan vars secrets are per-machine, so this generator is shared (share = true)
+# to make the same value available on both hosts. Files are root-owned: SNM reads
+# the hash as root, and parsedmarc's ExecStartPre reads the passphrase as root.
+# Imported by mx1 (via mail.nix) and control (via monitoring/parsedmarc.nix).
+{ pkgs, ... }:
+{
+  clan.core.vars.generators.mail-dmarc-cred = {
+    share = true;
+    files."passphrase".secret = true; # read by parsedmarc on control
+    files."hash".secret = true; # consumed by the mailserver on mx1
+    runtimeInputs = [
+      pkgs.xkcdpass
+      pkgs.mkpasswd
+    ];
+    script = ''
+      pass="$(xkcdpass --numwords=4 --delimiter=- --case=lower)-$((RANDOM % 90 + 10))"
+      printf '%s' "$pass" > "$out"/passphrase
+      printf '%s' "$pass" | mkpasswd -s -m sha-512 > "$out"/hash
+    '';
+  };
+}

modules/mail.nix

@@ -0,0 +1,163 @@
+# Declarative mail stack for mx1 (Simple NixOS Mailserver: Postfix + Dovecot +
+# Rspamd + OpenDKIM). Imported by machines/mx1 alongside the SNM flake module.
+#
+# Mailboxes are virtual (not system users): each address below is a login account
+# whose password is auto-generated by a clan vars generator as a four-word
+# passphrase with a trailing number (e.g. otter-lantern-cobalt-driftwood-42). The
+# generator stores both the passphrase and its sha-512 hash. To add a mailbox:
+# append the address to `accounts`, run `clan vars generate mx1`, redeploy mx1,
+# then hand the passphrase to the user:
+#   clan vars get mx1 mail-passwd-<addr>/passphrase
+# (addr with @ and . replaced by -at- and -, e.g. mail-passwd-postmaster-at-cnx-email)
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  hosts = import ./hosts.nix;
+  fqdn = "mx1.cnx.email";
+  mtaStsHost = "mta-sts.cnx.email";
+  # Client-facing alias (CNAME -> mx1) so Thunderbird etc. can use mail.cnx.email
+  # for submission/IMAP; added as a cert SAN so TLS validates against that name.
+  clientHost = "mail.cnx.email";
+
+  # MTA-STS policy served at https://mta-sts.cnx.email/.well-known/mta-sts.txt.
+  # enforce = a sending MTA that fetched this must use a valid, MX-matching TLS
+  # cert or refuse to deliver. Bump the _mta-sts TXT id (in the zone) whenever
+  # this changes.
+  mtaStsPolicy = pkgs.writeText "mta-sts.txt" ''
+    version: STSv1
+    mode: enforce
+    mx: ${fqdn}
+    max_age: 604800
+  '';
+
+  # The mailboxes mx1 serves. postmaster is required by RFC 5321.
+  accounts = [
+    "postmaster@cnx.email"
+  ];
+
+  genName = addr: "mail-passwd-" + lib.replaceStrings [ "@" "." ] [ "-at-" "-" ] addr;
+
+  passwdGenerators = lib.listToAttrs (
+    map (addr: {
+      name = genName addr;
+      value = {
+        files."passphrase".secret = true; # retrievable to hand to the user
+        files."hash".secret = true; # consumed by SNM's hashedPasswordFile
+        runtimeInputs = [
+          pkgs.xkcdpass
+          pkgs.mkpasswd
+        ];
+        script = ''
+          pass="$(xkcdpass --numwords=4 --delimiter=- --case=lower)-$((RANDOM % 90 + 10))"
+          printf '%s' "$pass" > "$out"/passphrase
+          printf '%s' "$pass" | mkpasswd -s -m sha-512 > "$out"/hash
+        '';
+      };
+    }) accounts
+  );
+
+  loginAccounts =
+    lib.listToAttrs (
+      map (addr: {
+        name = addr;
+        value.hashedPasswordFile = config.clan.core.vars.generators.${genName addr}.files."hash".path;
+      }) accounts
+    )
+    // {
+      # DMARC report inbox (rua/ruf target in the cnx.email zone). Its password
+      # comes from the *shared* mail-dmarc-cred generator instead of the per-machine
+      # set above, so parsedmarc on control can read the same passphrase over the
+      # mesh. Retrieve it with: clan vars get mx1 mail-dmarc-cred/passphrase
+      "dmarc@cnx.email".hashedPasswordFile =
+        config.clan.core.vars.generators.mail-dmarc-cred.files."hash".path;
+    };
+in
+{
+  imports = [
+    ./dns/acme-mx1-secret.nix
+    ./mail-dmarc-cred.nix
+  ];
+
+  clan.core.vars.generators = passwdGenerators // {
+    # Render the shared acme_mx1 TSIG secret into a lego rfc2136 env file. lego
+    # (via security.acme below) uses it to write the _acme-challenge.mx1.cnx.email
+    # TXT record to ns1, which authorizes the acme_mx1 key for exactly that owner.
+    dns-acme-rfc2136 = {
+      files."rfc2136.env".secret = true; # root-owned; systemd reads it as root
+      dependencies = [ "dns-acme-mx1-secret" ];
+      script = ''
+        printf 'RFC2136_NAMESERVER=${hosts.ns1.ipv4}:53\nRFC2136_TSIG_ALGORITHM=hmac-sha256.\nRFC2136_TSIG_KEY=acme_mx1\nRFC2136_TSIG_SECRET=%s\n' \
+          "$(cat "$in"/dns-acme-mx1-secret/secret)" > "$out"/rfc2136.env
+      '';
+    };
+  };
+
+  mailserver = {
+    enable = true;
+    # Fresh install: declare the latest layout the nixos-26.05 branch ships (3),
+    # so SNM uses the current dovecot mail directory layout with nothing to migrate.
+    stateVersion = 3;
+    inherit fqdn;
+    domains = [ "cnx.email" ];
+    accounts = loginAccounts;
+
+    # Consume a security.acme cert we obtain ourselves via DNS-01 (below); no
+    # web server and no inbound HTTP needed, so port 80 stays closed. The extra
+    # SAN hosts (MTA-STS, client alias) are attached to that cert via
+    # security.acme.certs.${fqdn}.extraDomainNames below.
+    x509.useACMEHost = fqdn;
+
+    dkim.defaults.selector = "mail";
+  };
+
+  security.acme = {
+    acceptTerms = true;
+    defaults.email = "postmaster@cnx.email";
+    certs.${fqdn} = {
+      # The MTA-STS endpoint and client-facing alias ride this one cert as SANs.
+      extraDomainNames = [
+        mtaStsHost
+        clientHost
+      ];
+      dnsProvider = "rfc2136";
+      environmentFile = config.clan.core.vars.generators.dns-acme-rfc2136.files."rfc2136.env".path;
+      # ns1 is the only nameserver that accepts the acme_mx1 UPDATE; check
+      # propagation against it directly rather than a public resolver.
+      dnsResolver = "${hosts.ns1.ipv4}:53";
+      # Keep the private key fixed across renewals so the DANE TLSA "3 1 1"
+      # record (public-key digest, published in the zone) stays valid.
+      extraLegoRenewFlags = [ "--reuse-key" ];
+      # Caddy serves the MTA-STS endpoint from explicit cert file paths, so it
+      # won't notice a renewal on its own — reload it whenever the cert changes.
+      # (Merges with the postfix/dovecot reloads SNM wires up for this cert.)
+      reloadServices = [ "caddy.service" ];
+    };
+  };
+
+  # The mail cert is owned group=acme (SNM adds postfix/dovecot); Caddy serves the
+  # MTA-STS endpoint from the same cert, so it needs to read the key too.
+  users.users.caddy.extraGroups = [ "acme" ];
+
+  # MTA-STS policy endpoint, served by Caddy (same web server as control's docs).
+  # The explicit `tls cert key` points at the lego-issued mail cert (which carries
+  # mta-sts.cnx.email as a SAN) and disables Caddy's automatic ACME, so no extra
+  # issuance happens and the DANE TLSA key stays stable. Only :443 is opened.
+  services.caddy = {
+    enable = true;
+    virtualHosts.${mtaStsHost}.extraConfig = ''
+      tls /var/lib/acme/${fqdn}/cert.pem /var/lib/acme/${fqdn}/key.pem
+      root * ${pkgs.writeTextDir ".well-known/mta-sts.txt" (builtins.readFile mtaStsPolicy)}
+      file_server
+    '';
+  };
+
+  # DKIM private keys are generated on first start under this dir. They're
+  # regenerable (rotate + republish the TXT), but declaring the path as clan
+  # state lets a borg client back it up to avoid a needless DNS round-trip on
+  # restore. Wiring mx1 into the borgbackup instance is a separate step.
+  clan.core.state.mail-dkim.folders = [ config.mailserver.dkimKeyDirectory ];
+}

modules/mesh-hosts.nix

@@ -2,25 +2,35 @@
 # DNS zone transfers and metrics scraping ride this mesh, never the public net.
 #
 # Rather than hardcoding the addresses, we read them from the public clan vars
-# that clan-core's zerotier generator already writes per machine
-# (vars/per-machine/<m>/zerotier/zerotier-ip/value). This keeps the mesh map in
+# that clan-core's zerotier service writes. As of clan-core 26.05 these are
+# shared, instance-scoped generators: the per-machine IP lives at
+# vars/shared/zerotier-ip-<machine>-<instance>/ip and the network id at
+# vars/shared/zerotier-network-<instance>/network-id (instance = "zerotier",
+# the inventory.instances.zerotier name in clan.nix). This keeps the mesh map in
 # lockstep with the actual identities: regenerate or re-key a node and its
 # address here follows automatically. Call as: import ../mesh-hosts.nix { inherit config lib; }.
 { config, lib }:
 let
   dir = config.clan.core.settings.directory;
+  instance = "zerotier";
 
-  readVar =
-    machine: file:
-    builtins.readFile "${dir}/vars/per-machine/${machine}/zerotier/${file}/value";
+  readIp =
+    machine: builtins.readFile "${dir}/vars/shared/zerotier-ip-${machine}-${instance}/ip/value";
 
-  hosts = lib.genAttrs [ "control" "ns1" "ns2" ] (m: readVar m "zerotier-ip");
+  hosts = lib.genAttrs [
+    "control"
+    "ns1"
+    "ns2"
+    "mx1"
+    "web01"
+    "gw-cnx-1"
+  ] readIp;
 
   # RFC 4193 prefix of this ZeroTier network: fd + the 8-byte network id + the
-  # 0x9993 marker. The network id is a public var on the controller (control).
+  # 0x9993 marker. The network id is a public, shared var for the instance.
   # The /88 (11 bytes) covers fd + network id + 0x99 + 0x93, i.e. every mesh peer,
   # and is used to scope mesh-only firewall rules.
-  networkId = readVar "control" "zerotier-network-id";
+  networkId = builtins.readFile "${dir}/vars/shared/zerotier-network-${instance}/network-id/value";
   full = "fd" + networkId + "9993"; # 22 hex chars = 11 bytes
   hextet = i: builtins.substring (i * 4) 4 full;
   subnet = "${hextet 0}:${hextet 1}:${hextet 2}:${hextet 3}:${hextet 4}:${builtins.substring 20 2 full}00::/88";

modules/monitoring/alerts.nix

@@ -0,0 +1,133 @@
+# Alerting rules, evaluated by vmalert against VictoriaMetrics on control.
+# Everything is declared here in git. vmalert remote-writes alert state back to
+# VM, so firing alerts surface as the `ALERTS{alertstate="firing"}` series and
+# can be viewed in Grafana. No notifier is wired yet: notifier.blackhole makes
+# that explicit (vmalert evaluates rules but sends nowhere). To deliver alerts
+# later, drop blackhole and set settings."notifier.url" to an Alertmanager.
+{ ... }:
+let
+  vmUrl = "http://127.0.0.1:8428";
+in
+{
+  services.vmalert.instances.cnx = {
+    enable = true;
+    settings = {
+      "datasource.url" = vmUrl;
+      "remoteWrite.url" = vmUrl; # persists ALERTS / ALERTS_FOR_STATE back to VM
+      "notifier.blackhole" = true;
+      "httpListenAddr" = "127.0.0.1:8880"; # vmalert UI/API, loopback only (like VM)
+    };
+    rules.groups = [
+      {
+        name = "dns";
+        rules = [
+          {
+            alert = "DNSSecondaryOutOfSync";
+            expr = "max by (zone) (knot_zone_serial) - min by (zone) (knot_zone_serial) > 0";
+            for = "15m";
+            labels.severity = "warning";
+            annotations.summary = "Zone {{ $labels.zone }} SOA serial differs between nameservers";
+            annotations.description = "The secondary is out of sync with the primary for {{ $labels.zone }}. `knotc zone-retransfer {{ $labels.zone }}` on ns2 forces a fresh pull.";
+          }
+          {
+            alert = "ZoneExpiryLow";
+            expr = "knot_zone_status_expiration < 3600";
+            for = "5m";
+            labels.severity = "critical";
+            annotations.summary = "Zone {{ $labels.zone }} on {{ $labels.instance }} is within 1h of expiry";
+            annotations.description = "Transfers to the secondary appear to be failing; the zone stops being served when the SOA expire timer hits zero.";
+          }
+        ];
+      }
+      {
+        name = "host";
+        rules = [
+          {
+            alert = "ScrapeTargetDown";
+            expr = "up == 0";
+            for = "5m";
+            labels.severity = "critical";
+            annotations.summary = "{{ $labels.job }} exporter on {{ $labels.instance }} is down";
+            annotations.description = "VictoriaMetrics cannot scrape this target; its metrics are missing.";
+          }
+          {
+            alert = "RootFilesystemFull";
+            expr = ''100 * (1 - node_filesystem_avail_bytes{mountpoint="/",fstype!="tmpfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!="tmpfs"}) > 90'';
+            for = "15m";
+            labels.severity = "warning";
+            annotations.summary = "Root filesystem on {{ $labels.instance }} is over 90% full";
+          }
+        ];
+      }
+      {
+        # WAN quality on the site gateways, fed by the hourly speedtest timer
+        # (modules/router/speedtest.nix) via the node_exporter textfile
+        # collector. Degradation is judged against the link's own 7-day median,
+        # so no per-site/per-ISP-plan threshold needs maintaining.
+        name = "wan";
+        rules = [
+          {
+            alert = "WANSpeedDegraded";
+            expr = "speedtest_download_mbps < 0.5 * quantile_over_time(0.5, speedtest_download_mbps[7d])";
+            for = "3h";
+            labels.severity = "warning";
+            annotations.summary = "WAN download on {{ $labels.instance }} is below half its 7-day median";
+            annotations.description = "Sustained for 3h (several test runs) — likely ISP degradation, line fault, or saturation. Compare speedtest_upload_mbps/speedtest_ping_ms and run `librespeed-cli` manually on the gateway.";
+          }
+          {
+            alert = "WANSpeedTestFailing";
+            expr = "speedtest_success == 0";
+            for = "3h";
+            labels.severity = "warning";
+            annotations.summary = "Speed tests on {{ $labels.instance }} have been failing for 3h";
+            annotations.description = "librespeed-cli cannot complete a test while other traffic may still flow; check `journalctl -u speedtest` on the gateway.";
+          }
+        ];
+      }
+      {
+        name = "backup";
+        rules = [
+          {
+            alert = "BackupJobFailed";
+            expr = ''node_systemd_unit_state{name=~"borgbackup-job-.+\\.service",state="failed"} == 1'';
+            for = "5m";
+            labels.severity = "warning";
+            annotations.summary = "Backup job {{ $labels.name }} on {{ $labels.instance }} failed";
+            annotations.description = "The borgbackup run did not complete. Check `systemctl status {{ $labels.name }}` and `journalctl -u {{ $labels.name }}` on the client; `borgbackup-create` re-runs it.";
+          }
+          {
+            alert = "BackupStale";
+            expr = ''time() - node_systemd_timer_last_trigger_seconds{name=~"borgbackup-job-.+\\.timer"} > 93600'';
+            for = "30m";
+            labels.severity = "warning";
+            annotations.summary = "No successful backup on {{ $labels.instance }} for over 26h";
+            annotations.description = "The daily backup timer {{ $labels.name }} has not fired within its expected window; the most recent archive is stale. A value far above 26h (or no data) means backups have stopped entirely.";
+          }
+        ];
+      }
+      {
+        # Outside-in DNS probes (blackbox on control). The `for` rides out a
+        # single dropped UDP packet; only a sustained failure fires.
+        name = "dns_probe";
+        rules = [
+          {
+            alert = "DNSResolutionProbeFailed";
+            expr = ''probe_success{query="SOA"} == 0'';
+            for = "5m";
+            labels.severity = "critical";
+            annotations.summary = "{{ $labels.zone }} is not resolving from {{ $labels.instance }}";
+            annotations.description = "The blackbox SOA probe to this public nameserver address is failing; from the outside the zone looks unavailable there, which the Knot stats would not show.";
+          }
+          {
+            alert = "DNSSECProbeFailed";
+            expr = ''probe_success{query="DNSKEY"} == 0'';
+            for = "5m";
+            labels.severity = "critical";
+            annotations.summary = "{{ $labels.zone }} DNSKEY missing from {{ $labels.instance }}";
+            annotations.description = "The DNSKEY probe to this public nameserver address is failing: the zone's signing keys are not being served, so validating resolvers will treat answers as bogus.";
+          }
+        ];
+      }
+    ];
+  };
+}

modules/monitoring/blackbox-probes.nix

@@ -0,0 +1,108 @@
+# Blackbox DNS probe definitions, shared between the exporter module
+# (modules/monitoring/blackbox.nix, which renders these into the blackbox
+# config) and the scraper (modules/monitoring/server.nix, which turns them into
+# VictoriaMetrics scrape jobs). Kept in one place so the module list and the
+# scrape jobs can never drift apart.
+#
+# These query the nameservers' PUBLIC addresses, i.e. the path a real internet
+# resolver takes, not the mesh — the whole point is to catch outside-in
+# resolution failures the Knot stats can't see. For each zone we run two probes
+# per endpoint: an SOA query (is the zone being served at all?) and a DNSKEY
+# query (is it still DNSSEC-signed?). Blackbox has no DO-bit option, so we ask
+# for DNSKEY directly — an authoritative signed zone returns it without EDNS0,
+# and its absence means signing has broken.
+{ lib }:
+let
+  domains = import ../dns/domains.nix;
+
+  blackboxAddr = "127.0.0.1:9115";
+
+  # Public endpoints of the authoritative nameservers. The v4 addresses also
+  # appear in the `internet` instance in clan.nix; the v6 ones in each ns
+  # machine's cnx.staticIPv6. IPv6 literals are bracketed for host:port.
+  endpoints = [
+    {
+      instance = "ns1 v4";
+      target = "46.224.170.206:53";
+    }
+    {
+      instance = "ns1 v6";
+      target = "[2a01:4f8:c014:b5c5::1]:53";
+    }
+    {
+      instance = "ns2 v4";
+      target = "157.180.70.82:53";
+    }
+    {
+      instance = "ns2 v6";
+      target = "[2a01:4f9:c014:6d87::1]:53";
+    }
+  ];
+
+  queries = [
+    {
+      name = "soa";
+      type = "SOA";
+    }
+    {
+      name = "dnskey";
+      type = "DNSKEY";
+    }
+  ];
+
+  sanitize = lib.replaceStrings [ "." ] [ "_" ];
+  moduleName = zone: q: "dns_${q.name}_${sanitize zone}";
+
+  modules = lib.listToAttrs (
+    lib.concatMap (
+      zone:
+      map (
+        q:
+        lib.nameValuePair (moduleName zone q) {
+          prober = "dns";
+          timeout = "5s";
+          dns = {
+            query_name = "${zone}.";
+            query_type = q.type;
+            valid_rcodes = [ "NOERROR" ];
+            # Fail unless at least one answer RR of the queried type is present:
+            # a NOERROR with an empty answer (or a missing DNSKEY) still fails.
+            validate_answer_rrs.fail_if_not_matches_regexp = [ "\\s${q.type}\\s" ];
+          };
+        }
+      ) queries
+    ) domains
+  );
+
+  scrapeConfigs = lib.concatMap (
+    zone:
+    map (q: {
+      job_name = "blackbox_${moduleName zone q}";
+      metrics_path = "/probe";
+      params.module = [ (moduleName zone q) ];
+      static_configs = map (e: {
+        targets = [ e.target ];
+        labels = {
+          instance = e.instance;
+          zone = zone;
+          query = q.type;
+        };
+      }) endpoints;
+      # Hand the real DNS server to blackbox as ?target=, then point the scrape
+      # at the exporter itself.
+      relabel_configs = [
+        {
+          source_labels = [ "__address__" ];
+          target_label = "__param_target";
+        }
+        {
+          target_label = "__address__";
+          replacement = blackboxAddr;
+        }
+      ];
+    }) queries
+  ) domains;
+in
+{
+  inherit modules scrapeConfigs blackboxAddr;
+}

modules/monitoring/blackbox.nix

@@ -0,0 +1,24 @@
+# Blackbox exporter on control: outside-in DNS probes against the public
+# nameserver addresses (see blackbox-probes.nix for what and why). Bound to
+# loopback — only VictoriaMetrics on the same host scrapes its /probe endpoint,
+# and the scrape jobs that drive it live in server.nix. The probes leave control
+# over the public internet to reach ns1/ns2, which is the path we want to test.
+{
+  lib,
+  pkgs,
+  ...
+}:
+let
+  probes = import ./blackbox-probes.nix { inherit lib; };
+in
+{
+  services.prometheus.exporters.blackbox = {
+    enable = true;
+    listenAddress = "127.0.0.1";
+    port = 9115;
+    # JSON is valid YAML; enableConfigCheck runs the exporter's own --config.check
+    # against this file at build time, so a malformed prober is caught here.
+    configFile = pkgs.writeText "blackbox.yml" (builtins.toJSON { inherit (probes) modules; });
+    enableConfigCheck = true;
+  };
+}

modules/monitoring/dashboards/backups.json

@@ -0,0 +1,217 @@
+{
+  "uid": "cnx-backups",
+  "title": "CNX Backups",
+  "tags": ["backup", "borg", "cnx"],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "version": 1,
+  "refresh": "1m",
+  "time": { "from": "now-7d", "to": "now" },
+  "templating": { "list": [] },
+  "annotations": { "list": [] },
+  "panels": [
+    {
+      "type": "row",
+      "title": "Backups",
+      "id": 1,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }
+    },
+    {
+      "type": "stat",
+      "title": "Backup health",
+      "description": "1 if any borgbackup job is in the failed state, 0 otherwise. A successful run leaves the oneshot unit inactive (still OK); only a real failure shows FAILED. Derived from the node_exporter systemd collector on the backup client (ns1).",
+      "id": 2,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 5, "w": 8, "x": 0, "y": 1 },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "thresholds" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "green", "value": null }]
+          },
+          "noValue": "no data",
+          "mappings": [
+            {
+              "type": "value",
+              "options": {
+                "0": { "text": "OK", "color": "green", "index": 0 },
+                "1": { "text": "FAILED", "color": "red", "index": 1 }
+              }
+            }
+          ]
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "auto",
+        "orientation": "auto"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "max(node_systemd_unit_state{name=~\"borgbackup-job-.+\\\\.service\",state=\"failed\"})",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "type": "stat",
+      "title": "Last backup run",
+      "description": "When the most recent backup timer last fired (the daily borgbackup job). 'No data' before the first run.",
+      "id": 3,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 5, "w": 8, "x": 8, "y": 1 },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "dateTimeFromNow",
+          "color": { "mode": "fixed", "fixedColor": "text" },
+          "noValue": "never"
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "none",
+        "graphMode": "none",
+        "textMode": "auto",
+        "orientation": "auto"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "max(node_systemd_timer_last_trigger_seconds{name=~\"borgbackup-job-.+\\\\.timer\"}) * 1000",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "type": "stat",
+      "title": "Time since last backup",
+      "description": "Age of the most recent backup. Backups run daily, so anything past ~25h means a run was missed. Red over 25h.",
+      "id": 4,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 5, "w": 8, "x": 16, "y": 1 },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s",
+          "color": { "mode": "thresholds" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "red", "value": 90000 }
+            ]
+          },
+          "noValue": "never"
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "auto",
+        "orientation": "auto"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "time() - max(node_systemd_timer_last_trigger_seconds{name=~\"borgbackup-job-.+\\\\.timer\"})",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "type": "table",
+      "title": "Backup jobs (current state)",
+      "description": "Every borgbackup job and the systemd unit state it is currently in, per client. 'inactive' is the normal resting state of a oneshot job between runs.",
+      "id": 5,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
+      "options": { "showHeader": true },
+      "fieldConfig": {
+        "defaults": { "custom": { "align": "auto" } },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "node_systemd_unit_state{name=~\"borgbackup-job-.+\\\\.service\"} == 1",
+          "format": "table",
+          "instant": true
+        }
+      ],
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true,
+              "Value": true,
+              "__name__": true,
+              "job": true,
+              "type": true
+            }
+          }
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Failed state over time",
+      "description": "1 while a backup job is in the failed state. A spike here is a backup that did not complete and was not retried before the next scrape.",
+      "id": 6,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
+      "fieldConfig": {
+        "defaults": { "unit": "short", "min": 0, "max": 1 },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "node_systemd_unit_state{name=~\"borgbackup-job-.+\\\\.service\",state=\"failed\"}",
+          "legendFormat": "{{instance}} {{name}}"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Time since last backup (history)",
+      "description": "Age of the latest backup over time. The sawtooth should reset to near zero once a day; a steady climb without a reset means backups stopped running.",
+      "id": 7,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 14 },
+      "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "time() - node_systemd_timer_last_trigger_seconds{name=~\"borgbackup-job-.+\\\\.timer\"}",
+          "legendFormat": "{{instance}} {{name}}"
+        }
+      ]
+    }
+  ]
+}

modules/monitoring/dashboards/dns.json

@@ -4,17 +4,58 @@
   "tags": ["dns", "knot", "cnx"],
   "timezone": "browser",
   "schemaVersion": 39,
-  "version": 1,
+  "version": 3,
   "refresh": "30s",
   "time": { "from": "now-6h", "to": "now" },
   "templating": { "list": [] },
   "annotations": { "list": [] },
   "panels": [
+    {
+      "type": "row",
+      "title": "Alerts",
+      "id": 11,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }
+    },
+    {
+      "type": "table",
+      "title": "Active alerts",
+      "description": "Firing vmalert alerts (the ALERTS series vmalert writes back to VictoriaMetrics). An empty table means all clear.",
+      "id": 12,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 7, "w": 24, "x": 0, "y": 1 },
+      "options": { "showHeader": true },
+      "fieldConfig": {
+        "defaults": { "custom": { "align": "auto" } },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "ALERTS{alertstate=\"firing\"}",
+          "format": "table",
+          "instant": true
+        }
+      ],
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true,
+              "Value": true,
+              "__name__": true,
+              "alertstate": true
+            }
+          }
+        }
+      ]
+    },
     {
       "type": "row",
       "title": "DNS / Zones",
       "id": 1,
-      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 8 }
     },
     {
       "type": "table",
@@ -22,7 +63,7 @@
       "description": "ns1 and ns2 should report the same serial per zone. A divergence here is the secondary-out-of-sync condition.",
       "id": 2,
       "datasource": { "type": "prometheus", "uid": "victoriametrics" },
-      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 },
       "options": { "showHeader": true },
       "fieldConfig": {
         "defaults": { "custom": { "align": "auto" } },
@@ -45,7 +86,7 @@
       "description": "On secondaries this counts down between successful transfers; a steady decline toward zero means transfers are failing.",
       "id": 3,
       "datasource": { "type": "prometheus", "uid": "victoriametrics" },
-      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 9 },
       "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] },
       "targets": [
         {
@@ -61,7 +102,7 @@
       "title": "Query rate by nameserver",
       "id": 4,
       "datasource": { "type": "prometheus", "uid": "victoriametrics" },
-      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 17 },
       "fieldConfig": { "defaults": { "unit": "qps" }, "overrides": [] },
       "targets": [
         {
@@ -77,7 +118,7 @@
       "title": "Response codes",
       "id": 5,
       "datasource": { "type": "prometheus", "uid": "victoriametrics" },
-      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 9 },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 17 },
       "fieldConfig": { "defaults": { "unit": "qps" }, "overrides": [] },
       "targets": [
         {
@@ -92,14 +133,14 @@
       "type": "row",
       "title": "Hosts",
       "id": 6,
-      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 17 }
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 25 }
     },
     {
       "type": "timeseries",
       "title": "CPU busy %",
       "id": 7,
       "datasource": { "type": "prometheus", "uid": "victoriametrics" },
-      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 18 },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 26 },
       "fieldConfig": {
         "defaults": { "unit": "percent", "min": 0, "max": 100 },
         "overrides": []
@@ -118,7 +159,7 @@
       "title": "Memory used %",
       "id": 8,
       "datasource": { "type": "prometheus", "uid": "victoriametrics" },
-      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 18 },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 26 },
       "fieldConfig": {
         "defaults": { "unit": "percent", "min": 0, "max": 100 },
         "overrides": []
@@ -137,7 +178,7 @@
       "title": "Root filesystem used %",
       "id": 9,
       "datasource": { "type": "prometheus", "uid": "victoriametrics" },
-      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 26 },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 34 },
       "fieldConfig": {
         "defaults": { "unit": "percent", "min": 0, "max": 100 },
         "overrides": []
@@ -156,7 +197,7 @@
       "title": "Load average (1m)",
       "id": 10,
       "datasource": { "type": "prometheus", "uid": "victoriametrics" },
-      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 26 },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 34 },
       "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
       "targets": [
         {
@@ -166,6 +207,120 @@
           "legendFormat": "{{instance}}"
         }
       ]
+    },
+    {
+      "type": "row",
+      "title": "DNS probes (outside-in)",
+      "id": 20,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 42 }
+    },
+    {
+      "type": "table",
+      "title": "Probe status (per zone / server)",
+      "description": "blackbox_exporter on control queries each nameserver's public address (v4 + v6) for every zone: an SOA query (zone served) and a DNSKEY query (still signed). UP = the resolver's-eye view is healthy.",
+      "id": 21,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 43 },
+      "options": { "showHeader": true },
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "align": "auto",
+            "cellOptions": { "type": "color-background" }
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "red", "value": null },
+              { "color": "green", "value": 1 }
+            ]
+          },
+          "mappings": [
+            {
+              "type": "value",
+              "options": {
+                "0": { "text": "DOWN", "index": 0 },
+                "1": { "text": "UP", "index": 1 }
+              }
+            }
+          ]
+        },
+        "overrides": [
+          {
+            "matcher": { "id": "byName", "options": "zone" },
+            "properties": [
+              { "id": "custom.cellOptions", "value": { "type": "auto" } }
+            ]
+          },
+          {
+            "matcher": { "id": "byName", "options": "query" },
+            "properties": [
+              { "id": "custom.cellOptions", "value": { "type": "auto" } }
+            ]
+          },
+          {
+            "matcher": { "id": "byName", "options": "instance" },
+            "properties": [
+              { "id": "custom.cellOptions", "value": { "type": "auto" } }
+            ]
+          }
+        ]
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "probe_success",
+          "format": "table",
+          "instant": true
+        }
+      ],
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": { "Time": true, "__name__": true, "job": true },
+            "renameByName": { "Value": "status" }
+          }
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Probe success (1 = ok)",
+      "description": "0 means the probe failed: the zone is not being served or not signed from that public address. Sustained failures fire DNSResolutionProbeFailed / DNSSECProbeFailed.",
+      "id": 22,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 43 },
+      "fieldConfig": {
+        "defaults": { "unit": "short", "min": 0, "max": 1 },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "probe_success",
+          "legendFormat": "{{zone}} {{query}} @ {{instance}}"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "DNS probe latency",
+      "description": "Total round-trip time of each blackbox DNS probe. A climbing trend points at a slow or overloaded nameserver before it starts failing outright.",
+      "id": 23,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 51 },
+      "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "probe_duration_seconds",
+          "legendFormat": "{{zone}} {{query}} @ {{instance}}"
+        }
+      ]
     }
   ]
 }

modules/monitoring/dashboards/uptime.json

@@ -0,0 +1,194 @@
+{
+  "uid": "cnx-uptime",
+  "title": "CNX Uptime",
+  "tags": ["uptime", "availability", "cnx"],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "version": 1,
+  "refresh": "30s",
+  "time": { "from": "now-24h", "to": "now" },
+  "templating": { "list": [] },
+  "annotations": { "list": [] },
+  "panels": [
+    {
+      "type": "row",
+      "title": "Uptime",
+      "id": 1,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }
+    },
+    {
+      "type": "stat",
+      "title": "Host status",
+      "description": "Whether VictoriaMetrics is currently able to scrape each host's node_exporter. UP means the host (and its mesh path) is reachable; DOWN means the scrape failed. One tile per machine.",
+      "id": 2,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 6, "w": 12, "x": 0, "y": 1 },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "thresholds" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "green", "value": null }]
+          },
+          "noValue": "no data",
+          "mappings": [
+            {
+              "type": "value",
+              "options": {
+                "0": { "text": "DOWN", "color": "red", "index": 0 },
+                "1": { "text": "UP", "color": "green", "index": 1 }
+              }
+            }
+          ]
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "value_and_name",
+        "orientation": "auto"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "up{job=\"node\"}",
+          "legendFormat": "{{instance}}",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "type": "stat",
+      "title": "Current uptime",
+      "description": "Time since each host last booted (now - node_boot_time_seconds). A value that drops back to near zero means the host rebooted.",
+      "id": 3,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 6, "w": 12, "x": 12, "y": 1 },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "dtdurations",
+          "color": { "mode": "fixed", "fixedColor": "text" },
+          "noValue": "no data"
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "none",
+        "graphMode": "none",
+        "textMode": "value_and_name",
+        "orientation": "auto"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "time() - node_boot_time_seconds{job=\"node\"}",
+          "legendFormat": "{{instance}}",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "type": "bargauge",
+      "title": "Availability over window",
+      "description": "Fraction of successful scrapes over the selected time range, per host (avg of up over $__range). 100% means every scrape in the window succeeded; dips reveal flapping or outages. Red below 99%.",
+      "id": 4,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 7 },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "color": { "mode": "thresholds" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "red", "value": null },
+              { "color": "yellow", "value": 99 },
+              { "color": "green", "value": 99.9 }
+            ]
+          },
+          "noValue": "no data"
+        },
+        "overrides": []
+      },
+      "options": {
+        "displayMode": "gradient",
+        "orientation": "horizontal",
+        "showUnfilled": true,
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        }
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "avg_over_time(up{job=\"node\"}[$__range]) * 100",
+          "legendFormat": "{{instance}}",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Uptime over time",
+      "description": "Host uptime across the window. The line should climb steadily; a reset to zero marks a reboot.",
+      "id": 5,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 7 },
+      "fieldConfig": {
+        "defaults": { "unit": "s", "custom": { "fillOpacity": 0 } },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "time() - node_boot_time_seconds{job=\"node\"}",
+          "legendFormat": "{{instance}}"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Up/down history",
+      "description": "1 while a host's node_exporter was scrapeable, 0 while it was not. Gaps to zero are outages or lost mesh connectivity.",
+      "id": 6,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 6, "w": 24, "x": 0, "y": 15 },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "min": 0,
+          "max": 1,
+          "custom": { "fillOpacity": 20, "lineInterpolation": "stepAfter" }
+        },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "up{job=\"node\"}",
+          "legendFormat": "{{instance}}"
+        }
+      ]
+    }
+  ]
+}

modules/monitoring/dashboards/wan.json

@@ -0,0 +1,267 @@
+{
+  "uid": "cnx-wan",
+  "title": "CNX WAN Speed",
+  "tags": ["wan", "speedtest", "gateway", "cnx"],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "version": 1,
+  "refresh": "5m",
+  "time": { "from": "now-7d", "to": "now" },
+  "templating": { "list": [] },
+  "annotations": { "list": [] },
+  "panels": [
+    {
+      "type": "row",
+      "title": "WAN speed (site gateways)",
+      "id": 1,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }
+    },
+    {
+      "type": "stat",
+      "title": "Download (latest)",
+      "description": "Most recent hourly librespeed result per gateway.",
+      "id": 2,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 5, "w": 6, "x": 0, "y": 1 },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bps",
+          "color": { "mode": "fixed", "fixedColor": "green" },
+          "noValue": "no data"
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "value",
+        "graphMode": "area",
+        "textMode": "value_and_name",
+        "orientation": "auto"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "speedtest_download_mbps * 1e6",
+          "legendFormat": "{{instance}}",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "type": "stat",
+      "title": "Upload (latest)",
+      "description": "Most recent hourly librespeed result per gateway.",
+      "id": 3,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 5, "w": 6, "x": 6, "y": 1 },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bps",
+          "color": { "mode": "fixed", "fixedColor": "blue" },
+          "noValue": "no data"
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "value",
+        "graphMode": "area",
+        "textMode": "value_and_name",
+        "orientation": "auto"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "speedtest_upload_mbps * 1e6",
+          "legendFormat": "{{instance}}",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "type": "stat",
+      "title": "Ping (latest)",
+      "description": "Latency to the librespeed test server. Yellow over 50 ms, red over 100 ms.",
+      "id": 4,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 5, "w": 6, "x": 12, "y": 1 },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms",
+          "color": { "mode": "thresholds" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 50 },
+              { "color": "red", "value": 100 }
+            ]
+          },
+          "noValue": "no data"
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "value",
+        "graphMode": "none",
+        "textMode": "value_and_name",
+        "orientation": "auto"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "speedtest_ping_ms",
+          "legendFormat": "{{instance}}",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "type": "stat",
+      "title": "Test status",
+      "description": "Whether the last speed test on each gateway completed. FAILING also fires the WANSpeedTestFailing alert after 3h.",
+      "id": 5,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 5, "w": 6, "x": 18, "y": 1 },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "thresholds" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "green", "value": null }]
+          },
+          "noValue": "no data",
+          "mappings": [
+            {
+              "type": "value",
+              "options": {
+                "0": { "text": "FAILING", "color": "red", "index": 0 },
+                "1": { "text": "OK", "color": "green", "index": 1 }
+              }
+            }
+          ]
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "value_and_name",
+        "orientation": "auto"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "speedtest_success",
+          "legendFormat": "{{instance}}",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Download",
+      "description": "Hourly download rate per gateway, with each link's own 7-day median as reference — the WANSpeedDegraded alert fires when a link stays below half its median for 3h.",
+      "id": 6,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 9, "w": 12, "x": 0, "y": 6 },
+      "fieldConfig": {
+        "defaults": { "unit": "bps", "min": 0 },
+        "overrides": [
+          {
+            "matcher": { "id": "byRegexp", "options": ".*7d median.*" },
+            "properties": [
+              {
+                "id": "custom.lineStyle",
+                "value": { "fill": "dash", "dash": [10, 10] }
+              },
+              { "id": "custom.fillOpacity", "value": 0 }
+            ]
+          }
+        ]
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "speedtest_download_mbps * 1e6",
+          "legendFormat": "{{instance}}"
+        },
+        {
+          "refId": "B",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "quantile_over_time(0.5, speedtest_download_mbps[7d]) * 1e6",
+          "legendFormat": "{{instance}} 7d median"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Upload",
+      "description": "Hourly upload rate per gateway.",
+      "id": 7,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 9, "w": 12, "x": 12, "y": 6 },
+      "fieldConfig": {
+        "defaults": { "unit": "bps", "min": 0 },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "speedtest_upload_mbps * 1e6",
+          "legendFormat": "{{instance}}"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Latency & jitter",
+      "description": "Ping and jitter towards the librespeed test server. A latency climb with stable throughput usually means bufferbloat or an ISP routing change.",
+      "id": 8,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 15 },
+      "fieldConfig": {
+        "defaults": { "unit": "ms", "min": 0 },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "speedtest_ping_ms",
+          "legendFormat": "{{instance}} ping"
+        },
+        {
+          "refId": "B",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "speedtest_jitter_ms",
+          "legendFormat": "{{instance}} jitter"
+        }
+      ]
+    }
+  ]
+}

modules/monitoring/exporters.nix

@@ -1,5 +1,6 @@
-# Metric exporters, imported by every machine. Host metrics everywhere; Knot DNS
-# metrics on the nameservers. Everything is reachable only over the ZeroTier mesh
+# Per-host observability agents, imported by every machine. Host metrics
+# everywhere; Knot DNS metrics on the nameservers; journald shipped to
+# VictoriaLogs on control. Everything is reachable only over the ZeroTier mesh
 # (see the firewall rule at the bottom); the public side is already closed by the
 # Hetzner cloud firewall.
 {
@@ -86,6 +87,30 @@ in
     ];
   };
 
+  # Ship journald to VictoriaLogs on control (services.victorialogs in
+  # server.nix). control uploads to loopback so its own logs survive a mesh
+  # outage; ns1/ns2 push over the mesh to control's ZeroTier address.
+  services.journald.upload = {
+    enable = true;
+    settings.Upload.URL =
+      let
+        dest =
+          if config.networking.hostName == "control" then
+            "127.0.0.1:9428"
+          else
+            "[${mesh.hosts.control}]:9428";
+      in
+      "http://${dest}/insert/journald";
+  };
+
+  # systemd-journal-upload exits if the sink is unreachable. Upstream already
+  # restarts it (Restart=always/RestartSec=3sec), but the default start-rate limit
+  # (5 tries / 10s) lets it give up permanently — so a transient VictoriaLogs
+  # outage leaves the uploader dead until the next deploy. Disable the limit so it
+  # retries forever and self-heals once the sink returns. (A persistent failure
+  # still surfaces loudly in a deploy, which is what we want.)
+  systemd.services.systemd-journal-upload.startLimitIntervalSec = 0;
+
   # Scrape ports reachable only from the ZeroTier mesh.
   networking.firewall.extraInputRules = ''
     ip6 saddr ${mesh.subnet} tcp dport { ${lib.concatMapStringsSep ", " toString ports} } accept

modules/monitoring/parsedmarc.nix

@@ -0,0 +1,122 @@
+# DMARC report analyzer, imported by control only. parsedmarc fetches the
+# aggregate/forensic reports that land in the dmarc@cnx.email mailbox on mx1,
+# parses the XML, and stores results in a local Elasticsearch; the official
+# parsedmarc dashboard + an Elasticsearch datasource are auto-provisioned into
+# the Grafana instance that server.nix already runs on this host.
+#
+# IMAP runs over the ZeroTier mesh, not the public net: we pin mx1.cnx.email to
+# its mesh address in /etc/hosts so TLS still validates against the public
+# Let's Encrypt cert (primary domain mx1.cnx.email) while the bytes stay on the
+# overlay. The mailbox passphrase is the shared mail-dmarc-cred secret; parsedmarc
+# reads it as root in its ExecStartPre, so root-owned (clan default) is fine.
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  mesh = import ../mesh-hosts.nix { inherit config lib; };
+
+  # Grafana 13 (since the externalisation of core data sources) no longer bundles
+  # the Elasticsearch data source, and nixpkgs has no grafanaPlugins entry for it,
+  # so a bare elasticsearch datasource reports "Plugin not registered" and every
+  # DMARC panel fails. Pull the official, Grafana-signed plugin from the catalog
+  # and load it declaratively.
+  elasticsearchPlugin = pkgs.grafanaPlugins.grafanaPlugin {
+    pname = "elasticsearch";
+    version = "12.6.4";
+    zipHash.x86_64-linux = "sha256-xyAUprdWyQM0IJSg/oBVZ0ltFAAffgPrphln9+IKcUY=";
+  };
+in
+{
+  imports = [ ../mail-dmarc-cred.nix ];
+
+  # Elasticsearch 7.x is under the (unfree) Elastic License; allow just this one
+  # package rather than opening allowUnfree globally.
+  nixpkgs.config.allowUnfreePredicate = pkg: lib.getName pkg == "elasticsearch";
+
+  services.grafana.declarativePlugins = [ elasticsearchPlugin ];
+
+  # Keep mx1's IMAP traffic on the mesh while presenting the public cert name.
+  networking.hosts.${mesh.hosts.mx1} = [ "mx1.cnx.email" ];
+
+  services.parsedmarc = {
+    enable = true;
+    provision = {
+      # Local Elasticsearch on 127.0.0.1:9200 (loopback; no firewall change).
+      # datasource + dashboard default to true once ES and Grafana are both on.
+      elasticsearch = true;
+      # GeoIP needs a MaxMind account/license key; skip it (reports still parse,
+      # just without source-IP geolocation).
+      geoIp = false;
+      grafana = {
+        # We provision the two Elasticsearch datasources ourselves (below) so we
+        # can pin their UIDs. Leaving this on would add a second, uid-less pair
+        # and collide on the dmarc-ag/dmarc-fo names.
+        datasource = false;
+        dashboard = true;
+      };
+    };
+    settings = {
+      imap = {
+        host = "mx1.cnx.email";
+        port = 993;
+        ssl = true;
+        user = "dmarc@cnx.email";
+        password = {
+          _secret = config.clan.core.vars.generators.mail-dmarc-cred.files."passphrase".path;
+        };
+      };
+      mailbox = {
+        watch = true; # IMAP IDLE: process reports as they arrive
+        delete = false; # archive processed reports, don't delete
+      };
+      general = {
+        save_aggregate = true;
+        save_forensic = true;
+      };
+    };
+  };
+
+  # The bundled parsedmarc dashboard selects its datasource through template
+  # variables ($datasourceag/$datasourcefo) whose stored value is the string
+  # "dmarc-ag"/"dmarc-fo". Grafana 11+ resolves a datasource template variable
+  # by UID, not by name, so without a matching UID every panel renders
+  # "datasource was not found". Pin the UIDs to those names so it resolves.
+  services.grafana.provision.datasources.settings = {
+    # These two were previously provisioned uid-less, so they exist in Grafana's
+    # DB with random UIDs. Provisioning a *new* UID onto an existing datasource
+    # makes Grafana abort with "data source not found" and crash-loop, so delete
+    # the stale records first; the entries below re-create them with pinned UIDs.
+    deleteDatasources = [
+      {
+        name = "dmarc-ag";
+        orgId = 1;
+      }
+      {
+        name = "dmarc-fo";
+        orgId = 1;
+      }
+    ];
+    datasources =
+      let
+        esVersion = lib.getVersion config.services.elasticsearch.package;
+        es = name: {
+          inherit name;
+          uid = name;
+          type = "elasticsearch";
+          access = "proxy";
+          url = "http://localhost:9200";
+          jsonData = {
+            timeField = "date_range";
+            inherit esVersion;
+          };
+        };
+      in
+      [
+        (es "dmarc-ag")
+        (es "dmarc-fo")
+      ];
+  };
+}

modules/monitoring/server.nix

@@ -10,7 +10,9 @@
 }:
 let
   mesh = import ../mesh-hosts.nix { inherit config lib; };
+  probes = import ./blackbox-probes.nix { inherit lib; };
   vmPort = 8428;
+  logsPort = 9428;
   grafanaPort = 3000;
   controlV6 = mesh.hosts.control;
 
@@ -23,6 +25,7 @@ let
   v6 = addr: "[${addr}]";
 
   adminPasswordFile = config.clan.core.vars.generators.grafana-admin.files."password".path;
+  secretKeyFile = config.clan.core.vars.generators.grafana-secret-key.files."secret-key".path;
 in
 {
   services.victoriametrics = {
@@ -43,6 +46,15 @@ in
             (target "control" "127.0.0.1" 9100)
             (target "ns1" (v6 mesh.hosts.ns1) 9100)
             (target "ns2" (v6 mesh.hosts.ns2) 9100)
+            (target "mx1" (v6 mesh.hosts.mx1) 9100)
+            (target "web01" (v6 mesh.hosts.web01) 9100)
+            (target "gw-cnx-1" (v6 mesh.hosts.gw-cnx-1) 9100)
+          ];
+        }
+        {
+          job_name = "blocky";
+          static_configs = [
+            (target "gw-cnx-1" (v6 mesh.hosts.gw-cnx-1) 4000)
           ];
         }
         {
@@ -52,10 +64,31 @@ in
             (target "ns2" (v6 mesh.hosts.ns2) 9433)
           ];
         }
-      ];
+      ]
+      # Outside-in DNS probes via the blackbox exporter (blackbox.nix). The job
+      # list is generated from the same probe definitions the exporter uses.
+      ++ probes.scrapeConfigs;
     };
   };
 
+  # Centralized logs: VictoriaLogs ingests journald from all three hosts, each
+  # of which runs systemd-journal-upload against /insert/journald (exporters.nix).
+  # Binds all interfaces because ns1/ns2 push over the mesh; the firewall rule at
+  # the bottom scopes 9428 to the mesh subnet and the Hetzner firewall closes the
+  # public side. Retention is set via extraOptions (no dedicated NixOS option).
+  services.victorialogs = {
+    enable = true;
+    listenAddress = ":${toString logsPort}";
+    # -enableTCP6: like the scraper above, VictoriaLogs is IPv4-only by default
+    # for *listening* too — ":9428" binds 0.0.0.0 only, so ns1/ns2 pushing over
+    # the IPv6 mesh get "connection refused". This makes it bind [::] (dual-stack)
+    # so the mesh can reach it. Retention has no dedicated NixOS option.
+    extraOptions = [
+      "-retentionPeriod=30d"
+      "-enableTCP6"
+    ];
+  };
+
   # Admin password generated once and stored as a clan secret. Retrieve with:
   #   clan vars get control grafana-admin/password
   clan.core.vars.generators.grafana-admin = {
@@ -70,8 +103,27 @@ in
     '';
   };
 
+  # NixOS 26.05 removed the built-in default for services.grafana's secret_key
+  # (used to sign/encrypt DB secrets). Mint our own instead of hardcoding the old
+  # upstream default. Safe to generate fresh here: the provisioned datasources
+  # carry no credentials, so nothing in Grafana's DB is encrypted with it.
+  clan.core.vars.generators.grafana-secret-key = {
+    files."secret-key" = {
+      secret = true;
+      owner = "grafana";
+      group = "grafana";
+    };
+    runtimeInputs = [ pkgs.openssl ];
+    script = ''
+      openssl rand -base64 32 | tr -d "\n" > "$out"/secret-key
+    '';
+  };
+
   services.grafana = {
     enable = true;
+    # VictoriaLogs datasource plugin so journald is greppable from Grafana,
+    # alongside the metrics datasource.
+    declarativePlugins = [ pkgs.grafanaPlugins.victoriametrics-logs-datasource ];
     settings = {
       server = {
         http_addr = "::";
@@ -81,6 +133,7 @@ in
       security = {
         admin_user = "admin";
         admin_password = "$__file{${adminPasswordFile}}";
+        secret_key = "$__file{${secretKeyFile}}";
       };
       "auth.anonymous".enabled = false;
       users.allow_sign_up = false;
@@ -98,6 +151,13 @@ in
             url = "http://127.0.0.1:${toString vmPort}";
             isDefault = true;
           }
+          {
+            name = "VictoriaLogs";
+            type = "victoriametrics-logs-datasource";
+            uid = "victorialogs";
+            access = "proxy";
+            url = "http://127.0.0.1:${toString logsPort}";
+          }
         ];
       };
       dashboards.settings = {
@@ -113,8 +173,9 @@ in
     };
   };
 
-  # Grafana reachable only from the ZeroTier mesh (admin laptops + servers).
+  # Grafana (admin laptops + servers) and VictoriaLogs ingestion (ns1/ns2 push
+  # journald over the mesh) reachable only from the ZeroTier mesh.
   networking.firewall.extraInputRules = ''
-    ip6 saddr ${mesh.subnet} tcp dport ${toString grafanaPort} accept
+    ip6 saddr ${mesh.subnet} tcp dport { ${toString grafanaPort}, ${toString logsPort} } accept
   '';
 }

modules/router/crowdsec.nix

@@ -0,0 +1,52 @@
+# CrowdSec security engine + nftables bouncer: parses sshd auth attempts from
+# the journal and bans offending source IPs at the firewall. Log-based (no
+# inline DPI) so it costs the N300 next to nothing.
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.cnx.router;
+  mesh = import ../mesh-hosts.nix { inherit config lib; };
+in
+{
+  config = lib.mkIf cfg.enable {
+    services.crowdsec = {
+      enable = true;
+      autoUpdateService = true;
+      hub.collections = [
+        "crowdsecurity/linux"
+        "crowdsecurity/sshd"
+      ];
+      localConfig = {
+        acquisitions = [
+          {
+            source = "journalctl";
+            journalctl_filter = [ "_SYSTEMD_UNIT=sshd.service" ];
+            labels.type = "syslog";
+          }
+        ];
+        # Never ban the ZeroTier mesh — it is the only admin path to these
+        # boxes (no public SSH), so a false positive would lock us out.
+        # Parser-stage whitelist: mesh events are dropped before any scenario.
+        parsers.s02Enrich = [
+          {
+            name = "cnx/mesh-whitelist";
+            description = "Whitelist the ZeroTier management mesh";
+            whitelist = {
+              reason = "ZeroTier mesh is the admin path";
+              cidr = [ mesh.subnet ];
+            };
+          }
+        ];
+      };
+    };
+
+    services.crowdsec-firewall-bouncer = {
+      enable = true;
+      registerBouncer.enable = true;
+      settings.mode = "nftables";
+    };
+  };
+}

modules/router/default.nix

@@ -0,0 +1,325 @@
+# Site gateway (OPNsense replacement) for the Topton 1U boxes: PPPoE WAN,
+# VLAN-filtering bridge over the LAN ports, and per-VLAN L3 interfaces.
+# Imported by machines/gw-<city>-<n>; everything is driven by cnx.router.*.
+#
+# Fleet addressing convention: each site owns 10.<siteId>.0.0/16. A VLAN's
+# subnet defaults to 10.<siteId>.<vlanId>.0/24 with the router at .1 and the
+# DHCP pool at .100-.199. VLANs that need more space (e.g. public-wifi guest)
+# override `subnet`/`address`/`dhcp.pool` and take a wider block from the
+# upper half (10.<siteId>.128.0/17), e.g. guest -> 10.<siteId>.128.0/22.
+# VLAN ids: 10 = mgmt, 20 = lan (mandatory); 30 = guest, 40 = iot (reserved).
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.cnx.router;
+
+  vlanIf = name: "vlan-${name}";
+  site = toString cfg.siteId;
+
+  vlanModule =
+    { name, config, ... }:
+    let
+      octet = toString config.id;
+    in
+    {
+      options = {
+        id = lib.mkOption {
+          type = lib.types.ints.between 1 4094;
+          description = "802.1Q VLAN id (fleet convention: 10 mgmt, 20 lan, 30 guest, 40 iot).";
+        };
+        address = lib.mkOption {
+          type = lib.types.str;
+          default = "10.${site}.${octet}.1";
+          description = "Router address on this VLAN.";
+        };
+        prefixLength = lib.mkOption {
+          type = lib.types.ints.between 8 30;
+          default = 24;
+        };
+        subnet = lib.mkOption {
+          type = lib.types.str;
+          default = "10.${site}.${octet}.0/24";
+          description = "The VLAN's network in CIDR form (must contain `address`).";
+        };
+        dhcp = {
+          enable = lib.mkOption {
+            type = lib.types.bool;
+            default = true;
+          };
+          pool = {
+            from = lib.mkOption {
+              type = lib.types.str;
+              default = "10.${site}.${octet}.100";
+            };
+            to = lib.mkOption {
+              type = lib.types.str;
+              default = "10.${site}.${octet}.199";
+            };
+          };
+          leaseTime = lib.mkOption {
+            type = lib.types.ints.positive;
+            default = 86400;
+            description = ''
+              Lease validity in seconds. Lower it for high-churn networks,
+              e.g. public-WiFi guest VLANs (3600-7200), so the pool recycles.
+            '';
+          };
+          reservations = lib.mkOption {
+            type = lib.types.attrsOf (
+              lib.types.submodule {
+                options = {
+                  hwAddress = lib.mkOption {
+                    type = lib.types.str;
+                    example = "aa:bb:cc:dd:ee:ff";
+                    description = "Client MAC address.";
+                  };
+                  ipAddress = lib.mkOption {
+                    type = lib.types.str;
+                    description = "Fixed address handed to this client (inside the VLAN's subnet, outside the pool).";
+                  };
+                };
+              }
+            );
+            default = { };
+            description = "Static DHCP leases; the attribute name becomes the client's hostname.";
+          };
+        };
+        allowWan = lib.mkOption {
+          type = lib.types.bool;
+          default = true;
+          description = "Whether clients on this VLAN may reach the internet.";
+        };
+      };
+    };
+in
+{
+  imports = [
+    ./pppoe.nix
+    ./ipv6.nix
+    ./firewall.nix
+    ./dns-dhcp.nix
+    ./crowdsec.nix
+    ./omada.nix
+    ./proxy.nix
+    ./iperf.nix
+    ./speedtest.nix
+  ];
+
+  options.cnx.router = {
+    enable = lib.mkEnableOption "site gateway (router) role";
+
+    site = lib.mkOption {
+      type = lib.types.str;
+      description = "City code of the site, e.g. \"cnx\".";
+    };
+
+    siteId = lib.mkOption {
+      type = lib.types.ints.between 1 254;
+      description = "Site number; drives the 10.<siteId>.<vlan>.0/24 addressing.";
+    };
+
+    wan.interface = lib.mkOption {
+      type = lib.types.str;
+      description = "Physical WAN port the PPPoE session runs on.";
+    };
+
+    wan.vlanId = lib.mkOption {
+      type = lib.types.nullOr (lib.types.ints.between 1 4094);
+      default = null;
+      description = ''
+        802.1Q tag the ISP requires for the PPPoE session (AIS Thailand: 10);
+        null for untagged PPPoE directly on the port. Unrelated to the LAN
+        VLANs — this tag exists only on the WAN port.
+      '';
+    };
+
+    wan.macAddress = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      example = "aa:bb:cc:dd:ee:ff";
+      description = ''
+        Spoofed MAC for the WAN port, e.g. to keep the MAC the ISP has
+        pinned (cloned from the old router). null keeps the hardware MAC.
+      '';
+    };
+
+    wan.pppInterface = lib.mkOption {
+      type = lib.types.str;
+      internal = true;
+      readOnly = true;
+      default = if cfg.wan.vlanId == null then cfg.wan.interface else "wan-vlan";
+      description = "Interface pppd dials on (the WAN port or its ISP VLAN).";
+    };
+
+    trunkPorts = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      description = "LAN ports carrying all VLANs tagged (incl. any 10G SFP+ ports).";
+    };
+
+    accessPorts = lib.mkOption {
+      type = lib.types.attrsOf lib.types.str;
+      default = { };
+      example = {
+        enp4s0 = "mgmt";
+      };
+      description = ''
+        Ports acting as untagged access ports on a single VLAN (port name ->
+        VLAN name). Frames are untagged on the wire; the bridge tags them with
+        the VLAN's PVID. Use for an always-available on-site mgmt port.
+      '';
+    };
+
+    vlans = lib.mkOption {
+      type = lib.types.attrsOf (lib.types.submodule vlanModule);
+      description = "VLANs served at this site; `mgmt` and `lan` are mandatory.";
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = cfg.vlans ? mgmt && cfg.vlans ? lan;
+        message = "cnx.router: every site must define the `mgmt` and `lan` VLANs.";
+      }
+      {
+        assertion = lib.all (v: cfg.vlans ? ${v}) (lib.attrValues cfg.accessPorts);
+        message = "cnx.router: every accessPorts value must name a defined VLAN.";
+      }
+      {
+        assertion = lib.all (p: !(cfg.accessPorts ? ${p})) cfg.trunkPorts;
+        message = "cnx.router: a port cannot be both a trunk and an access port.";
+      }
+    ];
+
+    # Router diagnostics toolkit: packets (tcpdump), path (mtr), link
+    # negotiation (ethtool), NAT state (conntrack), DNS (kdig), per-flow
+    # bandwidth (iftop), WAN throughput (librespeed-cli; iperf3 covers LAN).
+    environment.systemPackages = with pkgs; [
+      tcpdump
+      mtr
+      ethtool
+      conntrack-tools
+      knot-dns
+      iftop
+      librespeed-cli
+    ];
+
+    networking.useNetworkd = true;
+    networking.useDHCP = false;
+    systemd.network.enable = true;
+
+    systemd.network.netdevs = {
+      "20-br0" = {
+        netdevConfig = {
+          Name = "br0";
+          Kind = "bridge";
+        };
+        bridgeConfig.VLANFiltering = true;
+      };
+    }
+    // lib.optionalAttrs (cfg.wan.vlanId != null) {
+      "15-wan-vlan" = {
+        netdevConfig = {
+          Name = "wan-vlan";
+          Kind = "vlan";
+        };
+        vlanConfig.Id = cfg.wan.vlanId;
+      };
+    }
+    // lib.mapAttrs' (
+      name: vlan:
+      lib.nameValuePair "30-${vlanIf name}" {
+        netdevConfig = {
+          Name = vlanIf name;
+          Kind = "vlan";
+        };
+        vlanConfig.Id = vlan.id;
+      }
+    ) cfg.vlans;
+
+    systemd.network.networks =
+      let
+        taggedAll = lib.mapAttrsToList (_: vlan: { VLAN = vlan.id; }) cfg.vlans;
+      in
+      {
+        # WAN port carries only the PPPoE session; no IP config of its own.
+        "10-wan" = {
+          matchConfig.Name = cfg.wan.interface;
+          networkConfig.LinkLocalAddressing = "no";
+          vlan = lib.optional (cfg.wan.vlanId != null) "wan-vlan";
+          linkConfig = {
+            RequiredForOnline = "carrier";
+          }
+          # The wan-vlan subinterface (and thus the PPPoE session) inherits
+          # the parent port's MAC, so spoofing here covers both cases.
+          // lib.optionalAttrs (cfg.wan.macAddress != null) {
+            MACAddress = cfg.wan.macAddress;
+          };
+        };
+      }
+      // lib.optionalAttrs (cfg.wan.vlanId != null) {
+        # The ISP-side VLAN subinterface pppd dials on (e.g. AIS tags PPPoE).
+        "15-wan-vlan" = {
+          matchConfig.Name = "wan-vlan";
+          networkConfig.LinkLocalAddressing = "no";
+          linkConfig.RequiredForOnline = "no";
+        };
+      }
+      // {
+        # The bridge itself is L2-only; L3 lives on the vlan-* interfaces,
+        # which hang off the bridge (tagged on the bridge "self" port).
+        "20-br0" = {
+          matchConfig.Name = "br0";
+          networkConfig.LinkLocalAddressing = "no";
+          vlan = lib.mapAttrsToList (name: _: vlanIf name) cfg.vlans;
+          bridgeVLANs = taggedAll;
+          linkConfig.RequiredForOnline = "no";
+        };
+      }
+      // lib.listToAttrs (
+        map (port: {
+          name = "25-trunk-${port}";
+          value = {
+            matchConfig.Name = port;
+            networkConfig.Bridge = "br0";
+            bridgeVLANs = taggedAll;
+            linkConfig.RequiredForOnline = "no";
+          };
+        }) cfg.trunkPorts
+      )
+      // lib.mapAttrs' (
+        port: vlanName:
+        lib.nameValuePair "25-access-${port}" {
+          matchConfig.Name = port;
+          networkConfig.Bridge = "br0";
+          bridgeVLANs = [
+            {
+              VLAN = cfg.vlans.${vlanName}.id;
+              PVID = cfg.vlans.${vlanName}.id;
+              EgressUntagged = cfg.vlans.${vlanName}.id;
+            }
+          ];
+          linkConfig.RequiredForOnline = "no";
+        }
+      ) cfg.accessPorts
+      // lib.mapAttrs' (
+        name: vlan:
+        lib.nameValuePair "40-${vlanIf name}" {
+          matchConfig.Name = vlanIf name;
+          address = [ "${vlan.address}/${toString vlan.prefixLength}" ];
+          networkConfig = {
+            IPv6AcceptRA = false;
+            # Announce a /64 carved from the DHCPv6-PD prefix on ppp0 (SLAAC).
+            IPv6SendRA = true;
+            DHCPPrefixDelegation = true;
+          };
+          dhcpPrefixDelegationConfig.SubnetId = "auto";
+          linkConfig.RequiredForOnline = "no";
+        }
+      ) cfg.vlans;
+  };
+}

modules/router/dns-dhcp.nix

@@ -0,0 +1,84 @@
+# LAN DHCP (Kea) and DNS (Blocky). Fully declarative: one Kea subnet per VLAN
+# with dhcp.enable, Blocky as the blocklist resolver every DHCP lease points
+# at. Blocky's HTTP listener (:4000) serves Prometheus metrics, scraped by
+# control over the mesh (firewall.nix scopes it to the mesh subnet).
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.cnx.router;
+  dhcpVlans = lib.filterAttrs (_: vlan: vlan.dhcp.enable) cfg.vlans;
+in
+{
+  config = lib.mkIf cfg.enable {
+    services.kea.dhcp4 = {
+      enable = true;
+      settings = {
+        interfaces-config.interfaces = lib.mapAttrsToList (name: _: "vlan-${name}") dhcpVlans;
+        lease-database = {
+          type = "memfile";
+          persist = true;
+          name = "/var/lib/kea/dhcp4.leases";
+        };
+        valid-lifetime = 86400;
+        subnet4 = lib.mapAttrsToList (name: vlan: {
+          id = vlan.id;
+          subnet = vlan.subnet;
+          interface = "vlan-${name}";
+          valid-lifetime = vlan.dhcp.leaseTime;
+          pools = [ { pool = "${vlan.dhcp.pool.from} - ${vlan.dhcp.pool.to}"; } ];
+          reservations = lib.mapAttrsToList (host: res: {
+            hostname = host;
+            hw-address = res.hwAddress;
+            ip-address = res.ipAddress;
+          }) vlan.dhcp.reservations;
+          option-data = [
+            {
+              name = "routers";
+              data = vlan.address;
+            }
+            {
+              name = "domain-name-servers";
+              data = vlan.address;
+            }
+          ];
+        }) dhcpVlans;
+      };
+    };
+
+    services.blocky = {
+      enable = true;
+      settings = {
+        ports = {
+          dns = 53;
+          http = 4000;
+        };
+        upstreams.groups.default = [
+          "9.9.9.9"
+          "149.112.112.112"
+          "2620:fe::fe"
+        ];
+        blocking = {
+          denylists.ads = [
+            "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts"
+          ];
+          clientGroupsBlock.default = [ "ads" ];
+        };
+        caching = {
+          minTime = "5m";
+          prefetching = true;
+        };
+        prometheus.enable = true;
+      };
+    };
+
+    # The router itself resolves via public resolvers, not via Blocky, so DNS
+    # for deploys/updates survives a broken local resolver.
+    networking.nameservers = [
+      "9.9.9.9"
+      "1.1.1.1"
+    ];
+  };
+}

modules/router/firewall.nix

@@ -0,0 +1,62 @@
+# Router firewall/NAT policy (nftables). Trust model:
+#   mgmt VLAN  -> trusted: router services, all VLANs, WAN
+#   other VLANs -> DNS/DHCP on the router + WAN (if allowWan); no inter-VLAN
+#   WAN (ppp0) -> nothing inbound beyond established/related
+#   mesh       -> admin SSH + metrics scrapes (same trust boundary as the fleet)
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.cnx.router;
+  mesh = import ../mesh-hosts.nix { inherit config lib; };
+
+  vlanIfs = lib.mapAttrsToList (name: _: "vlan-${name}") cfg.vlans;
+  wanVlanIfs = lib.mapAttrsToList (name: _: "vlan-${name}") (
+    lib.filterAttrs (_: vlan: vlan.allowWan) cfg.vlans
+  );
+  nonMgmtIfs = lib.filter (i: i != "vlan-mgmt") vlanIfs;
+  ifSet = ifs: "{ ${lib.concatStringsSep ", " (map (i: "\"${i}\"") ifs)} }";
+in
+{
+  config = lib.mkIf cfg.enable {
+    networking.nftables.enable = true;
+
+    # SSH reachable only from the mgmt VLAN (trusted) and the mesh — never
+    # from the WAN or the other VLANs.
+    services.openssh.openFirewall = false;
+
+    networking.firewall = {
+      enable = true;
+      filterForward = true;
+      trustedInterfaces = [ "vlan-mgmt" ];
+
+      # Non-mgmt VLANs may only talk to the router's DNS and DHCP.
+      interfaces = lib.genAttrs nonMgmtIfs (_: {
+        allowedTCPPorts = [ 53 ];
+        allowedUDPPorts = [
+          53
+          67
+        ];
+      });
+
+      extraInputRules = ''
+        ip6 saddr ${mesh.subnet} tcp dport 22 accept comment "admin ssh over the mesh"
+        ip6 saddr ${mesh.subnet} tcp dport 4000 accept comment "blocky metrics scrape from control"
+      '';
+
+      extraForwardRules = ''
+        tcp flags syn tcp option maxseg size set rt mtu comment "MSS clamp for PPPoE mtu 1492"
+        iifname "vlan-mgmt" accept comment "mgmt reaches all VLANs and the WAN"
+        iifname ${ifSet wanVlanIfs} oifname "ppp0" accept comment "LAN to internet"
+      '';
+    };
+
+    networking.nat = {
+      enable = true;
+      externalInterface = "ppp0";
+      internalInterfaces = vlanIfs;
+    };
+  };
+}

modules/router/iperf.nix

@@ -0,0 +1,28 @@
+# iperf3 server on every gateway, for throughput testing from any LAN segment
+# (e.g. validating AP/switch links: `iperf3 -c 10.<siteId>.<vlan>.1`) and from
+# admin machines over the mesh. Never reachable from the WAN (default-deny).
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.cnx.router;
+  mesh = import ../mesh-hosts.nix { inherit config lib; };
+  vlanIfs = lib.mapAttrsToList (name: _: "vlan-${name}") cfg.vlans;
+in
+{
+  config = lib.mkIf cfg.enable {
+    services.iperf3.enable = true;
+
+    networking.firewall.interfaces = lib.genAttrs vlanIfs (_: {
+      allowedTCPPorts = [ 5201 ];
+      allowedUDPPorts = [ 5201 ];
+    });
+
+    networking.firewall.extraInputRules = ''
+      ip6 saddr ${mesh.subnet} tcp dport 5201 accept comment "iperf3 over the mesh"
+      ip6 saddr ${mesh.subnet} udp dport 5201 accept comment "iperf3 over the mesh"
+    '';
+  };
+}

modules/router/ipv6.nix

@@ -0,0 +1,31 @@
+# IPv6 on the PPPoE uplink: run networkd's DHCPv6 client on ppp0 to obtain a
+# delegated prefix; each vlan-* interface (default.nix) carves a /64 out of it
+# via DHCPPrefixDelegation and announces it to clients with SLAAC.
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.cnx.router;
+in
+{
+  config = lib.mkIf cfg.enable {
+    systemd.network.networks."45-ppp0" = {
+      matchConfig.Name = "ppp0";
+      networkConfig = {
+        DHCP = "ipv6";
+        # pppd owns the v4 address/route on this link; don't let networkd
+        # tear them down.
+        KeepConfiguration = "static";
+        # Default v6 route comes from the ISP's RA when they send one.
+        IPv6AcceptRA = true;
+      };
+      # Many PPPoE ISPs never send an RA with the M flag; solicit regardless.
+      dhcpV6Config.WithoutRA = "solicit";
+      linkConfig.RequiredForOnline = "no";
+    };
+
+    boot.kernel.sysctl."net.ipv6.conf.all.forwarding" = lib.mkDefault 1;
+  };
+}

modules/router/omada.nix

@@ -0,0 +1,43 @@
+# TP-Link Omada SDN controller for sites with Omada APs/switches. There is no
+# nixpkgs package, so it runs as a podman container (mbentley/omada-controller,
+# the de-facto standard image). Host networking because device adoption relies
+# on L2 broadcast discovery (UDP 29810) on the mgmt VLAN; the default-deny
+# input firewall keeps its ports unreachable from WAN and non-mgmt VLANs.
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.cnx.router;
+  mesh = import ../mesh-hosts.nix { inherit config lib; };
+in
+{
+  options.cnx.router.omada.enable =
+    lib.mkEnableOption "TP-Link Omada SDN controller (podman container)";
+
+  config = lib.mkIf (cfg.enable && cfg.omada.enable) {
+    virtualisation.podman.enable = true;
+    virtualisation.oci-containers = {
+      backend = "podman";
+      containers.omada = {
+        image = "docker.io/mbentley/omada-controller:5.15";
+        extraOptions = [ "--network=host" ];
+        environment.TZ = config.time.timeZone;
+        volumes = [
+          "/var/lib/omada/data:/opt/tplink/EAPController/data"
+          "/var/lib/omada/logs:/opt/tplink/EAPController/logs"
+        ];
+      };
+    };
+
+    # Admin UI (8043) also reachable over the mesh, like Grafana on control.
+    networking.firewall.extraInputRules = ''
+      ip6 saddr ${mesh.subnet} tcp dport 8043 accept comment "omada ui over the mesh"
+    '';
+
+    # Controller state (adopted devices, site config, cert) — declared as clan
+    # state so a borgbackup client can pick it up; backup wiring is a later step.
+    clan.core.state.omada.folders = [ "/var/lib/omada" ];
+  };
+}

modules/router/pppoe.nix

@@ -0,0 +1,64 @@
+# PPPoE WAN session. ISP credentials are entered once at `clan vars generate`
+# (prompts). Both are secret — AIS often uses the same string for username and
+# password — so neither may land in the Nix store: pppd reads the username from
+# an included secret options file and the password from chap/pap-secrets.
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.cnx.router;
+  creds = config.clan.core.vars.generators.pppoe-credentials;
+in
+{
+  config = lib.mkIf cfg.enable {
+    clan.core.vars.generators.pppoe-credentials = {
+      prompts.username = {
+        description = "PPPoE username (from the ISP)";
+        type = "hidden";
+      };
+      prompts.password = {
+        description = "PPPoE password (from the ISP)";
+        type = "hidden";
+      };
+      files."user-opts".secret = true;
+      files."chap-secrets".secret = true;
+      script = ''
+        user="$(cat "$prompts"/username)"
+        pass="$(cat "$prompts"/password)"
+        printf 'user "%s"\n' "$user" > "$out"/user-opts
+        printf '"%s" * "%s"\n' "$user" "$pass" > "$out"/chap-secrets
+      '';
+    };
+
+    services.pppd = {
+      enable = true;
+      peers.wan = {
+        autostart = true;
+        config = ''
+          plugin pppoe.so ${cfg.wan.pppInterface}
+          ifname ppp0
+          file ${creds.files."user-opts".path}
+          noipdefault
+          defaultroute
+          noauth
+          hide-password
+          persist
+          maxfail 0
+          holdoff 5
+          lcp-echo-interval 15
+          lcp-echo-failure 3
+          +ipv6
+          mtu 1492
+          mru 1492
+        '';
+      };
+    };
+
+    # pppd looks up the password for `user` in these files at dial time; both
+    # point at the same generated `"<user>" * "<pass>"` line (PAP and CHAP).
+    environment.etc."ppp/chap-secrets".source = creds.files."chap-secrets".path;
+    environment.etc."ppp/pap-secrets".source = creds.files."chap-secrets".path;
+  };
+}

modules/router/proxy.nix

@@ -0,0 +1,141 @@
+# Internal reverse proxy for the gateway: Caddy terminates TLS for
+# <service>.<site><siteId>.cnx.network (e.g. omada.cnx1.cnx.network) and
+# forwards to backends by their internal address. The cert is a real Let's
+# Encrypt wildcard (*.<site><siteId>.cnx.network) obtained via ACME DNS-01
+# against ns1 with a gateway-scoped TSIG key, so browsers trust it without
+# any CA install; the names only *resolve* internally — Blocky answers them
+# with the router's LAN address, the public zone never carries them.
+#
+# Requires the machine to also import the shared secret generator:
+#   (import ../../modules/dns/acme-gw-secret.nix "<hostname>")
+# and ns1 to hold the matching key + acl (see machines/ns1/configuration.nix).
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.cnx.router;
+  hosts = import ../hosts.nix;
+  hostname = config.networking.hostName;
+  tsigKey = "acme_${lib.replaceStrings [ "-" ] [ "_" ] hostname}";
+  certName = "${cfg.site}${toString cfg.siteId}.cnx.network";
+
+  serviceModule = {
+    options = {
+      backend = lib.mkOption {
+        type = lib.types.str;
+        example = "https://127.0.0.1:8043";
+        description = "URL Caddy forwards to (internal/mesh address).";
+      };
+      insecureSkipVerify = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = "Skip TLS verification towards the backend (self-signed upstreams like Omada).";
+      };
+    };
+  };
+in
+{
+  options.cnx.router.proxy = {
+    enable = lib.mkEnableOption "internal reverse proxy (Caddy, wildcard cert via DNS-01)";
+
+    services = lib.mkOption {
+      type = lib.types.attrsOf (lib.types.submodule serviceModule);
+      default = { };
+      description = "Proxied services; attr name becomes <name>.${certName}.";
+    };
+
+    allowVlans = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [
+        "mgmt"
+        "lan"
+      ];
+      description = "VLANs whose clients may reach the proxy (443, plus 80 for the redirect).";
+    };
+  };
+
+  config = lib.mkIf (cfg.enable && cfg.proxy.enable) {
+    assertions = [
+      {
+        assertion = lib.all (v: cfg.vlans ? ${v}) cfg.proxy.allowVlans;
+        message = "cnx.router.proxy.allowVlans must name VLANs defined in cnx.router.vlans.";
+      }
+    ];
+
+    # Render the shared per-gateway TSIG secret into a lego rfc2136 env file;
+    # same pattern as web01 (modules/web-proxy.nix), scoped on ns1 to
+    # _acme-challenge.<site><siteId> TXT records only.
+    clan.core.vars.generators."dns-acme-${hostname}-rfc2136" = {
+      files."rfc2136.env".secret = true; # root-owned; systemd reads it as root
+      dependencies = [ "dns-acme-${hostname}-secret" ];
+      script = ''
+        printf 'RFC2136_NAMESERVER=${hosts.ns1.ipv4}:53\nRFC2136_TSIG_ALGORITHM=hmac-sha256.\nRFC2136_TSIG_KEY=${tsigKey}\nRFC2136_TSIG_SECRET=%s\n' \
+          "$(cat "$in"/dns-acme-${hostname}-secret/secret)" > "$out"/rfc2136.env
+      '';
+    };
+
+    security.acme = {
+      acceptTerms = true;
+      defaults.email = "postmaster@cnx.email";
+      # One wildcard for every proxied service; DNS-01 against ns1, so issuance
+      # works behind PPPoE with no inbound reachability at all.
+      certs.${certName} = {
+        domain = "*.${certName}";
+        dnsProvider = "rfc2136";
+        environmentFile =
+          config.clan.core.vars.generators."dns-acme-${hostname}-rfc2136".files."rfc2136.env".path;
+        # ns1 is the only nameserver that accepts this key's UPDATE; check
+        # propagation against it directly rather than a public resolver.
+        dnsResolver = "${hosts.ns1.ipv4}:53";
+        # Caddy reads the cert from explicit file paths (tls directive below),
+        # so it won't notice a renewal on its own.
+        reloadServices = [ "caddy.service" ];
+      };
+    };
+
+    # The lego-issued cert is owned group=acme; Caddy needs to read the key.
+    users.users.caddy.extraGroups = [ "acme" ];
+
+    # The explicit `tls cert key` points Caddy at the wildcard cert and disables
+    # its automatic ACME, so no extra issuance happens.
+    services.caddy = {
+      enable = true;
+      virtualHosts = lib.mapAttrs' (
+        name: svc:
+        lib.nameValuePair "${name}.${certName}" {
+          extraConfig = ''
+            tls /var/lib/acme/${certName}/cert.pem /var/lib/acme/${certName}/key.pem
+            ${
+              if svc.insecureSkipVerify then
+                ''
+                  reverse_proxy ${svc.backend} {
+                    transport http {
+                      tls_insecure_skip_verify
+                    }
+                  }''
+              else
+                "reverse_proxy ${svc.backend}"
+            }
+          '';
+        }
+      ) cfg.proxy.services;
+    };
+
+    # Blocky answers <anything>.<site><siteId>.cnx.network (customDNS covers
+    # subdomains) with the router's LAN address — clients on any allowed VLAN
+    # reach that address through the router's input path.
+    services.blocky.settings.customDNS.mapping.${certName} = cfg.vlans.lan.address;
+
+    # 443 serves the proxy; 80 only carries Caddy's automatic HTTP->HTTPS
+    # redirect. mgmt is already a trusted interface; listed anyway so shrinking
+    # trustedInterfaces later doesn't silently break the proxy.
+    networking.firewall.interfaces = lib.genAttrs (map (v: "vlan-${v}") cfg.proxy.allowVlans) (_: {
+      allowedTCPPorts = [
+        80
+        443
+      ];
+    });
+  };
+}

modules/router/speedtest.nix

@@ -0,0 +1,67 @@
+# Periodic WAN speed test so ISP degradation shows up as a trend instead of a
+# complaint. A timer runs librespeed-cli and writes the results as Prometheus
+# metrics into node_exporter's textfile collector — they ride the existing
+# 9100 scrape to VictoriaMetrics, where alerts.nix compares each run against
+# the link's own 7-day median (no per-site threshold to maintain).
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.cnx.router;
+  textfileDir = "/var/lib/speedtest";
+in
+{
+  options.cnx.router.speedtest.interval = lib.mkOption {
+    type = lib.types.str;
+    default = "hourly";
+    description = "systemd OnCalendar spec for the WAN speed test.";
+  };
+
+  config = lib.mkIf cfg.enable {
+    services.prometheus.exporters.node.extraFlags = [
+      "--collector.textfile.directory=${textfileDir}"
+    ];
+
+    systemd.services.speedtest = {
+      description = "WAN speed test to Prometheus textfile metrics";
+      after = [ "network-online.target" ];
+      wants = [ "network-online.target" ];
+      path = [
+        pkgs.librespeed-cli
+        pkgs.jq
+      ];
+      serviceConfig = {
+        Type = "oneshot";
+        StateDirectory = "speedtest";
+        # One test at boot would race PPPoE and log a spurious failure.
+        ExecCondition = "${pkgs.iproute2}/bin/ip link show ppp0";
+      };
+      script = ''
+        tmp="${textfileDir}/.speedtest.prom.tmp"
+        if result=$(librespeed-cli --json); then
+          jq -r '.[0]
+            | "speedtest_download_mbps \(.download)",
+              "speedtest_upload_mbps \(.upload)",
+              "speedtest_ping_ms \(.ping)",
+              "speedtest_jitter_ms \(.jitter)",
+              "speedtest_success 1"' <<<"$result" > "$tmp"
+        else
+          echo "speedtest_success 0" > "$tmp"
+        fi
+        mv "$tmp" "${textfileDir}/speedtest.prom"
+      '';
+    };
+
+    systemd.timers.speedtest = {
+      wantedBy = [ "timers.target" ];
+      timerConfig = {
+        OnCalendar = cfg.speedtest.interval;
+        RandomizedDelaySec = "10m";
+        Persistent = true;
+      };
+    };
+  };
+}

modules/web-proxy.nix

@@ -0,0 +1,73 @@
+# Public reverse proxy with TLS termination for web01. Caddy fronts internal
+# services and forwards to them over the ZeroTier mesh, never the public net.
+# The cert is a single wildcard (*.cnx.network) obtained via ACME DNS-01, so
+# adding a vhost needs no new issuance. Public ports: 443 for the proxy and 80
+# only for Caddy's HTTP->HTTPS redirect (issuance never uses inbound HTTP).
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  mesh = import ./mesh-hosts.nix { inherit config lib; };
+  hosts = import ./hosts.nix;
+  certName = "cnx.network";
+in
+{
+  imports = [ ./dns/acme-web01-secret.nix ];
+
+  # Render the shared acme_web01 TSIG secret into a lego rfc2136 env file. lego
+  # (via security.acme below) uses it to write _acme-challenge.cnx.network TXT
+  # records on ns1, which authorizes the acme_web01 key for exactly that owner.
+  clan.core.vars.generators.dns-acme-web01-rfc2136 = {
+    files."rfc2136.env".secret = true; # root-owned; systemd reads it as root
+    dependencies = [ "dns-acme-web01-secret" ];
+    script = ''
+      printf 'RFC2136_NAMESERVER=${hosts.ns1.ipv4}:53\nRFC2136_TSIG_ALGORITHM=hmac-sha256.\nRFC2136_TSIG_KEY=acme_web01\nRFC2136_TSIG_SECRET=%s\n' \
+        "$(cat "$in"/dns-acme-web01-secret/secret)" > "$out"/rfc2136.env
+    '';
+  };
+
+  security.acme = {
+    acceptTerms = true;
+    defaults.email = "postmaster@cnx.email";
+    # One wildcard cert for every vhost this proxy serves, via DNS-01 (so issuance
+    # never depends on inbound HTTP). Port 80 is open only for Caddy's
+    # HTTP->HTTPS redirect, not for ACME.
+    certs.${certName} = {
+      domain = "*.cnx.network";
+      extraDomainNames = [ "cnx.network" ];
+      dnsProvider = "rfc2136";
+      environmentFile = config.clan.core.vars.generators.dns-acme-web01-rfc2136.files."rfc2136.env".path;
+      # ns1 is the only nameserver that accepts the acme_web01 UPDATE; check
+      # propagation against it directly rather than a public resolver.
+      dnsResolver = "${hosts.ns1.ipv4}:53";
+      # Caddy reads the cert from explicit file paths (tls directive below), so it
+      # won't notice a renewal on its own — reload it whenever the cert changes.
+      reloadServices = [ "caddy.service" ];
+    };
+  };
+
+  # The lego-issued cert is owned group=acme; Caddy needs to read the key.
+  users.users.caddy.extraGroups = [ "acme" ];
+
+  # Reverse proxy. The explicit `tls cert key` points Caddy at the wildcard cert
+  # and disables its automatic ACME, so no extra issuance happens. Backends are
+  # dialed over the mesh by their ZeroTier address (mesh.hosts.<name>).
+  services.caddy = {
+    enable = true;
+    virtualHosts."grafana.cnx.network".extraConfig = ''
+      tls /var/lib/acme/${certName}/cert.pem /var/lib/acme/${certName}/key.pem
+      reverse_proxy http://[${mesh.hosts.control}]:3000
+    '';
+  };
+
+  # 443 serves the proxy; 80 only carries Caddy's automatic HTTP->HTTPS redirect
+  # (the Hetzner cloud firewall also scopes these in
+  # modules/hetzner-firewall-rules.nix). Admin still rides the mesh.
+  networking.firewall.allowedTCPPorts = [
+    80
+    443
+  ];
+}

sops/machines/gw-cnx-1/key.json

@@ -0,0 +1,6 @@
+[
+  {
+    "publickey": "age1g8a6fqhqg8q8qu4jyuhxc7hf00t5qvra55chyfm0qk45wlmvc9qs2cft8t",
+    "type": "age"
+  }
+]

sops/machines/mx1/key.json

@@ -0,0 +1,6 @@
+[
+  {
+    "publickey": "age1l5hw95p5h4sthrgn0usms9yfkwwmcvv34tjgrtv9s4e6x39chacshgxavs",
+    "type": "age"
+  }
+]

sops/machines/web01/key.json

@@ -0,0 +1,6 @@
+[
+  {
+    "publickey": "age1yey6gxgsyl4tj6ek0tve2pckt6qersqspk66ukkzum8mrr6zppqsj4jn3m",
+    "type": "age"
+  }
+]

sops/secrets/control-age.key/groups/admins

@@ -0,0 +1 @@
+../../../groups/admins

sops/secrets/control-age.key/secret

@@ -3,8 +3,12 @@
 	"sops": {
 		"age": [
 			{
-				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBcmJ4cXNt\nL3k4UmtqQUJZOGRoZFhab0pRSTdBUm9ZZkRxdHNvaCtRc05JVgphbHhHQThiRDNF\neW8xVGhXQ1pvSkRDR2hod2Zpa3pwaWZDVUpGZlpPWkNZCi0tLSB2T3JCcElnbVU2\nekJ5ZTdyQzZwbmR4aHBJUTc3M3loUm0wbHpHTGM0ZW1RCs0kHes9rkjd3uwHzxhi\n8x8AcQcQTR+CGsV5XPzGVMKKUZpwXWeNnvcHfVwYcqOnXucbUdCxLc6d57c9GQpC\nd3Y=\n-----END AGE ENCRYPTED FILE-----\n"
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBNRWlKNFVSOThSWlI3LzAz\ndnQxbnNGSlhoVmZrQmpIVEJieStlUzlseDJJCmdGbldCSU1oTGszZmtzSWhJaWta\nVDhoQU8vVXB0bG1VUkxoWDlyR0I1a0EKLS0tIDFzdklCTTBjeXBoT3FqVW1Pbkxl\nYVZ5Vm55OFp4YWwzS3dVNXRoZmpBT1EK0pNbk20gwPe5Q1bZR3XRHMWz+x+8FzRa\niq5tLcGkgfMO6WTzj8vtB7g0Q3sFgmPXTqb8yMzqAkZd4z9FKJGt6A==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBcFBHaFpS\nbjJiL0ZnOXlzMVVNdjhvUGxmaEJhTklSR2ZaTnhPWXFCaC8vbgpLdE04aW16dEls\neGw3VllEYVAzbGJ5TVZoOXhkR3dCUkREdUkxMU13clVzCi0tLSA3eW93QnFsNW1r\nK0ZiWEtrWFVqeTJyTk1maGVTMzdBTkJUdk5WN2N3WlprCp5oak+Bh/g4+ikE1LR5\n7kYrWQlzjC0Z+F5GegEtRjGCPnolWllnPUH4dGGjfuyAxy9qfi1J6Bi6m8uzo5rY\n+zs=\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa"
 			}
 		],
 		"lastmodified": "2026-06-14T05:20:21Z",

sops/secrets/gw-cnx-1-age.key/groups/admins

@@ -0,0 +1 @@
+../../../groups/admins

sops/secrets/gw-cnx-1-age.key/secret

@@ -0,0 +1,19 @@
+{
+	"data": "ENC[AES256_GCM,data:bsb98446kjvnDAVWNIqvH64uD6XyfEpohPFZs38jDaxxTp1wqJmFdeDW+p3SGld8Q+4h6rq7rFtTUJnQyslvgsPfYN5tBhJuTbI=,iv:pPUf3RgHqS8LlRIBdZ7jmzxp5VmlP68nrX8TCVP8aGU=,tag:SseZa3Sf5LuMXWN8CNSbxw==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBLRkppdUoza2l0SnFpME5n\nM0xBRVk4c2lhaHMvazFNaGNTdkJDcnJEQkVrCjhFUlZMUitnZ20zQlFEcm5WQmVG\nWmJVeUh6S09MS0xpQUR4aDF1V1dCZFkKLS0tICtOMEJ6QmRRR3BkaU5sZHdzOElO\nSnV4K0dHMHhDOURPTEpucXFuNEUwaU0Kx4xAaKTrw1bKJPQVCHcByTcapFhslwrV\nqssEBnYWXqFgI3vh3CatBXuZY1Lm3eTKhDDQ2ZyPKgc/l0gXyFwH2g==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBMHFKQTNG\nazdEcThReFZGVVBtVFBDY1luQk1UTkl0a0p1bzQ0cUxBZWVyYwo4aU5qaEs0czJB\nUzZFcG9uTUZ1OUFScW9ZeHljbDBEbjkzZ08vS2F1Qjd3Ci0tLSBHTXhVWjNsWHhK\nbWtrMWw3Nm8wSU9MNXZwcWxVZXRJS1dTK0hycnFoVlRJChemDXVb7LgM7OeUUeqa\nTl0IuDHmIIx/lUYnJWGZmWQHz2znk679j36VQQNWcK6eZlKUIqYmBj3fMu+OYir2\nMLI=\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa"
+			}
+		],
+		"lastmodified": "2026-07-28T09:03:13Z",
+		"mac": "ENC[AES256_GCM,data:MKIyykYwtQqpUnXqZQKzKwoRhSon2PH0wYiTDTvogaVREHfpEPs/mdOjQBUHtbr1QGqWNh4AxdBTKfpjXTlncHAEtcIdVH3qPccVxsXqfqRmZPDFzcHZAIw+VSvDsKwD3anixiXDeHmeLawwbyb1rnS0P8KquXbmjvirwGUIQLU=,iv:+hdLXMJMCby/6Q6JGxp4p95GgPKbdA/OTAZloyylr08=,tag:ZZ4dKjhjpE2xqcoNf4qeoA==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.13.1"
+	}
+}

sops/secrets/gw-cnx-1-age.key/users/berwn

@@ -0,0 +1 @@
+../../../users/berwn

sops/secrets/mx1-age.key/groups/admins

@@ -0,0 +1 @@
+../../../groups/admins

sops/secrets/mx1-age.key/secret

@@ -0,0 +1,18 @@
+{
+	"data": "ENC[AES256_GCM,data:lAgbSvyxBl0/NG8rHweKEEqYyDsHa+SrPZnxVubB2x5H3cqydhTCs8NMxJp+RKTPW2nhfmB/lrKgLHFfFMJfMg5jZ2BvZtR1IzA=,iv:1PgMbra/ec7yiCm7K5yo+1lCLJ89ryfP3SQML7uYv4k=,tag:lmaG+dZPCLHtQ59uFeSkVg==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBELzcrdUc0cW9pSXhGbzRY\naTd5aWgzcFpLQ3MyVElaZU5JK2Z4ZGRtMXpJCk14bmZLRlY4Z3pDeUxPM203anVY\nbE1aUEdDZ3NkcGxrcCtWS1ZmRWIrcmMKLS0tIDVTOUJhenR0WVhZSWNWT2VtRFpv\nc1h6Nk95Z3Y4eFQ1NUdmWWJiaWdDdGMKB3whh/RgAePTJnGmeDJ/WFv4NI42vA5O\nB0F6jmSDNa5beP8Um2DjWdPENkJJjv9yv38b7hP8BLDe9Ba4WBNfBA==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBby9pdmFZ\nOElJVCtUQmJWVVljQi9KcHE2a2FaeU5HeUhyUm05Zm0wZDZKVwptY2hHRFJKMi9I\nN3NDY3ZjYVloTjF6R0l4RllDS1dpNzA1TDJDc0hGQnFvCi0tLSB1akhUWk9RU0Rt\naDV5MHdHcExlSzZWbSs0S1A3NGhES0V1U0pEaUd5WE04ChtGuEq0HnRiVTDwhJnO\nIWMhwCYaewHk+k0a0Z9qCwqqKxhfiGS6kg/YTKHTNhQ0bxIA8yqgQoaE8Hl5Zhzs\nKqk=\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2026-06-18T04:33:03Z",
+		"mac": "ENC[AES256_GCM,data:Iffy/eS80qOJLdaGOIxti1otdxSLHA1TV06R8xq4zI1qFklJX8OWRy9CAYwAcTURS88gd5c2VGHsBI8yHOBj7LKiLJiu5/xRgD106hotBOJUy90UZ6MMbM2wwHomhQg5r5kNpRsJaPFaLFzm5YJN9gLUU1kxO4Nnt2L3bxZTogQ=,iv:cqbFD3ZIJnaZU7j7l9Qt1QqiUSQtclV7MGZWEZRZQoY=,tag:F0ZZQSQKYtBI65uutjNxow==,type:str]",
+		"version": "3.12.1"
+	}
+}

sops/secrets/mx1-age.key/users/berwn

@@ -0,0 +1 @@
+../../../users/berwn

sops/secrets/ns1-age.key/groups/admins

@@ -0,0 +1 @@
+../../../groups/admins

sops/secrets/ns1-age.key/secret

@@ -3,8 +3,12 @@
 	"sops": {
 		"age": [
 			{
-				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBemRVWUs1\naXFpVm80UmloUUpnR1BVaGsxampHV1JFNWFnSUhiY0orOTU0cgppVEJ5QVcvbGpC\nZlErd01XUzh0MnZOekkvTkc5Z3owS3pBbkNCUDJ6SlJRCi0tLSBRVVgrd3hIVEla\nMEhoN3FDSG1HdGk5YU8ydzlVUGd1RW5zM1lka3VuSHFNClrBItoFo7uqpS5fuvaG\nOvh6mTylk4bHgcSRGKTk383srgsvPdZaioJfUs7fbVmLD3+bhfM/Wgv7tI6kO1dj\nmDg=\n-----END AGE ENCRYPTED FILE-----\n"
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBcU01TXl0\nRmU5ZlZOK3NZZE8vNktOWXBkRnZtZHlUbnBHYlVQbDdzZlRDbApFeFI0OE1KdndQ\nZWczZGRYZXBONTc2aG4vZTJEZ1hCdGljNFJBcmF0UGpRCi0tLSBGMGJVWUtJMWFP\nQUZwWVZTWUFKZnVJUTJSMHlRMGRNcExPSEpXTFpLSkVNCj1WEuRyATcxp0GYeoqC\nACpQpcqgIBVD1dYB0pPo+I6yDXvx+JbwVdgGl+2Cxj40HQOfyvJecoOO3efOZrB3\nUZU=\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBRb0ZvUmN2RWhtQkpXMXJr\nY1o2MXZsQ0dXNFYxa2RLUGlJWnN3Z25zbnp3CmtLbkYvdHR2MUZIY2U1MzhyU1VD\ndm5MTmMveEJLOHZWWnVWdVpDcTFsTmMKLS0tIFEwYzZYeU5uc29BTEVxN2YvN0Zw\nd0VSeTNhQk1pNVdaZVdTVk1oNjFwQWcKwEbHyT354Cbam9wuiIhapre+nbqtMr9h\nbOjF53LR9RZ9/mbpTNDLOkq2QW2aM4tKkRfdUgP1XgboilzcTjba8A==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06"
 			}
 		],
 		"lastmodified": "2026-06-14T05:20:24Z",

sops/secrets/ns2-age.key/groups/admins

@@ -0,0 +1 @@
+../../../groups/admins

sops/secrets/ns2-age.key/secret

@@ -3,8 +3,12 @@
 	"sops": {
 		"age": [
 			{
-				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBL2doZFpO\na2Y2QzZSckJaZmZsTlFtQU1mckxJWTBiUXA2SUljVWZDemFJYgpwcFl6RHh2ZXdn\nNTBsaEVpWm5oYWtteU5WZmxWK1RRK2NoYW0yWlpCd3dBCi0tLSBsZVNpczR4dUdv\nSmtqQzMvc1ZYY0xUY0JMQmt2YzB5YmZOMEtXcW9vWUcwCnCdgcgiwF2rkWsV0IdO\n/6cs35FBypWpfflGwOwP0GnTrNizc9HphcMcwAi5NMQVi6X90Xgm5aFdpAsSWepX\n/SI=\n-----END AGE ENCRYPTED FILE-----\n"
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBaTVGQWRC\nZUJocGx5Zjd0c1JpUEdHbGczNUh1RjRqSERRNHNSbnRndWFUMAoxbFBEU0Ezajhr\nSUlIRStWUUdVTDJMQWxmSkw0c0FNYnFxNEJIdnExTmx3Ci0tLSBMRGRPLzlXTEN3\ncHZCMGF3WDZTcFhTd2srL3Vpc2dRdExrVzFJcWhjQVJrCrA1y2SapkZoppI3z4LD\nH71D/WxIcTHp3fIzICNvgxWHn3+vkDDZcpcgH+gRiPCDqrgDp0NZcZiDlz6zyHGr\nmaM=\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB0Rmp1S29hVE5LNkZpLzRV\nMGZJL242YmdHNkFXR2d0TmlVWXViTlJFNGpnClBSdEhaRlNWbVNrb05tTHh0Q3Rk\naVFNdUd1clNVckREajVoRmZCenFEaVkKLS0tIExOaHd1WWxZRFM1UFk4K0E0cWpE\nK2duUE1KeEtmeVorRnc3MFdDZXFGZjAKKw4ykWu+HzBN3WAj/gx0mMYk0RiyRR50\nYv7hg1B/+x4lDCTjAJIWrrBEjLJBZTZRnECmTMTlJ39uOhxeUOizCA==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06"
 			}
 		],
 		"lastmodified": "2026-06-14T05:20:27Z",

sops/secrets/web01-age.key/groups/admins

@@ -0,0 +1 @@
+../../../groups/admins

sops/secrets/web01-age.key/secret

@@ -0,0 +1,18 @@
+{
+	"data": "ENC[AES256_GCM,data:u+O5nWbFlvp6SGyHJggfkCLCT0ZuDy89e/VGGQNdt3yYzgdNmnrtd+2q+Ft3MtoOSSCLvStriGQzfLhcqEgqGgt3PqfIzCO1IG4=,iv:GyrZ1XUiOZe1I1Z/HebTy2NM2tfDHxIH5zGVk7HD+xQ=,tag:js6fbXWajVZSxt0hmnnA5g==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBWenArV1pZMURzNElTeGlC\nRSttUFRwM2E5d2htVDlFcTkwVDIrV2JxQjE0CmE0NUZ4UERHdHpqYk9uK01GOTFQ\ndW5UNlRrY0hvU2pNQmpSanh5RG5YbVkKLS0tIFdYQmkva1NORG80U3dDSWszZTlO\nOXViWUxMTVR1NE00cjdpVXpVS1J5YU0KHBkeKAJZDc+R1GLKwDYLyQBlEW7tPnMh\nf3tsUvtD0flqPAXNeDgyOmKufP7U6oDy/OriFC9+zYQbWyEEc6CZHg==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBL05xT1BJ\nSldCSkFhaFErWDNMeXMvRksvekkwNUZLZFVtSmUydVRrNG9JTwpFaHNsVThZN2gw\nMlNxUXhnN2xYNFluK1hadGxzMFZuaWR3cVFRYW9mWndVCi0tLSBUUmRMSnJCeTRM\nUVdKY3hzWVRkQkFuQ3FaRDVZSEp1b2N1ajg5RnlhT3VRCjX/vWj0We88ATiz808w\nz60RL0BvDGJ6m1BNmqAdtfCCClH33YXQBGrKT2E5elvOTl0iOCrT7HPjzXxJZXuw\nkKg=\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2026-06-20T19:26:29Z",
+		"mac": "ENC[AES256_GCM,data:uD+L3op6ZPtvmLreJ7S4GE6bQuItV87w1LTMFvjI1Kb5+Z0sXlL0TpYO8WLx8X0yaL3HddwlmBKYQGp/OlRPqZMFDbQuK25oeKB54jfm6YDn66rrMQJl1FOw68fLJaHYNjNelEg/bj2WG7YpfZBoWO67MW6F+44Rg4XF85w/1x0=,iv:X9TmEL5JkcC9waLumWpgpBwp3YWLMslZi++dv2HZ0mk=,tag:i7c1CxT1Xv/+T/jY9E0cdg==,type:str]",
+		"version": "3.12.1"
+	}
+}

sops/secrets/web01-age.key/users/berwn

@@ -0,0 +1 @@
+../../../users/berwn

vars/per-machine/control/grafana-secret-key/secret-key/secret

@@ -0,0 +1,22 @@
+{
+	"data": "ENC[AES256_GCM,data:78fZXLuEBqiLerV0HZ5cEKCeoWayeCSJcfhW7fFck0t08I44UqPW+yzXwkM=,iv:P6OX1cwnY0dmu7IRChwfcsWPsEBHmlBHZfnJTFf8FHw=,tag:sBAE2h0rNxFqhtqQIt/nmQ==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBWQ3lYVmhQRkt3TU5mZ1Fr\nR2Z3MXAwU2J2Zi9VSzJ4YlJhN1BZRVVPSVU4ClVSWlNxdjU0NkNXOFNnUG5ZWno3\nUFQxZzE2bVFMNHFiWWZJdDVYU0o3cG8KLS0tIElQbzhoZHh4Tk5MUjNEdVdrZmxF\nckIwYktjb2hZQStDcXU0Z0NTbkdIYlEKQkFi/lgBoC6zyzu08hhEOUvJse0E6YV2\nLwlLcWl+303HWeWob2R4VZhuIkkqk5pE2Do83AFsU8vick8dUC1sQg==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1env5y2eey0p3dggs8tydwmtyhtwqylpg7spuququ4vvax6arzp2qjnm4tx"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKYkl2MmRPbW90Q3doZ3JY\nb2F0TG10RjEyNXdiSGZGNHlvbVpjQUdXVjB3CjVCZ1UwRkRzK2NKMnVxQ0N0MmJp\ncElvdm9ucGNpTEZnSHFld0Fhb1lLU0UKLS0tIGpZWmhXcVJSS1NHbFYwc01JaDN6\nVm9yOFhZYzU2ZjQ0KzNtV201L3ZnUTQKhGGDHPHcizFBuJx1ZhObb+yqDQWROG1k\nLNqoSvLtd2glN7kC6ro8cvAGW/Oj4Baawetmtz4ctew4b0/8xG3PRQ==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBMnJyMEdV\nUndYaWtiMEtIakxXWUxPYTdvSTcxK0NaOThUcmN1d0hBaWpiUQowbTE3Q080UTVQ\nYnRjRVFwQkZPYUE2c0JFZklnL1RILzVET01JbEg1ZVF3Ci0tLSBsNkVRRXR6WDM0\nR3ZQcmE2VDdETURYYThCRzlabWErc1g3UDFFdkxFeW1rCgbVXMv1eD1Bu8QV/2gk\nNgVeQSvH1qEPQuUIeHqps/x3NyNpii/IymvkNvZJerowuD5TeSIeE2tqCA+0d5ph\nUpw=\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa"
+			}
+		],
+		"lastmodified": "2026-06-25T00:27:59Z",
+		"mac": "ENC[AES256_GCM,data:BDQfNDgwJGvExG9jTOs/BULCSpBvMjnpYtlPUP8m9/9kL+2cKBgHukQqqM4Eaju5dm38hGW9345PQvLeqoD53thtUyPR+GRlfAtKwGSqLfur1KlMEM/b50phtiJ2GjzS4bKu+weTYdmfux9Quh+HI+kVuHHLyRs0A13qSsSikvs=,iv:MmXMr2m2or1dlMTa4LvqRjPGndOedcFZdq/fq8fnzBU=,tag:6bBVwKGqRnWYmFY6q6nSnQ==,type:str]",
+		"version": "3.13.1"
+	}
+}

vars/per-machine/control/openssh-cert/.validation-hash

@@ -0,0 +1 @@
+9a9e2f23526941b204dacfe8fc77dad5f4b1c7aff945cdbf301ec61129959228

vars/per-machine/control/openssh-cert/ssh.id_ed25519-cert.pub/value

@@ -0,0 +1 @@
+ssh-ed25519-cert-v01@openssh.com AAAAIHNzaC1lZDI1NTE5LWNlcnQtdjAxQG9wZW5zc2guY29tAAAAIHiQTWR/pRz9vlrtQSe+u2g8XPDZvFpRBCpF7bQ6mBPaAAAAIH+Ix/XbsbqHU87Kg7Y9frnhYgfUEMNKOmQ3DSlb9LCmAAAAAAAAAAAAAAACAAAAB2NvbnRyb2wAAAAgAAAAHGNvbnRyb2wuY254LW5ldHdvcmsuaW50ZXJuYWwAAAAAAAAAAP//////////AAAAAAAAAAAAAAAAAAAAMwAAAAtzc2gtZWQyNTUxOQAAACCboWwGkseX7mqk2GVtM2o1WPRzh8U3X6A3woanYG9xoQAAAFMAAAALc3NoLWVkMjU1MTkAAABAbWgsXjGd/QGdbYHNfHs4LiMgF1weHHirWk4ZLMuTLRDXObwdJ4tbc9IGDfNBZnAfOSxBBy+U/sjGbKQtRWt4Aw== /tmp/vars-irrhty3h/in/openssh/ssh.id_ed25519.pub