machines/gw-cnx-1: wan.macAddress

New cnx.router.accessPorts option pins a port untagged to one VLAN via
bridge PVID/EgressUntagged; convention is the last copper port as an
always-available mgmt recovery port. gw-cnx-1 port roles: enp1s0 WAN,
enp2s0 trunk, enp3s0 temporary DHCP uplink into the old OPNsense LAN
(back to trunk at cutover), enp4s0 untagged mgmt.

.gitignore

@@ -1,2 +1,2 @@
-result
+result*
 .direnv/

clan.nix

@@ -1,46 +1,61 @@
 let
-  # This clan-core pins the zerotier `allowedIps` interface (admit by network
+  hosts = import ./modules/hosts.nix;
-  # IPv6), but node IDs are the stable per-device handle (what `zerotier-cli
+
-  # info` prints). Derive a member's IP on THIS network from the controller's
+  # Single source of the machine list: inventory AND the per-machine baseline
-  # network id so external members can be listed by node id, as below.
+  # below are both derived from it, so no machine can be added without the
-  ztNetworkId = builtins.readFile ./vars/per-machine/control/zerotier/zerotier-network-id/value;
+  # baseline (e.g. admins group encryption for all its generated secrets).
-  ztMemberIp =
+  fleet = {
-    nodeId:
+    control = { };
-    let
+    ns1 = { };
-      full = "fd" + ztNetworkId + "9993" + nodeId;
+    ns2 = { };
-      h = i: builtins.substring (i * 4) 4 full;
+    mx1 = { };
-    in
+    web01 = { };
-    "${h 0}:${h 1}:${h 2}:${h 3}:${h 4}:${h 5}:${h 6}:${h 7}";
+    # Site gateways (Topton 1U routers): dynamic PPPoE WAN, so they are NOT in
+    # modules/hosts.nix / the `internet` instance — clan reaches them over the
+    # zerotier mesh (or Tor) instead.
+    gw-cnx-1 = { };
+  };
 in
 {
   # Ensure this is unique among all clans you want to use.
   meta.name = "cnx-network-clan";
   meta.domain = "cnx-network.internal";
 
-  inventory.machines = {
+  inventory.machines = fleet;
-    control = { };
-    ns1 = { };
-    ns2 = { };
-  };
 
   inventory.instances = {
 
-    admin = {
+    # Admin SSH keys + root password, split per the clan-core migration off
-      roles.default.tags.all = { };
+    # the deprecated `admin` service (sshd handles keys, users the password).
-      roles.default.settings.allowedKeys = {
+    sshd = {
+      roles.server.tags.all = { };
+      roles.server.settings.authorizedKeys = {
         "berwn" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIENAjhGQGraQoAjJzsomKP8GAmQPeGL1rNRNHgRcLqtT";
         "kurogeek" =
           "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEcZ/p1Ofa9liwIzPWzNtONhJ7+FUWd2lCz33r81t8+w kurogeek@kurogeek";
       };
     };
 
+    root-user = {
+      module = {
+        name = "users";
+        input = "clan-core";
+      };
+      roles.default.tags.all = { };
+      roles.default.settings = {
+        user = "root";
+        prompt = false; # auto-generate, like the old admin service
+        share = false; # per-machine password, not fleet-wide
+      };
+    };
+
     zerotier = {
       roles.controller.machines."control" = { };
       roles.peer.tags.all = { };
       # External members admitted by ZeroTier node id (stable per device).
       # Inventory machines are auto-accepted; this is only for peers outside the
       # clan. Node id comes from `zerotier-cli info` on the joining device.
-      roles.controller.settings.allowedIps = map ztMemberIp [
+      roles.controller.settings.allowedIds = [
         "8802c8d7e0" # alex-nixos
         "2bd36db8cc" # kurogeek-thinkpad
       ];
@@ -51,14 +66,12 @@ in
     };
 
     # Direct SSH to public IPs — clan's priority-1 connection path, with the
-    # ZeroTier mesh and Tor kept as automatic fallbacks. Raw IPs (not the
+    # ZeroTier mesh and Tor kept as automatic fallbacks. Raw IPs (from
-    # ns1/ns2 DNS names) so reaching these hosts never depends on their own
+    # modules/hosts.nix, not the ns1/ns2 DNS names) so reaching these hosts never
-    # DNS being up.
+    # depends on their own DNS being up.
-    internet = {
+    internet.roles.default.machines = builtins.mapAttrs (_: h: {
-      roles.default.machines.control.settings.host = "77.42.68.181";
+      settings.host = h.ipv4;
-      roles.default.machines.ns1.settings.host = "46.224.170.206";
+    }) hosts;
-      roles.default.machines.ns2.settings.host = "157.180.70.82";
-    };
 
     # Recovery root password for console access when a machine fails to boot.
     emergency-access = {
@@ -76,6 +89,11 @@ in
     };
   };
 
-  machines = {
+  # Fleet-wide baseline applied to every machine. Secrets minted by
-  };
+  # `clan vars generate` are encrypted for the admins group from the very
+  # first run — generating before this took effect is what forced the
+  # re-encryption dance (`clan vars fix`) on gw-cnx-1.
+  machines = builtins.mapAttrs (_: _: {
+    clan.core.sops.defaultGroups = [ "admins" ];
+  }) fleet;
 }

docs/src/SUMMARY.md

@@ -3,5 +3,7 @@
 - [Overview](./overview.md)
 - [ZeroTier mesh](./mesh.md)
 - [DNS](./dns.md)
+- [Mail](./mail.md)
+- [Site gateways](./gateways.md)
 - [Monitoring](./monitoring.md)
 - [Backups](./backups.md)

docs/src/dns.md

@@ -32,18 +32,54 @@ Automatic signing on `ns1` only, policy `cnx`: ECDSA P-256/SHA-256. The ZSK
 auto-rolls; the KSK is kept stable, so the DS at the registrar only changes on a
 manual KSK rollover.
 
+### Registrar DS records
+
+Knot manages all key material itself on `ns1` (the only signer); the KSK/ZSK
+private keys live in the KASP keystore under `/var/lib/knot` (backed up nightly —
+see [Backups](./backups.md)). You never touch the private keys directly.
+
+What a registrar needs is the **DS record** for a zone's KSK, which anchors the
+zone into the parent's chain of trust. Generate it on `ns1` — the `keymgr` wrapper
+is already pointed at Knot's config, and it runs as the `knot` user that owns the
+keystore:
+
+```
+sudo -u knot keymgr <zone> ds
+```
+
+e.g. `sudo -u knot keymgr cnx.email ds`. Paste the printed DS record (key tag,
+algorithm 13, digest type, digest) into the registrar's DNSSEC form for that
+domain. Repeat per signed zone (`cnx.network`, `buildfor.life`, `cnx.email`) at
+whichever registrar holds each delegation. After submitting, confirm the parent
+publishes it with `dig +short DS <zone>`.
+
+The ZSK rolls automatically and needs no registrar action; only a **KSK rollover**
+requires re-submitting the DS.
+
 > **Pending (manual):** submit DS records for `buildfor.life` and `cnx.email`
 > once they're at a DNSSEC-capable registrar.
 
 ## ACME DNS-01
 
-A dedicated TSIG key (`acme_ddns`), scoped by `acl_acme` to `TXT` updates at or
+Certificates are issued by `_acme-challenge` TXT updates that `ns1` accepts over
-under `_acme-challenge.<zone>` on `ns1` only. Knot signs the record and transfers
+TSIG, signs, and transfers to `ns2` (which never needs these keys). Each consumer
-it to `ns2`, which never needs this key. Retrieve the client config with:
+gets its **own** key, scoped by an ACL to exactly the owner names it needs and
+attached only to the zone it lives in — so a leaked key can write nothing but its
+own challenges.
 
+- **`acme_ddns`** (`acl_acme`) — the general key, scoped to `TXT` at or under
+  `_acme-challenge.<zone>` and attached to every zone. Client config:
   ```
   clan vars get ns1 dns-acme-tsig/acme.conf
   ```
+- **`acme_mx1`** (`acl_acme_mx1`) — held only by `mx1`, scoped to
+  `_acme-challenge.{mx1,mta-sts,mail}` and attached only to `cnx.email` (the mail
+  cert plus its MTA-STS and client-alias SANs). Secret shared via the
+  `dns-acme-mx1-secret` generator.
+- **`acme_web01`** (`acl_acme_web01`) — held only by `web01`, scoped to
+  `_acme-challenge` and attached only to `cnx.network` (where the wildcard
+  `*.cnx.network` challenge lands, at the apex). Secret shared via the
+  `dns-acme-web01-secret` generator.
 
 ## Runbook: stale secondary
 

docs/src/gateways.md

@@ -0,0 +1,145 @@
+# Site gateways
+
+NixOS routers (OPNsense replacements) on Topton 1U fanless boxes — Intel N300,
+4x i226-V 2.5G ports (some units add 2x 10G SFP+), 16GB DDR5, 256GB mSATA.
+Four sites are planned; the first is `gw-cnx-1` (Chiang Mai). Everything lives
+in the reusable `modules/router/` module (`cnx.router.*` options); a site's
+machine config only sets port names, VLANs, and feature flags.
+
+Naming: `gw-<city>-<n>`, e.g. `gw-cnx-1`.
+
+## What each gateway runs
+
+| Function     | Implementation                                                                                                                                                                                |
+| ------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| WAN          | PPPoE (`pppd`), per-site ISP credentials via clan vars prompts; `wan.vlanId` when the ISP tags the session (AIS: 10); `wan.macAddress` to clone the old router's MAC if the ISP has it pinned |
+| LAN          | VLAN-filtering bridge `br0` over the trunk ports (networkd); `accessPorts` pin a port untagged to one VLAN — convention: the last copper port is an untagged `mgmt` recovery port             |
+| Firewall/NAT | nftables: default-deny WAN, no inter-VLAN, MSS clamp, v4 NAT                                                                                                                                  |
+| DHCP         | Kea, one subnet per VLAN                                                                                                                                                                      |
+| DNS          | Blocky (blocklist resolver), metrics on :4000 scraped by control                                                                                                                              |
+| IPv6         | DHCPv6-PD on ppp0, /64 per VLAN via SLAAC                                                                                                                                                     |
+| Bans         | CrowdSec + nftables bouncer (sshd log parsing)                                                                                                                                                |
+| Omada        | Optional per site: TP-Link Omada controller as a podman container                                                                                                                             |
+| Proxy        | Optional: Caddy reverse proxy for internal services under `*.<site><n>.cnx.network` with a real Let's Encrypt wildcard (DNS-01 against ns1)                                                   |
+| Diagnostics  | iperf3 server on 5201, reachable from every VLAN and the mesh (`iperf3 -c <router-vlan-addr>`); CLI toolkit: tcpdump, mtr, ethtool, conntrack, kdig, iftop, librespeed-cli                    |
+| Speed test   | Hourly librespeed run (`speedtest.timer`) → `speedtest_*` metrics via node_exporter; vmalert flags download < 50% of the link's own 7-day median (`WANSpeedDegraded`)                         |
+| Management   | ZeroTier mesh: SSH, node_exporter, journald upload — like the fleet                                                                                                                           |
+| Storage      | Single-disk ZFS (zstd, `/var` its own dataset for snapshots)                                                                                                                                  |
+
+## Addressing plan
+
+Each site owns `10.<siteId>.0.0/16`. Defaults per VLAN: subnet
+`10.<siteId>.<vlanId>.0/24`, router at `.1`, DHCP pool `.100–.199`
+(`.2–.99` static/infra, `.200–.254` reserved).
+
+VLAN id convention (fleet-wide): **10 = mgmt**, **20 = lan** (both mandatory at
+every site), 30 = guest, 40 = iot (reserved). Sites add their own beyond that.
+
+Larger subnets (e.g. public-WiFi guest networks that outgrow a /24) are carved
+from the **upper half** `10.<siteId>.128.0/17` and set explicitly on the VLAN.
+The lower half stays reserved for /24s indexed by VLAN id. High-churn VLANs
+should also shorten `dhcp.leaseTime` (default 86400 s) so the pool recycles.
+First user: `gw-cnx-2` (site 2) runs the public WiFi — guest VLAN 30 at
+`10.2.128.0/22`, pool `10.2.128.100 – 10.2.131.250`, `dhcp.leaseTime = 3600`.
+
+| Site | siteId | mgmt           | lan            | site-specific VLANs                                             |
+| ---- | ------ | -------------- | -------------- | --------------------------------------------------------------- |
+| cnx  | 1      | `10.1.10.0/24` | `10.1.20.0/24` | iot 40, voip 50, dmz 60, unit1–5 110/120/130/140/150 (all /24s) |
+
+Static DHCP leases are declared per VLAN via `dhcp.reservations` (attribute
+name = hostname, plus `hwAddress`/`ipAddress`); park them outside the pool,
+in the `.2–.99` infra range or `.200–.254`.
+
+Trust model: mgmt → everything; other VLANs → router DNS/DHCP + internet only
+(no inter-VLAN); WAN → nothing inbound; mesh → SSH, metrics, Omada UI.
+
+## Adding a site
+
+1. Copy `machines/gw-cnx-1/` to `machines/gw-<city>-<n>/`; set `site`,
+   `siteId` (next free number), port names, VLANs, and `omada.enable`.
+   Fill the real disk id in `disko.nix` (from the installer:
+   `ls -l /dev/disk/by-id/`). Keep the `mgmt`/`lan` VLANs.
+2. Add the machine to `inventory.machines` in `clan.nix` and to the machine
+   list in `modules/mesh-hosts.nix`. Do **not** add it to `modules/hosts.nix`
+   (dynamic PPPoE IP; clan connects over the mesh).
+3. Add its node_exporter and Blocky scrape targets in
+   `modules/monitoring/server.nix`.
+4. If the site uses the internal proxy (`cnx.router.proxy.enable`), wire ns1:
+   import `(import ../../modules/dns/acme-gw-secret.nix "gw-<city>-<n>")` in
+   **both** the gateway's and ns1's configuration, and on ns1 add the
+   `dns-acme-gw-<city>-<n>-knot` generator, its `keyFiles` entry, and an
+   `acl_acme_gw_<city>_<n>` scoped to `_acme-challenge.<city><n>` on the
+   `cnx.network` zone (copy the `gw-cnx-1` blocks). Then
+   `clan vars generate ns1` and redeploy ns1.
+5. `git add` the new machine directory — flake evaluation only sees
+   git-tracked files, so an untracked `machines/gw-…/` is silently ignored.
+6. `clan vars generate gw-<city>-<n>` — prompts for the site's PPPoE
+   credentials, mints the ZeroTier identity etc. (`nix flake check` fails
+   until this has run, because mesh-hosts reads the ZeroTier IP var.)
+7. Boot the box from a NixOS installer USB on the local network, then:
+   `clan machines install gw-<city>-<n> --target-host root@<lan-ip>`
+8. Check `facter.json` for the real NIC names, fix `wan.interface` /
+   `trunkPorts` if the enumeration differs, and
+   `clan machines update gw-<city>-<n>` (rides the mesh from then on).
+9. Add a row to the site table above and to the machines table in
+   [Overview](./overview.md).
+
+## Omada controller
+
+Sites with TP-Link Omada APs/switches set `cnx.router.omada.enable = true`.
+The controller runs as a podman container (`mbentley/omada-controller` — there
+is no nixpkgs package) with host networking on the mgmt VLAN, where adoption
+broadcasts (UDP 29810) live. UI: `https://<mgmt-addr>:8043` from mgmt, or the
+gateway's mesh address from an admin machine. Controller state is under
+`/var/lib/omada`, declared as clan state (`clan.core.state.omada`); wiring it
+into the borgbackup instance is still a follow-up.
+
+## Internal reverse proxy
+
+`cnx.router.proxy.enable` puts Caddy on the gateway, terminating TLS for
+`<service>.<site><n>.cnx.network` (e.g. `https://omada.cnx1.cnx.network`) and
+forwarding to internal backends:
+
+```nix
+cnx.router.proxy = {
+  enable = true;
+  services.omada = {
+    backend = "https://127.0.0.1:8043";
+    insecureSkipVerify = true; # Omada's cert is self-signed
+  };
+};
+```
+
+- **Cert**: one real Let's Encrypt wildcard `*.<site><n>.cnx.network` per
+  gateway, issued via ACME DNS-01 (lego/RFC2136) against ns1 — works behind
+  PPPoE with zero inbound reachability, and browsers trust it without a CA
+  install. Each gateway has its own TSIG key (`acme_gw_<city>_<n>`) that ns1
+  scopes to `_acme-challenge.<site><n>` TXT records only.
+- **Resolution**: the names exist only internally — Blocky answers
+  `*.<site><n>.cnx.network` with the router's `lan` address; the public
+  `cnx.network` zone never carries them.
+- **Access**: `proxy.allowVlans` (default `mgmt` + `lan`) get 443 (and 80 for
+  the HTTP→HTTPS redirect). Not exposed to WAN, guest VLANs, or the mesh.
+
+## Runbook
+
+- **PPPoE down**: `systemctl status pppd-wan`, `journalctl -u pppd-wan` on the
+  gateway (over the mesh). ISP credentials live in the `pppoe-credentials`
+  vars generator; re-enter with `clan vars generate gw-<city>-<n> --regenerate`.
+- **A VLAN gets no leases**: `systemctl status kea-dhcp4-server`; check the
+  port's `bridgeVLANs` tagging and that the switch trunk carries the VLAN.
+- **DNS/blocklist issues**: Blocky metrics are in VictoriaMetrics (job
+  `blocky`); `journalctl -u blocky` on the gateway.
+- **Slow internet reported / `WANSpeedDegraded` firing**: check the
+  `speedtest_download_mbps` / `speedtest_ping_ms` trend in VictoriaMetrics,
+  then `librespeed-cli` and `mtr <isp-gw>` on the gateway to separate ISP
+  degradation from local saturation (`iftop` on ppp0 shows who's eating it).
+- **Banned yourself**: `cscli decisions list` / `cscli decisions delete --ip <ip>`
+  on the gateway. The ZeroTier mesh `/88` is whitelisted at the parser stage
+  (`cnx/mesh-whitelist`), so admin access over the mesh can never be banned.
+
+## Follow-ups
+
+- Borgbackup client for Omada + Kea lease state.
+- Suricata (IDS-only) if CPU headroom allows — deliberately skipped for now.
+- Remaining three sites.

docs/src/mail.md

@@ -0,0 +1,90 @@
+# Mail
+
+A single mail server, `mx1`, runs the [Simple NixOS Mailserver](https://nixos-mailserver.readthedocs.io/)
+stack (Postfix + Dovecot + Rspamd + OpenDKIM) for the `cnx.email` domain. All of
+it is declared in `modules/mail.nix`, imported by `machines/mx1/configuration.nix`.
+
+Mailboxes are **virtual** (not system users): each address is a login account
+whose password is a four-word passphrase auto-generated by a clan vars generator
+(e.g. `otter-lantern-cobalt-driftwood-42`). The generator stores both the
+passphrase (handed to the user) and its sha-512 hash (consumed by SNM); plaintext
+never lands in the repo.
+
+The mail cert is obtained by `mx1` itself via ACME DNS-01 against `ns1` (key
+`acme_mx1`), and carries `mx1.cnx.email` plus the `mta-sts.cnx.email` and
+`mail.cnx.email` SANs. See [DNS → ACME DNS-01](./dns.md#acme-dns-01).
+
+## Add a mailbox
+
+1. Append the address to `accounts` in `modules/mail.nix`:
+   ```nix
+   accounts = [
+     "postmaster@cnx.email"
+     "alice@cnx.email"      # new
+   ];
+   ```
+2. Mint its generated password (user-run, YubiKey touch):
+   ```
+   clan vars generate mx1
+   ```
+3. Redeploy: `clan machines update mx1`.
+4. Hand the passphrase to the user. The generator name replaces `@`→`-at-` and
+   `.`→`-`:
+   ```
+   clan vars get mx1 mail-passwd-alice-at-cnx-email/passphrase
+   ```
+
+The DMARC report inbox (`dmarc@cnx.email`) is special: its password comes from
+the **shared** `mail-dmarc-cred` generator (not the per-machine set) so parsedmarc
+on `control` can read the same passphrase over the mesh.
+
+## Add a domain
+
+Adding a second mail domain is more than a config line — it needs its own DNS
+records and DKIM key. Steps:
+
+1. **Serve the zone** (if not already): add it in `modules/dns/domains.nix` and
+   drop a `<domain>.zone` file — see [DNS](./dns.md).
+2. **Declare the domain** in `modules/mail.nix`:
+   ```nix
+   mailserver.domains = [ "cnx.email" "newdomain.tld" ];
+   ```
+   Add at least a `postmaster@newdomain.tld` account (RFC 5321 requires it) the
+   same way as above.
+3. **Publish the mail DNS records** in the new zone, mirroring `cnx.email`'s
+   `; ---- Mail ----` block:
+   - `MX 10 mx1.cnx.email.` — point at the existing MX; do **not** stand up a new
+     hostname unless you really want a separate server.
+   - SPF: `@ TXT "v=spf1 mx -all"`.
+   - DKIM: SNM generates a key per domain on first start under
+     `mailserver.dkimKeyDirectory` (selector `mail`). After deploy, read the
+     public key off `mx1` and paste it as `mail._domainkey TXT`. The private keys
+     are declared as clan state and backed up.
+   - DMARC: `_dmarc TXT "v=DMARC1; p=quarantine; rua=mailto:dmarc@cnx.email; ..."`
+     — reports can keep flowing to the existing `dmarc@cnx.email` inbox.
+4. **DANE / MTA-STS are tied to `mx1.cnx.email`**, the MX hostname — they do **not**
+   need duplicating per sending-domain. A new domain that uses `mx1.cnx.email` as
+   its MX inherits the existing TLSA and MTA-STS policy. Only add new TLSA/MTA-STS
+   records if you introduce a new MX hostname.
+5. `clan vars generate mx1` (if you added accounts), then
+   `clan machines update mx1`.
+
+## DKIM, DANE, MTA-STS reminders
+
+These three are easy to get subtly wrong; the live records for `cnx.email` and
+the exact commands to regenerate them are in `modules/dns/zones/cnx.email.zone`:
+
+- **DKIM** key rotation = regenerate under `dkimKeyDirectory`, then republish the
+  `mail._domainkey TXT`.
+- **DANE TLSA** (`_25._tcp.mx1 TLSA 3 1 1 …`) is the SHA-256 of `mx1`'s cert
+  **public key**. It stays valid across renewals because lego runs with
+  `--reuse-key`; only recompute it if the key changes. The `openssl` one-liner is
+  in the zone file.
+- **MTA-STS**: bump the `_mta-sts TXT` id **every time** the policy in
+  `modules/mail.nix` changes, or senders keep the cached policy.
+
+## DMARC reporting
+
+`mx1` delivers aggregate/forensic DMARC reports to `dmarc@cnx.email`; parsedmarc
+on `control` polls that mailbox over the mesh and feeds the results into Grafana.
+See [Monitoring](./monitoring.md).

docs/src/mesh.md

@@ -10,10 +10,12 @@ transfers and metrics scraping ride this mesh, never the public net.
 ## The mesh map
 
 `modules/mesh-hosts.nix` does **not** hardcode addresses. It reads each machine's
-IP from the public clan vars that clan-core's zerotier generator already writes
+IP from the public clan vars that clan-core's zerotier service writes. As of
-(`vars/per-machine/<m>/zerotier/zerotier-ip/value`) and derives the `/88` subnet
+clan-core 26.05 these are shared, instance-scoped generators
-from `control`'s `zerotier-network-id`. Regenerate or re-key a node and the map
+(`vars/shared/zerotier-ip-<machine>-zerotier/ip/value`), and the `/88` subnet is
-follows automatically.
+derived from the instance network id
+(`vars/shared/zerotier-network-zerotier/network-id/value`). Regenerate or re-key a
+node and the map follows automatically.
 
 Consumers: `modules/dns/authoritative.nix` (transfer ACLs), `modules/monitoring/*`
 (scrape targets and firewall scoping).
@@ -21,12 +23,11 @@ Consumers: `modules/dns/authoritative.nix` (transfer ACLs), `modules/monitoring/
 ## Admitting external members
 
 Inventory machines are auto-accepted. External devices (admin laptops) are listed
-in `clan.nix` under the controller's `allowedIps`. Because this clan-core pins the
+in `clan.nix` under the controller's `allowedIds` — clan-core 26.05 admits members
-`allowedIps` interface (admit by network IPv6), we keep a **node-id** list and a
+by their 10-char ZeroTier node id directly, so no IP derivation is needed:
-`ztMemberIp` helper derives each device's IP on this network:
 
 ```nix
-roles.controller.settings.allowedIps = map ztMemberIp [
+roles.controller.settings.allowedIds = [
   "8802c8d7e0" # alex-nixos
   "2bd36db8cc" # kurogeek-thinkpad
 ];
@@ -34,6 +35,3 @@ roles.controller.settings.allowedIps = map ztMemberIp [
 
 A device's 10-char node id comes from `zerotier-cli info` on that device. After
 editing, deploy `control`; the controller admits the new member on its next run.
-
-> A newer clan-core exposes `allowedIds` (admit by node id directly), but adopting
-> it means a zerotier vars-schema migration, so we stay on the IP-derivation path.

docs/src/monitoring.md

@@ -1,6 +1,7 @@
 # Monitoring
 
-Metrics and dashboards live on `control`, reachable only over the ZeroTier mesh.
+Metrics and logs live on `control` over the ZeroTier mesh; the Grafana dashboards
+are also published publicly through `web01` (see [Dashboards](#dashboards)).
 
 ## Collection
 
@@ -18,8 +19,8 @@ Metrics and dashboards live on `control`, reachable only over the ZeroTier mesh.
 ## Storage & scraping
 
 **VictoriaMetrics** on `control`, bound to `127.0.0.1:8428`, 180-day retention
-(`modules/monitoring/server.nix`). It scrapes `control` over loopback and `ns1`/
+(`modules/monitoring/server.nix`). It scrapes `control` over loopback and
-`ns2` over the mesh.
+`ns1`/`ns2`/`mx1`/`web01` over the mesh.
 
 > The scraper dials IPv4-only by default, so mesh (IPv6) targets need
 > `extraOptions = [ "-enableTCP6" ]`. Without it, ns1/ns2 are dropped with
@@ -31,8 +32,10 @@ Metrics and dashboards live on `control`, reachable only over the ZeroTier mesh.
 
 ## Dashboards
 
-**Grafana** on `control` (`:3000`), mesh-only, anonymous access disabled. The
+**Grafana** on `control` (`:3000`), anonymous access disabled. Reachable directly
-admin password is a clan var:
+over the mesh, and publicly at `https://grafana.cnx.network` via `web01`'s reverse
+proxy (TLS termination — see [Overview](./overview.md)). The admin password is a
+clan var:
 
 ```
 clan vars get control grafana-admin/password
@@ -46,6 +49,51 @@ there is picked up):
   outside-in DNS probes.
 - **CNX Backups** (`backups.json`) — borgbackup job health, time since the last
   run, and per-job state. See [Backups](./backups.md).
+- **CNX Uptime** (`uptime.json`) — per-host up/down status, current uptime,
+  availability over the selected window, and up/down history. Label-driven, so
+  every scraped host appears automatically.
+- **CNX WAN Speed** (`wan.json`) — per-gateway WAN speed from the hourly
+  librespeed runs: latest download/upload/ping, throughput history with each
+  link's 7-day median as reference, latency/jitter, and test status. See
+  [Site gateways](./gateways.md).
+- **parsedmarc** — DMARC aggregate/forensic report viewer. Auto-provisioned by
+  the `parsedmarc` module (not from `dashboards/`); reads its own Elasticsearch
+  datasource, not VictoriaMetrics. See [DMARC reports](#dmarc-reports) below.
+
+## DMARC reports
+
+The `cnx.email` DMARC record (`rua`/`ruf`) points at the `dmarc@cnx.email`
+mailbox on `mx1`. **parsedmarc** on `control` (`modules/monitoring/parsedmarc.nix`)
+polls that mailbox over IMAPS, parses the XML reports, and stores them in a local
+**Elasticsearch** (`127.0.0.1:9200`, loopback-only); Grafana renders them via the
+parsedmarc dashboard + two Elasticsearch datasources (`dmarc-ag`/`dmarc-fo`).
+
+> Two Grafana-13 gotchas, both handled in `modules/monitoring/parsedmarc.nix`:
+>
+> 1. **The Elasticsearch data source plugin is no longer bundled.** Grafana 13
+>    externalised its core data sources and nixpkgs ships no `grafanaPlugins`
+>    entry for Elasticsearch, so a bare ES datasource fails with "Plugin not
+>    registered" and every panel errors (and Explore silently falls back to the
+>    default datasource). We pull the official, Grafana-signed `elasticsearch`
+>    plugin from the catalog via `services.grafana.declarativePlugins`.
+> 2. **Datasource variables now resolve by UID, not name.** The bundled dashboard
+>    binds its datasource through template variables whose stored value is the
+>    datasource _name_ (`dmarc-ag`/`dmarc-fo`); Grafana 11+ matches that against
+>    the datasource **UID**. So we provision the two datasources ourselves with
+>    their **UID pinned to their name** (and leave the module's own
+>    `provision.grafana.datasource` off to avoid a duplicate, uid-less pair).
+>    `deleteDatasources` clears any stale uid-less records left by an earlier
+>    deploy so the pinned-UID versions can be re-created.
+
+The IMAP fetch rides the **mesh**, not the public net: `control` pins
+`mx1.cnx.email` to mx1's mesh address in `/etc/hosts`, so TLS still validates
+against the public cert while the bytes stay on the overlay. The mailbox
+passphrase is the shared `mail-dmarc-cred` clan var (so both mx1's mailserver and
+control's parsedmarc see the same value):
+
+```
+clan vars get mx1 mail-dmarc-cred/passphrase
+```
 
 ## Logs
 
@@ -53,8 +101,8 @@ there is picked up):
 (`modules/monitoring/server.nix`). All three hosts ship journald to it via
 systemd's own `services.journald.upload` → the `/insert/journald` endpoint
 (`modules/monitoring/exporters.nix`); no extra agent. `control` uploads over
-loopback so its logs survive a mesh outage, `ns1`/`ns2` push over the mesh, and
+loopback so its logs survive a mesh outage, the other hosts push over the mesh,
-9428 is firewall-scoped to the mesh like everything else.
+and 9428 is firewall-scoped to the mesh like everything else.
 
 > Same IPv4-only default as the scraper: VictoriaLogs binds `0.0.0.0:9428` for a
 > bare `:9428`, so mesh (IPv6) pushes from ns1/ns2 are refused until you pass

docs/src/overview.md

@@ -7,10 +7,13 @@ this book is built from `docs/` and served on `control` over the ZeroTier mesh.
 ## Machines
 
 | Machine    | Role                                   | Public IPv4      | Public IPv6             |
-| --------- | -------------------------------------- | ---------------- | --------------------------- |
+| ---------- | -------------------------------------- | ---------------- | ----------------------- |
 | `control`  | ZeroTier controller, monitoring, docs  | `77.42.68.181`   | `2a01:4f9:c013:e6d0::1` |
 | `ns1`      | Knot DNS **primary** (master)          | `46.224.170.206` | `2a01:4f8:c014:b5c5::1` |
 | `ns2`      | Knot DNS **secondary** (slave)         | `157.180.70.82`  | `2a01:4f9:c014:6d87::1` |
+| `mx1`      | Mail server (**MX** for cnx.email)     | `5.223.65.38`    | `2a01:4ff:2f0:1963::1`  |
+| `web01`    | Public reverse proxy (TLS termination) | `5.223.55.246`   | `2a01:4ff:2f0:2d8f::1`  |
+| `gw-cnx-1` | Site gateway Chiang Mai (router)       | dynamic (PPPoE)  | —                       |
 
 ## Access
 
@@ -19,6 +22,14 @@ this book is built from `docs/` and served on `control` over the ZeroTier mesh.
 - clan reaches machines by their public IPs first (the `internet` instance), with
   the mesh and Tor as automatic fallbacks.
 
+## Host baseline
+
+Every machine sets a fixed-offset `time.timeZone` and runs **chrony** for time
+sync (`services.chrony.enable`). chrony is the single NTP implementation across
+the fleet — do **not** use `systemd-timesyncd`; the two are mutually exclusive
+and we standardise on chrony everywhere. New hosts must set both the timezone and
+chrony as part of their baseline config.
+
 ## Editing these docs
 
 Commit-to-edit: change the markdown under `docs/src/`, commit, and redeploy

flake.lock

@@ -1,5 +1,21 @@
 {
   "nodes": {
+    "blobs": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1604995301,
+        "narHash": "sha256-wcLzgLec6SGJA8fx1OEN1yV/Py5b+U5iyYpksUY/yLw=",
+        "owner": "simple-nixos-mailserver",
+        "repo": "blobs",
+        "rev": "2cccdf1ca48316f2cfd1c9a0017e8de5a7156265",
+        "type": "gitlab"
+      },
+      "original": {
+        "owner": "simple-nixos-mailserver",
+        "repo": "blobs",
+        "type": "gitlab"
+      }
+    },
     "clan-core": {
       "inputs": {
         "data-mesher": "data-mesher",
@@ -7,22 +23,21 @@
         "flake-parts": "flake-parts",
         "nix-darwin": "nix-darwin",
         "nix-select": "nix-select",
-        "nixos-facter-modules": "nixos-facter-modules",
         "nixpkgs": "nixpkgs",
         "sops-nix": "sops-nix",
         "systems": "systems",
         "treefmt-nix": "treefmt-nix"
       },
       "locked": {
-        "lastModified": 1779116933,
+        "lastModified": 1783497933,
-        "narHash": "sha256-BoSQT4vbuWbGcDbXp4nqBd1I9X8r+dLSPtTbc5yJd28=",
+        "narHash": "sha256-TxmwEews6URFPqOWEHNychtXbFDgLZjbOfEXtvtOm6U=",
-        "rev": "10d86d5d8291fe5162a5535f433a4d50efc4b2bc",
+        "rev": "3dc0221ca09033599fe98055e9bbc81bdf32732a",
         "type": "tarball",
-        "url": "https://git.clan.lol/api/v1/repos/clan/clan-core/archive/10d86d5d8291fe5162a5535f433a4d50efc4b2bc.tar.gz"
+        "url": "https://git.clan.lol/api/v1/repos/clan/clan-core/archive/3dc0221ca09033599fe98055e9bbc81bdf32732a.tar.gz"
       },
       "original": {
         "type": "tarball",
-        "url": "https://git.clan.lol/clan/clan-core/archive/25.11.tar.gz"
+        "url": "https://git.clan.lol/clan/clan-core/archive/26.05.tar.gz"
       }
     },
     "data-mesher": {
@@ -41,11 +56,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1767582502,
+        "lastModified": 1778718524,
-        "narHash": "sha256-WVcYGWcAlWzVt38OaTC5i5Q3QkIKJKZsJ7LcQZVVxeE=",
+        "narHash": "sha256-pXLoI6Ax0EnUK6r34UM1vibVC7CfTu6j72R2692ZzPs=",
-        "rev": "31f2e3ecf207fd2760e3cebf7c2cf3cb7170ea3d",
+        "rev": "12c552ad547d87254f33f33bddd1a2cdbeac754d",
         "type": "tarball",
-        "url": "https://git.clan.lol/api/v1/repos/clan/data-mesher/archive/31f2e3ecf207fd2760e3cebf7c2cf3cb7170ea3d.tar.gz"
+        "url": "https://git.clan.lol/api/v1/repos/clan/data-mesher/archive/12c552ad547d87254f33f33bddd1a2cdbeac754d.tar.gz"
       },
       "original": {
         "type": "tarball",
@@ -60,11 +75,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1766150702,
+        "lastModified": 1780290312,
-        "narHash": "sha256-P0kM+5o+DKnB6raXgFEk3azw8Wqg5FL6wyl9jD+G5a4=",
+        "narHash": "sha256-eTAlX0CwgB84Ts3GaBd944A3DRXVMzgA0EqroZBISUo=",
         "owner": "nix-community",
         "repo": "disko",
-        "rev": "916506443ecd0d0b4a0f4cf9d40a3c22ce39b378",
+        "rev": "115e5211780054d8a890b41f0b7734cafad54dfe",
         "type": "github"
       },
       "original": {
@@ -73,6 +88,22 @@
         "type": "github"
       }
     },
+    "flake-compat": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1767039857,
+        "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=",
+        "owner": "NixOS",
+        "repo": "flake-compat",
+        "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
     "flake-parts": {
       "inputs": {
         "nixpkgs-lib": [
@@ -81,11 +112,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1777988971,
+        "lastModified": 1778716662,
-        "narHash": "sha256-qIoWPDs+0/8JecyYgE3gpKQxW/4bLW/gp45vow9ioCQ=",
+        "narHash": "sha256-m1Yf0wZ8j1OHjTc2UwHwyQRSnNeSgLJOd7q5Y45hzi4=",
         "owner": "hercules-ci",
         "repo": "flake-parts",
-        "rev": "0678d8986be1661af6bb555f3489f2fdfc31f6ff",
+        "rev": "f7c1a2d347e4c52d5fb8d10cb4d94b5884e546fb",
         "type": "github"
       },
       "original": {
@@ -94,6 +125,54 @@
         "type": "github"
       }
     },
+    "git-hooks": {
+      "inputs": {
+        "flake-compat": [
+          "nixos-mailserver",
+          "flake-compat"
+        ],
+        "gitignore": "gitignore",
+        "nixpkgs": [
+          "nixos-mailserver",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1778507602,
+        "narHash": "sha256-kTwur1wV+01SdqskVMSo6JMEpg71ps3HpbFY2GsflKs=",
+        "owner": "cachix",
+        "repo": "git-hooks.nix",
+        "rev": "61ab0e80d9c7ab14c256b5b453d8b3fb0189ba0a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "git-hooks.nix",
+        "type": "github"
+      }
+    },
+    "gitignore": {
+      "inputs": {
+        "nixpkgs": [
+          "nixos-mailserver",
+          "git-hooks",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1709087332,
+        "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=",
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "rev": "637db329424fd7e46cf4185293b9cc8c88c95394",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "type": "github"
+      }
+    },
     "nix-darwin": {
       "inputs": {
         "nixpkgs": [
@@ -102,16 +181,15 @@
         ]
       },
       "locked": {
-        "lastModified": 1772129556,
+        "lastModified": 1779036909,
-        "narHash": "sha256-Utk0zd8STPsUJPyjabhzPc5BpPodLTXrwkpXBHYnpeg=",
+        "narHash": "sha256-zXcwYQGCT6pzinK+1dBB2ekTVtfxGZAapb3Evdcu4fY=",
         "owner": "nix-darwin",
         "repo": "nix-darwin",
-        "rev": "ebec37af18215214173c98cf6356d0aca24a2585",
+        "rev": "56c666e108467d87d13508936aade6d567f2a501",
         "type": "github"
       },
       "original": {
         "owner": "nix-darwin",
-        "ref": "nix-darwin-25.11",
         "repo": "nix-darwin",
         "type": "github"
       }
@@ -129,40 +207,51 @@
         "url": "https://git.clan.lol/clan/nix-select/archive/main.tar.gz"
       }
     },
-    "nixos-facter-modules": {
+    "nixos-mailserver": {
+      "inputs": {
+        "blobs": "blobs",
+        "flake-compat": "flake-compat",
+        "git-hooks": "git-hooks",
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
       "locked": {
-        "lastModified": 1773858690,
+        "lastModified": 1785198005,
-        "narHash": "sha256-oW0/lC0oRG5H5LaK6Rmh9L1wmkn9TbenM4bXwnIEDKA=",
+        "narHash": "sha256-falBPi+PJtMx0vwII8L24wjDGXNZMwRiVXsw8WTXLEo=",
-        "owner": "nix-community",
+        "owner": "simple-nixos-mailserver",
-        "repo": "nixos-facter-modules",
+        "repo": "nixos-mailserver",
-        "rev": "139dcef4dfc97009629c445806f197883351ab4a",
+        "rev": "d357b9f048c5532ec81b0e0034c0b8463d5ddd46",
-        "type": "github"
+        "type": "gitlab"
       },
       "original": {
-        "owner": "nix-community",
+        "owner": "simple-nixos-mailserver",
-        "repo": "nixos-facter-modules",
+        "ref": "nixos-26.05",
-        "type": "github"
+        "repo": "nixos-mailserver",
+        "type": "gitlab"
       }
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1778003029,
+        "lastModified": 1780453794,
-        "narHash": "sha256-q/nkKLDtHIyLjZpKhWk3cSK5IYsFqtMd6UtXF3ddjgA=",
+        "narHash": "sha256-hhAl/iKiurXPn7rdzDgiSuRB8tqOB6f0buWkh8Y9mkY=",
-        "owner": "nixos",
+        "rev": "6b316287bae2ee04c9b93c8c858d930fd07d7338",
-        "repo": "nixpkgs",
+        "type": "tarball",
-        "rev": "0c88e1f2bdb93d5999019e99cb0e61e1fe2af4c5",
+        "url": "https://releases.nixos.org/nixos/26.05/nixos-26.05.1183.6b316287bae2/nixexprs.tar.xz"
-        "type": "github"
       },
       "original": {
-        "owner": "nixos",
+        "type": "tarball",
-        "ref": "nixos-25.11",
+        "url": "https://nixos.org/channels/nixos-26.05/nixexprs.tar.xz"
-        "repo": "nixpkgs",
-        "type": "github"
       }
     },
     "root": {
       "inputs": {
         "clan-core": "clan-core",
+        "flake-parts": [
+          "clan-core",
+          "flake-parts"
+        ],
+        "nixos-mailserver": "nixos-mailserver",
         "nixpkgs": [
           "clan-core",
           "nixpkgs"
@@ -178,11 +267,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1777944972,
+        "lastModified": 1780547341,
-        "narHash": "sha256-VfGRo1qTBKOe3s2gOv8LSoA6Fk19PvBlwQ1ECN0Evn8=",
+        "narHash": "sha256-Gq8KNx5A7hBB3uGJaj6eQfLDIz5YdLu92gqBcvHvoUo=",
         "owner": "Mic92",
         "repo": "sops-nix",
-        "rev": "c591bf665727040c6cc5cb409079acb22dcce33c",
+        "rev": "9ed65852b6257fbeae4355bc24ecfea307ca759a",
         "type": "github"
       },
       "original": {
@@ -193,15 +282,16 @@
     },
     "systems": {
       "locked": {
-        "lastModified": 1681028828,
+        "lastModified": 1774449309,
-        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "narHash": "sha256-brhZ8DmuGtzkCYHJg4HEd602amKm89Y9ytsFZ5uWD1w=",
         "owner": "nix-systems",
         "repo": "default",
-        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "rev": "c29398b59d2048c4ab79345812849c9bd15e9150",
         "type": "github"
       },
       "original": {
         "owner": "nix-systems",
+        "ref": "future-26.11",
         "repo": "default",
         "type": "github"
       }
@@ -213,26 +303,6 @@
           "nixpkgs"
         ]
       },
-      "locked": {
-        "lastModified": 1768158989,
-        "narHash": "sha256-67vyT1+xClLldnumAzCTBvU0jLZ1YBcf4vANRWP3+Ak=",
-        "owner": "numtide",
-        "repo": "treefmt-nix",
-        "rev": "e96d59dff5c0d7fddb9d113ba108f03c3ef99eca",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "treefmt-nix",
-        "type": "github"
-      }
-    },
-    "treefmt-nix_2": {
-      "inputs": {
-        "nixpkgs": [
-          "nixpkgs"
-        ]
-      },
       "locked": {
         "lastModified": 1780220602,
         "narHash": "sha256-eynAfOmbmxJnkp7YewvCEbShNnnYJ9gLLqkzsYtBPeM=",
@@ -246,6 +316,26 @@
         "repo": "treefmt-nix",
         "type": "github"
       }
+    },
+    "treefmt-nix_2": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1784369104,
+        "narHash": "sha256-47cxbcZODibHv3rELFQ9vZly0vUNkND/atn/U7HLeb0=",
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "rev": "df3c0640565d04a0261253cdd89fce78ec50168a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "type": "github"
+      }
     }
   },
   "root": "root",

flake.nix

@@ -1,16 +1,22 @@
 {
-  inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/25.11.tar.gz";
+  inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/26.05.tar.gz";
   inputs.nixpkgs.follows = "clan-core/nixpkgs";
   inputs.treefmt-nix.url = "github:numtide/treefmt-nix";
   inputs.treefmt-nix.inputs.nixpkgs.follows = "nixpkgs";
+  # Simple NixOS Mailserver, pinned to the branch matching clan-core's nixpkgs.
+  inputs.nixos-mailserver.url = "gitlab:simple-nixos-mailserver/nixos-mailserver/nixos-26.05";
+  inputs.nixos-mailserver.inputs.nixpkgs.follows = "nixpkgs";
+
+  inputs.flake-parts.follows = "clan-core/flake-parts";
 
   outputs =
-    {
+    inputs@{
       self,
       clan-core,
       nixpkgs,
+      flake-parts,
       ...
-    }@inputs:
+    }:
     let
       # Usage see: https://docs.clan.lol
       clan = clan-core.lib.clan {
@@ -33,36 +39,60 @@
           "age-plugin-fido2-hmac"
         ];
       };
-
+    in
+    flake-parts.lib.mkFlake { inherit inputs; } {
       systems = [
         "x86_64-linux"
         "aarch64-linux"
         "aarch64-darwin"
         "x86_64-darwin"
       ];
-      forAllSystems = nixpkgs.lib.genAttrs systems;
+
-      pkgsFor = system: clan-core.inputs.nixpkgs.legacyPackages.${system};
+      flake = {
-      treefmtFor = system: inputs.treefmt-nix.lib.evalModule (pkgsFor system) ./fmt.nix;
-    in
-    {
         inherit (clan.config) nixosConfigurations nixosModules clanInternals;
         clan = clan.config;
+      };
 
+      perSystem =
+        { system, ... }:
+        let
+          pkgs = clan-core.inputs.nixpkgs.legacyPackages.${system};
+          treefmtEval = inputs.treefmt-nix.lib.evalModule pkgs ./fmt.nix;
+          clanCli = clan-core.packages.${system}.clan-cli;
+          # `clan machines update a b c` normally runs machines in parallel,
+          # which interleaves their output and buries the YubiKey PIN prompts.
+          # This wrapper (first in PATH) runs them one at a time instead; any
+          # flags fall through to the real CLI untouched.
+          clanSequential = pkgs.writeShellScriptBin "clan" ''
+            if [ "$#" -gt 3 ] && [ "$1" = machines ] && [ "$2" = update ]; then
+              shift 2
+              for arg in "$@"; do
+                case "$arg" in
+                  -*) exec ${clanCli}/bin/clan machines update "$@" ;;
+                esac
+              done
+              for machine in "$@"; do
+                ${clanCli}/bin/clan machines update "$machine" || exit
+              done
+              exit 0
+            fi
+            exec ${clanCli}/bin/clan "$@"
+          '';
+        in
+        {
           # `nix fmt` and the `nix flake check` formatting gate.
-      formatter = forAllSystems (system: (treefmtFor system).config.build.wrapper);
+          formatter = treefmtEval.config.build.wrapper;
-      checks = forAllSystems (system: {
+          checks.formatting = treefmtEval.config.build.check self;
-        formatting = (treefmtFor system).config.build.check self;
-      });
 
           # Add the Clan cli tool to the dev shell.
           # Use "nix develop" to enter the dev shell.
-      devShells = forAllSystems (system: {
+          devShells.default = pkgs.mkShell {
-        default = (pkgsFor system).mkShell {
             packages = [
-            clan-core.packages.${system}.clan-cli
+              clanSequential
-            (treefmtFor system).config.build.wrapper
+              clanCli
+              treefmtEval.config.build.wrapper
             ];
           };
-      });
+        };
     };
 }

fmt.nix

@@ -24,6 +24,7 @@
 
       # No formatter, or reformatting would corrupt them.
       "*.zone" # Knot zone files
+      "docs/book.toml" # mdBook config; no TOML formatter enabled
       "flake.lock"
       ".envrc"
       ".gitignore"

inventory.json

@@ -8,6 +8,15 @@
     },
     "ns2": {
       "installedAt": 1781418857
+    },
+    "mx1": {
+      "installedAt": 1781757322
+    },
+    "web01": {
+      "installedAt": 1781983723
+    },
+    "gw-cnx-1": {
+      "installedAt": 1785494267
     }
   }
 }

machines/control/configuration.nix

@@ -1,3 +1,7 @@
+{ config, ... }:
+let
+  hosts = import ../../modules/hosts.nix;
+in
 {
   imports = [
     ../../modules/hetzner-firewall.nix
@@ -6,19 +10,20 @@
     ../../modules/monitoring/server.nix
     ../../modules/monitoring/blackbox.nix
     ../../modules/monitoring/alerts.nix
+    ../../modules/monitoring/parsedmarc.nix
     ../../modules/docs.nix
   ];
 
   clan.core.sops.defaultGroups = [ "admins" ];
 
-  # Public IPv6; SLAAC doesn't bring it up here.
+  # Public IPv6 (from modules/hosts.nix); SLAAC doesn't bring it up here.
   cnx.staticIPv6 = {
     enable = true;
-    address = "2a01:4f9:c013:e6d0::1";
+    address = hosts.${config.networking.hostName}.ipv6;
   };
 
   time.timeZone = "Etc/GMT-3"; # UTC+3 (fixed offset, no DST)
-  services.timesyncd.enable = true;
+  services.chrony.enable = true;
 
   # Public Hetzner Cloud firewalls, synced from this config on every deploy.
   # Rules live in their own data file; see that file for the no-public-SSH note.

machines/gw-cnx-1/configuration.nix

@@ -0,0 +1,92 @@
+# Site gateway Chiang Mai (site 1): Topton 1U, Intel N300, 4x i226-V 2.5G.
+# Port roles below use the expected igc names — verify against facter.json
+# after the first install and adjust if the box enumerates differently.
+{ config, lib, ... }:
+{
+  imports = [
+    ../../modules/router
+    ../../modules/monitoring/exporters.nix
+    (import ../../modules/dns/acme-gw-secret.nix "gw-cnx-1")
+  ];
+
+  clan.core.sops.defaultGroups = [ "admins" ];
+
+  # Until the install generates facter.json (which normally provides this).
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+
+  # ZFS (disko.nix) needs a stable machine-unique hostId; derive it from the
+  # hostname so every gateway gets one for free when copied for a new site.
+  networking.hostId = builtins.substring 0 8 (
+    builtins.hashString "sha256" config.networking.hostName
+  );
+
+  cnx.router = {
+    enable = true;
+    site = "cnx";
+    siteId = 1;
+    wan.interface = "enp1s0";
+    wan.vlanId = null; # this ISP runs PPPoE untagged on the port
+    wan.macAddress = "a8:b8:e0:01:06:87";
+    trunkPorts = [
+      "enp2s0"
+      # "enp3s0" # STAGING: serves as the uplink below until cutover
+    ];
+    # Dedicated on-site recovery port: untagged mgmt, always available even
+    # if the switch config is broken.
+    accessPorts.enp4s0 = "mgmt";
+    # Replaces the newedge.house OPNsense box; renumbered to the fleet
+    # convention (10.1.<vlanId>.0/24, router .1, pool .100-.199). The old
+    # untagged LAN becomes tagged mgmt — infra switch ports get PVID 10.
+    vlans = {
+      mgmt = {
+        id = 10; # 10.1.10.0/24 — servers, APs, switches, Omada, admin
+        dhcp.reservations.storinator01 = {
+          hwAddress = "7c:c2:55:e0:d6:40";
+          ipAddress = "10.1.10.53";
+        };
+      };
+      lan.id = 20; # 10.1.20.0/24 — trusted clients
+      iot.id = 40; # 10.1.40.0/24
+      voip.id = 50; # 10.1.50.0/24
+      dmz.id = 60; # 10.1.60.0/24
+      unit1.id = 110; # 10.1.110.0/24
+      unit2.id = 120; # 10.1.120.0/24
+      unit3.id = 130; # 10.1.130.0/24
+      unit4.id = 140; # 10.1.140.0/24
+      unit5 = {
+        id = 150; # 10.1.150.0/24
+        dhcp.reservations.newt = {
+          hwAddress = "7c:d3:0a:21:58:0b";
+          ipAddress = "10.1.150.22";
+        };
+      };
+    };
+    # This site runs the Omada controller for its APs/switches.
+    omada.enable = true;
+
+    # Internal reverse proxy: real wildcard cert *.cnx1.cnx.network; Blocky
+    # resolves the names to the router's LAN address for mgmt+lan clients.
+    proxy = {
+      enable = true;
+      services.omada = {
+        # Omada's UI is HTTPS with a self-signed cert on the host network.
+        backend = "https://127.0.0.1:8043";
+        insecureSkipVerify = true;
+      };
+    };
+  };
+
+  # STAGING (remove at cutover, and restore enp3s0 to trunkPorts): DHCP-client
+  # uplink into the existing OPNsense LAN so the box has internet + mesh while
+  # it runs alongside the old router. Default-deny firewall on this interface
+  # (it's in no VLAN zone); PPPoE simply retries until the WAN port is cabled.
+  # Do NOT connect the trunk ports to the production switch while staging —
+  # Kea on tag 10 would fight the OPNsense LAN DHCP in one broadcast domain.
+  systemd.network.networks."05-staging" = {
+    matchConfig.Name = "enp3s0";
+    networkConfig.DHCP = "ipv4";
+  };
+
+  time.timeZone = "Etc/GMT-7"; # UTC+7 (Thailand, fixed offset, no DST)
+  services.chrony.enable = true;
+}

machines/gw-cnx-1/disko.nix

@@ -0,0 +1,97 @@
+{ ... }:
+let
+  hashDisk = disk: "os-${builtins.substring 0 5 (builtins.hashString "sha256" disk)}";
+  os = "/dev/disk/by-id/ata-TS256GMSA230S_J927900030";
+in
+{
+
+  boot.loader = {
+    systemd-boot = {
+      enable = true;
+    };
+    efi = {
+      canTouchEfiVariables = true;
+    };
+  };
+
+  boot.zfs.forceImportRoot = true;
+
+  disko.devices = {
+    disk = {
+      "os-${hashDisk os}" = {
+        type = "disk";
+        device = os;
+        content = {
+          type = "gpt";
+          partitions = {
+            ESP = {
+              size = "1G";
+              type = "EF00";
+              content = {
+                type = "filesystem";
+                format = "vfat";
+                mountpoint = "/boot";
+                mountOptions = [ "nofail" ];
+              };
+            };
+            system = {
+              size = "100%";
+              content = {
+                type = "zfs";
+                pool = "zroot";
+              };
+            };
+            swap = {
+              size = "16G";
+              content = {
+                type = "swap";
+              };
+            };
+          };
+        };
+      };
+    };
+    zpool = {
+      zroot = {
+        type = "zpool";
+        rootFsOptions = {
+          mountpoint = "none";
+          compression = "lz4";
+          acltype = "posixacl";
+          xattr = "sa";
+          "com.sun:auto-snapshot" = "true";
+        };
+        options.ashift = "12";
+        datasets = {
+          "root" = {
+            type = "zfs_fs";
+            options.mountpoint = "none";
+          };
+          "root/nixos" = {
+            type = "zfs_fs";
+            options.mountpoint = "/";
+            mountpoint = "/";
+          };
+          "root/home" = {
+            type = "zfs_fs";
+            options.mountpoint = "/home";
+            mountpoint = "/home";
+          };
+          "root/tmp" = {
+            type = "zfs_fs";
+            mountpoint = "/tmp";
+            options = {
+              mountpoint = "/tmp";
+              sync = "disabled";
+            };
+          };
+          "root/var" = {
+            type = "zfs_fs";
+            options.mountpoint = "/var";
+            mountpoint = "/var";
+          };
+        };
+      };
+    };
+  };
+}

machines/mx1/configuration.nix

@@ -0,0 +1,23 @@
+{ config, inputs, ... }:
+let
+  hosts = import ../../modules/hosts.nix;
+in
+{
+  imports = [
+    inputs.nixos-mailserver.nixosModules.default
+    ../../modules/mail.nix
+    ../../modules/static-ipv6.nix
+    ../../modules/monitoring/exporters.nix
+  ];
+
+  clan.core.sops.defaultGroups = [ "admins" ];
+
+  # Public IPv6 (from modules/hosts.nix); SLAAC doesn't bring it up here.
+  cnx.staticIPv6 = {
+    enable = true;
+    address = hosts.${config.networking.hostName}.ipv6;
+  };
+
+  time.timeZone = "Etc/GMT-8"; # UTC+8 (Singapore, fixed offset, no DST)
+  services.chrony.enable = true;
+}

machines/mx1/disko.nix

@@ -0,0 +1,50 @@
+# ---
+# schema = "single-disk"
+# [placeholders]
+# mainDisk = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_117494657"
+# ---
+# This file was automatically generated!
+# CHANGING this configuration requires wiping and reinstalling the machine
+{
+
+  boot.loader.grub.efiSupport = true;
+  boot.loader.grub.efiInstallAsRemovable = true;
+  boot.loader.grub.enable = true;
+  disko.devices = {
+    disk = {
+      main = {
+        name = "main-5a0919ffeb6044a39b7d44bba8895ff2";
+        device = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_117494657";
+        type = "disk";
+        content = {
+          type = "gpt";
+          partitions = {
+            "boot" = {
+              size = "1M";
+              type = "EF02"; # for grub MBR
+              priority = 1;
+            };
+            ESP = {
+              type = "EF00";
+              size = "500M";
+              content = {
+                type = "filesystem";
+                format = "vfat";
+                mountpoint = "/boot";
+                mountOptions = [ "umask=0077" ];
+              };
+            };
+            root = {
+              size = "100%";
+              content = {
+                type = "filesystem";
+                format = "ext4";
+                mountpoint = "/";
+              };
+            };
+          };
+        };
+      };
+    };
+  };
+}

machines/ns1/configuration.nix

@@ -1,11 +1,20 @@
-{ config, lib, pkgs, ... }:
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
 let
   domains = import ../../modules/dns/domains.nix;
   mesh = import ../../modules/mesh-hosts.nix { inherit config lib; };
+  hosts = import ../../modules/hosts.nix;
 in
 {
   imports = [
     ../../modules/dns/authoritative.nix
+    ../../modules/dns/acme-mx1-secret.nix
+    ../../modules/dns/acme-web01-secret.nix
+    (import ../../modules/dns/acme-gw-secret.nix "gw-cnx-1")
     ../../modules/static-ipv6.nix
     ../../modules/monitoring/exporters.nix
   ];
@@ -22,20 +31,19 @@ in
   # resolution, so map the control machine name to its ZeroTier mesh address.
   networking.hosts.${mesh.hosts.control} = [ "control" ];
 
-  # Public IPv6 (matches the ns1 AAAA glue); SLAAC doesn't bring it up here.
+  # Public IPv6 (from modules/hosts.nix; matches the ns1 AAAA glue); SLAAC
+  # doesn't bring it up here.
   cnx.staticIPv6 = {
     enable = true;
-    address = "2a01:4f8:c014:b5c5::1";
+    address = hosts.${config.networking.hostName}.ipv6;
   };
 
   time.timeZone = "Etc/GMT-1"; # UTC+1 (fixed offset, no DST)
-  services.timesyncd.enable = true;
+  services.chrony.enable = true;
 
-  # ACME DNS-01 (RFC 2136): a dedicated TSIG key, scoped to ns1 only, that an
+  # ACME DNS-01 (RFC 2136), general key. A dedicated TSIG key scoped by acl_acme
-  # external ACME client uses to write _acme-challenge TXT records. acl_acme
+  # (referenced by every zone below) to TXT updates at or under _acme-challenge.
-  # (referenced by each zone below) limits the key to TXT updates at or under
+  # Retrieve the client config with:
-  # _acme-challenge.<zone>; Knot then signs the record and transfers it to ns2,
-  # which never needs this key. Retrieve the secret for the client with:
   #   clan vars get ns1 dns-acme-tsig/acme.conf
   clan.core.vars.generators.dns-acme-tsig = {
     files."acme.conf" = {
@@ -49,8 +57,66 @@ in
     '';
   };
 
+  # ACME DNS-01, dedicated mx1 key. A *separate* TSIG key (acme_mx1) that only
+  # mx1 holds, rendered from the shared secret (generator dns-acme-mx1-secret,
+  # imported above). acl_acme_mx1 scopes it to TXT updates at exactly
+  # _acme-challenge.{mx1,mta-sts,mail} (the mail cert and its MTA-STS + client-
+  # alias SANs), and it is attached only to the cnx.email zone below — so this
+  # credential can write nothing but mx1's own cert challenges.
+  clan.core.vars.generators.dns-acme-mx1-knot = {
+    files."acme.conf" = {
+      secret = true;
+      owner = "knot";
+      group = "knot";
+    };
+    dependencies = [ "dns-acme-mx1-secret" ];
+    script = ''
+      printf 'key:\n  - id: acme_mx1\n    algorithm: hmac-sha256\n    secret: %s\n' \
+        "$(cat "$in"/dns-acme-mx1-secret/secret)" > "$out"/acme.conf
+    '';
+  };
+
+  # ACME DNS-01, dedicated web01 key. A *separate* TSIG key (acme_web01) that only
+  # web01 holds, rendered from the shared secret (generator dns-acme-web01-secret,
+  # imported above). acl_acme_web01 scopes it to TXT updates at _acme-challenge on
+  # the cnx.network zone — the owner the wildcard *.cnx.network challenge uses — so
+  # this credential can write nothing but web01's own cert challenges.
+  clan.core.vars.generators.dns-acme-web01-knot = {
+    files."acme.conf" = {
+      secret = true;
+      owner = "knot";
+      group = "knot";
+    };
+    dependencies = [ "dns-acme-web01-secret" ];
+    script = ''
+      printf 'key:\n  - id: acme_web01\n    algorithm: hmac-sha256\n    secret: %s\n' \
+        "$(cat "$in"/dns-acme-web01-secret/secret)" > "$out"/acme.conf
+    '';
+  };
+
+  # ACME DNS-01, dedicated gateway keys. Same pattern as web01: each gateway
+  # holds its own TSIG key (acme_gw_<site>_<n>), rendered from the shared
+  # secret generator imported above. acl_acme_gw_<site>_<n> scopes it to TXT
+  # updates at _acme-challenge.<site><n> on cnx.network — the owner its internal
+  # wildcard *.<site><n>.cnx.network challenge uses — and nothing else.
+  clan.core.vars.generators.dns-acme-gw-cnx-1-knot = {
+    files."acme.conf" = {
+      secret = true;
+      owner = "knot";
+      group = "knot";
+    };
+    dependencies = [ "dns-acme-gw-cnx-1-secret" ];
+    script = ''
+      printf 'key:\n  - id: acme_gw_cnx_1\n    algorithm: hmac-sha256\n    secret: %s\n' \
+        "$(cat "$in"/dns-acme-gw-cnx-1-secret/secret)" > "$out"/acme.conf
+    '';
+  };
+
   services.knot.keyFiles = [
     config.clan.core.vars.generators.dns-acme-tsig.files."acme.conf".path
+    config.clan.core.vars.generators.dns-acme-mx1-knot.files."acme.conf".path
+    config.clan.core.vars.generators.dns-acme-web01-knot.files."acme.conf".path
+    config.clan.core.vars.generators.dns-acme-gw-cnx-1-knot.files."acme.conf".path
   ];
 
   services.knot.settings.acl = [
@@ -63,6 +129,41 @@ in
       "update-owner-match" = "sub-or-equal";
       "update-owner-name" = [ "_acme-challenge" ];
     }
+    {
+      id = "acl_acme_mx1";
+      key = "acme_mx1";
+      action = [ "update" ];
+      "update-type" = [ "TXT" ];
+      "update-owner" = "name";
+      "update-owner-match" = "sub-or-equal";
+      "update-owner-name" = [
+        "_acme-challenge.mx1"
+        "_acme-challenge.mta-sts"
+        "_acme-challenge.mail"
+      ];
+    }
+    {
+      id = "acl_acme_web01";
+      key = "acme_web01";
+      action = [ "update" ];
+      "update-type" = [ "TXT" ];
+      "update-owner" = "name";
+      "update-owner-match" = "sub-or-equal";
+      # Wildcard *.cnx.network places its challenge at _acme-challenge.cnx.network,
+      # i.e. _acme-challenge at the cnx.network apex (where this acl is attached).
+      "update-owner-name" = [ "_acme-challenge" ];
+    }
+    {
+      id = "acl_acme_gw_cnx_1";
+      key = "acme_gw_cnx_1";
+      action = [ "update" ];
+      "update-type" = [ "TXT" ];
+      "update-owner" = "name";
+      "update-owner-match" = "sub-or-equal";
+      # The internal wildcard *.cnx1.cnx.network places its challenge at
+      # _acme-challenge.cnx1.cnx.network.
+      "update-owner-name" = [ "_acme-challenge.cnx1" ];
+    }
   ];
 
   # Automatic DNSSEC signing policy (primary only). ECDSA P-256/SHA-256 with
@@ -91,9 +192,16 @@ in
     "dnssec-signing" = true;
     "dnssec-policy" = "cnx";
     notify = [ "ns2" ];
+    # ns2 transfers; acme_ddns does general DNS-01 updates. The dedicated
+    # acme_mx1 key is attached only to cnx.email, so it can't touch other zones.
     acl = [
       "acl_ns2"
       "acl_acme"
-    ]; # ns2 transfers; acme_ddns key does DNS-01 updates
+    ]
+    ++ lib.optionals (d == "cnx.email") [ "acl_acme_mx1" ]
+    ++ lib.optionals (d == "cnx.network") [
+      "acl_acme_web01"
+      "acl_acme_gw_cnx_1"
+    ];
   }) domains;
 }

machines/ns2/configuration.nix

@@ -1,6 +1,7 @@
-{ ... }:
+{ config, ... }:
 let
   domains = import ../../modules/dns/domains.nix;
+  hosts = import ../../modules/hosts.nix;
 in
 {
   imports = [
@@ -11,14 +12,15 @@ in
 
   clan.core.sops.defaultGroups = [ "admins" ];
 
-  # Public IPv6 (matches the ns2 AAAA glue); SLAAC doesn't bring it up here.
+  # Public IPv6 (from modules/hosts.nix; matches the ns2 AAAA glue); SLAAC
+  # doesn't bring it up here.
   cnx.staticIPv6 = {
     enable = true;
-    address = "2a01:4f9:c014:6d87::1";
+    address = hosts.${config.networking.hostName}.ipv6;
   };
 
   time.timeZone = "Etc/GMT-3"; # UTC+3 (fixed offset, no DST)
-  services.timesyncd.enable = true;
+  services.chrony.enable = true;
 
   # ns2 = secondary (slave): pulls every zone from ns1 and accepts its NOTIFY.
   services.knot.settings.zone = map (d: {

machines/web01/configuration.nix

@@ -0,0 +1,22 @@
+{ config, ... }:
+let
+  hosts = import ../../modules/hosts.nix;
+in
+{
+  imports = [
+    ../../modules/static-ipv6.nix
+    ../../modules/monitoring/exporters.nix
+    ../../modules/web-proxy.nix
+  ];
+
+  clan.core.sops.defaultGroups = [ "admins" ];
+
+  # Public IPv6 (from modules/hosts.nix); SLAAC doesn't bring it up here.
+  cnx.staticIPv6 = {
+    enable = true;
+    address = hosts.${config.networking.hostName}.ipv6;
+  };
+
+  time.timeZone = "Etc/GMT-8"; # UTC+8 (Singapore, fixed offset, no DST)
+  services.chrony.enable = true;
+}

machines/web01/disko.nix

@@ -0,0 +1,50 @@
+# ---
+# schema = "single-disk"
+# [placeholders]
+# mainDisk = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_108706511"
+# ---
+# This file was automatically generated!
+# CHANGING this configuration requires wiping and reinstalling the machine
+{
+
+  boot.loader.grub.efiSupport = true;
+  boot.loader.grub.efiInstallAsRemovable = true;
+  boot.loader.grub.enable = true;
+  disko.devices = {
+    disk = {
+      main = {
+        name = "main-ddd46ebf135244608078712d6ec76691";
+        device = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_108706511";
+        type = "disk";
+        content = {
+          type = "gpt";
+          partitions = {
+            "boot" = {
+              size = "1M";
+              type = "EF02"; # for grub MBR
+              priority = 1;
+            };
+            ESP = {
+              type = "EF00";
+              size = "500M";
+              content = {
+                type = "filesystem";
+                format = "vfat";
+                mountpoint = "/boot";
+                mountOptions = [ "umask=0077" ];
+              };
+            };
+            root = {
+              size = "100%";
+              content = {
+                type = "filesystem";
+                format = "ext4";
+                mountpoint = "/";
+              };
+            };
+          };
+        };
+      };
+    };
+  };
+}

modules/dns/acme-gw-secret.nix

@@ -0,0 +1,17 @@
+# Shared TSIG secret for a gateway's dedicated ACME key (function: machine
+# name -> module). The acme_gw_<x> key lets that gateway — and only it — write
+# _acme-challenge.<label> TXT records on ns1 to obtain its internal wildcard
+# cert via DNS-01. ns1 scopes it with a matching acl on the cnx.network zone.
+# Import on BOTH ns1 and the gateway machine, applied with the machine name:
+#   (import ../../modules/dns/acme-gw-secret.nix "gw-cnx-1")
+machine:
+{ pkgs, ... }:
+{
+  clan.core.vars.generators."dns-acme-${machine}-secret" = {
+    share = true;
+    files."secret".secret = true;
+    runtimeInputs = [ pkgs.openssl ];
+    # 32 random bytes, base64 — a valid hmac-sha256 TSIG secret.
+    script = ''openssl rand -base64 32 | tr -d '\n' > "$out"/secret'';
+  };
+}

modules/dns/acme-mx1-secret.nix

@@ -0,0 +1,19 @@
+# Shared TSIG secret for the dedicated acme_mx1 key.
+#
+# This key lets mx1 — and only mx1 — write _acme-challenge.mx1.cnx.email TXT
+# records on ns1 to obtain its mail TLS cert via ACME DNS-01. ns1 scopes it with
+# acl_acme_mx1 (attached only to the cnx.email zone) so the credential can touch
+# nothing else. ns1 renders this secret into a Knot key file; mx1 into a lego
+# rfc2136 env file; both must carry the same secret, hence one shared generator
+# with a per-host renderer that depends on it. Imported by ns1 and (via mail.nix)
+# mx1.
+{ pkgs, ... }:
+{
+  clan.core.vars.generators.dns-acme-mx1-secret = {
+    share = true;
+    files."secret".secret = true;
+    runtimeInputs = [ pkgs.openssl ];
+    # 32 random bytes, base64 — a valid hmac-sha256 TSIG secret.
+    script = ''openssl rand -base64 32 | tr -d '\n' > "$out"/secret'';
+  };
+}

modules/dns/acme-web01-secret.nix

@@ -0,0 +1,19 @@
+# Shared TSIG secret for the dedicated acme_web01 key.
+#
+# This key lets web01 — and only web01 — write _acme-challenge.cnx.network TXT
+# records on ns1 to obtain its wildcard (*.cnx.network) TLS cert via ACME DNS-01.
+# ns1 scopes it with acl_acme_web01 (attached only to the cnx.network zone) so the
+# credential can touch nothing else. ns1 renders this secret into a Knot key file;
+# web01 into a lego rfc2136 env file; both must carry the same secret, hence one
+# shared generator with a per-host renderer that depends on it. Imported by ns1
+# and (via web-proxy.nix) web01.
+{ pkgs, ... }:
+{
+  clan.core.vars.generators.dns-acme-web01-secret = {
+    share = true;
+    files."secret".secret = true;
+    runtimeInputs = [ pkgs.openssl ];
+    # 32 random bytes, base64 — a valid hmac-sha256 TSIG secret.
+    script = ''openssl rand -base64 32 | tr -d '\n' > "$out"/secret'';
+  };
+}

modules/dns/authoritative.nix

@@ -1,4 +1,9 @@
-{ config, lib, pkgs, ... }:
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
 let
   # ZeroTier addresses — zone transfers run over the mesh, not the public net.
   mesh = import ../mesh-hosts.nix { inherit config lib; };
@@ -27,7 +32,7 @@ in
 
   # knot binds 0.0.0.0@53 and ::@53, so free port 53 by disabling the
   # systemd-resolved stub listener. Resolution still works via nss-resolve.
-  services.resolved.extraConfig = "DNSStubListener=no";
+  services.resolved.settings.Resolve.DNSStubListener = "no";
 
   services.knot = {
     enable = true;

modules/dns/domains.nix

@@ -4,4 +4,5 @@
   "cnx.network"
   "buildfor.life"
   "cnx.email"
+  "newedge.house"
 ]

modules/dns/zones/cnx.email.zone

@@ -11,8 +11,36 @@ $TTL 3600
 @       IN  NS      ns1.cnx.network.
 @       IN  NS      ns2.cnx.network.
 
-; ---- Mail (fill in once the mail host exists) ----
+; ---- Mail ----
-;@      IN  MX 10   mail.cnx.email.
+mx1     IN  A       5.223.65.38
-;mail   IN  A       <mail-ipv4>
+mx1     IN  AAAA    2a01:4ff:2f0:1963::1
-;@      IN  TXT     "v=spf1 mx -all"
+; Client-facing alias for IMAP/submission (Thunderbird etc.); the cert carries
-;_dmarc IN  TXT     "v=DMARC1; p=quarantine; rua=mailto:postmaster@cnx.email"
+; mail.cnx.email as a SAN. The MX must never point here (CNAMEs are illegal MX
+; targets) — server-to-server delivery and DANE stay on mx1.cnx.email.
+mail    IN  CNAME   mx1.cnx.email.
+@       IN  MX 10   mx1.cnx.email.
+@       IN  TXT     "v=spf1 mx -all"
+; Aggregate (rua) + forensic (ruf) reports go to the dmarc@cnx.email mailbox,
+; which parsedmarc on control polls and feeds into Grafana. fo=1 asks reporters
+; to send a forensic report on any SPF/DKIM failure.
+_dmarc  IN  TXT     "v=DMARC1; p=quarantine; rua=mailto:dmarc@cnx.email; ruf=mailto:dmarc@cnx.email; fo=1"
+
+; ---- DANE / TLSA ----
+; "3 1 1" = DANE-EE, SPKI, SHA-256: the digest of mx1's certificate public key.
+; Valid because the zone is DNSSEC-signed and the lego cert uses --reuse-key, so
+; the key (and thus this digest) is stable across renewals. Compute it AFTER the
+; first issuance and paste the hex below:
+;   ssh mx1 'openssl x509 -in /var/lib/acme/mx1.cnx.email/cert.pem -noout -pubkey \
+;     | openssl pkey -pubin -outform DER | openssl dgst -sha256 -binary | xxd -p -c256'
+_25._tcp.mx1  IN  TLSA  3 1 1  bd9a51f60b6d2dd20f18b3553d2795053ac52f87567a46bc892006bb58506404
+
+; ---- MTA-STS ----
+; Policy host (A/AAAA point at mx1); the _mta-sts TXT id MUST be bumped whenever
+; the policy file in modules/mail.nix changes, or senders keep the cached policy.
+mta-sts   IN  A       5.223.65.38
+mta-sts   IN  AAAA    2a01:4ff:2f0:1963::1
+_mta-sts  IN  TXT     "v=STSv1; id=2026061801"
+mail._domainkey IN TXT ( "v=DKIM1; k=rsa; "
+        "p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAr9QxTs5dLtY76bo156+Tp0GUoE554rMwIooIYa2MMYHNs8zPb0thFmaCKGAINdHKNIq2phXAlk51iBTfdqXjx7gVWSrs+ftykqO3b5hUjgImsgqPWGUTzy5/bUgcDELiD9KKEyKYD3+ebZEw6d0uvBvEsA6a1CPzOsufoCDtyKjByCuQzkCBrK25TUHFolGvEYcZexR0LSF+8hMss"
+        "xyw9NYiPpTXVCWQJnrZZpuOBiX0K2l5CAXVyuT/B5RcBXlAUhBTp3390VEhL0wAZMTOnvtvBYK3NnsTIh96fkh6MfWmre7Fi9hEq//xGf40N5/aomMjJrJdqFZJLZpDotb/XwIDAQAB"
+        )

modules/dns/zones/cnx.network.zone

@@ -25,3 +25,10 @@ control IN  AAAA    fd06:1bad:ece2:92ad:ba99:9306:1bad:ece2
 ;@      IN  A       <web-ipv4>
 ;www    IN  CNAME   cnx.network.
 monitor IN  A       5.223.66.36
+
+; ---- web01 (public reverse proxy / TLS termination) ----
+; Serves a wildcard *.cnx.network TLS cert (ACME DNS-01) and forwards to internal
+; services over the mesh. Add a vhost in modules/web-proxy.nix and a CNAME here.
+web01   IN  A       5.223.55.246
+web01   IN  AAAA    2a01:4ff:2f0:2d8f::1
+grafana IN  CNAME   web01.cnx.network.

modules/dns/zones/newedge.house.zone

@@ -0,0 +1,13 @@
+$ORIGIN newedge.house.
+$TTL 3600
+
+@   IN  SOA ns1.cnx.network. hostmaster.cnx.network. (
+        2026061401  ; serial   (ignored: Knot auto-assigns a dateserial on signing)
+        3600        ; refresh
+        900         ; retry
+        604800      ; expire
+        300 )       ; negative-cache TTL
+
+; Served by the same nameservers (out-of-bailiwick, no glue needed here).
+@       IN  NS      ns1.cnx.network.
+@       IN  NS      ns2.cnx.network.

modules/docs.nix

@@ -30,6 +30,12 @@ in
     enable = true;
     virtualHosts.":${toString port}".extraConfig = ''
       root * ${site}
+      # mdBook doesn't fingerprint asset filenames, and every file in the Nix
+      # store carries an epoch (1970) mtime, so file_server's only validator never
+      # changes across redeploys — conditional requests would 304 forever and pin
+      # browsers to stale docs. no-store sidesteps caching entirely; pages are tiny
+      # and mesh-only, so refetching on each load is free.
+      header Cache-Control "no-store"
       file_server
     '';
   };

modules/hetzner-firewall-rules.nix

@@ -24,6 +24,46 @@ let
     description = "ICMP (ping / PMTUD)";
   };
 
+  # Public mail ports for mx1 (MX for cnx.email). 25 is server-to-server
+  # delivery; 587/465 are client submission; 143/993 are IMAP. 443 serves only the
+  # MTA-STS policy (https://mta-sts.cnx.email/.well-known/mta-sts.txt); the cert
+  # itself uses ACME DNS-01 so port 80 stays closed. Admin still rides the mesh.
+  mailPort = port: description: {
+    direction = "in";
+    protocol = "tcp";
+    inherit port;
+    source_ips = world;
+    inherit description;
+  };
+  mailRules = [
+    (mailPort "25" "SMTP (inbound mail)")
+    (mailPort "587" "Submission (STARTTLS)")
+    (mailPort "465" "Submission (implicit TLS)")
+    (mailPort "143" "IMAP (STARTTLS)")
+    (mailPort "993" "IMAP (implicit TLS)")
+    (mailPort "443" "MTA-STS policy (HTTPS)")
+  ];
+
+  # web01 is a public reverse proxy with TLS termination. 443 serves the proxy;
+  # 80 only carries Caddy's HTTP->HTTPS redirect (the cert uses ACME DNS-01, not
+  # HTTP-01). Admin rides the mesh.
+  webRules = [
+    {
+      direction = "in";
+      protocol = "tcp";
+      port = "80";
+      source_ips = world;
+      description = "HTTP (redirect to HTTPS)";
+    }
+    {
+      direction = "in";
+      protocol = "tcp";
+      port = "443";
+      source_ips = world;
+      description = "HTTPS (reverse proxy / TLS termination)";
+    }
+  ];
+
   dnsRules = [
     {
       direction = "in";
@@ -50,4 +90,12 @@ in
   ];
   "clan-ns1" = dnsRules;
   "clan-ns2" = dnsRules;
+  "clan-mx1" = mailRules ++ [
+    zerotier
+    ping
+  ];
+  "clan-web01" = webRules ++ [
+    zerotier
+    ping
+  ];
 }

modules/hosts.nix

@@ -0,0 +1,32 @@
+# Per-host public network facts: single source of truth for each machine's
+# public IPv4 and its static public IPv6. Consumed by clan.nix's `internet`
+# connection hosts (ipv4) and each machine's `cnx.staticIPv6` (ipv6), so an
+# address is written once instead of being duplicated across configs.
+#
+# NOT a driver for the DNS zone files — those stay hand-edited text, so a record
+# here that also appears as A/AAAA glue still needs a matching manual zone edit.
+#
+# ipv6 is the single address to assign from the host's allocated /64 (we take
+# ::1), without prefix length; cnx.staticIPv6 supplies the /64 default.
+{
+  control = {
+    ipv4 = "77.42.68.181";
+    ipv6 = "2a01:4f9:c013:e6d0::1";
+  };
+  ns1 = {
+    ipv4 = "46.224.170.206";
+    ipv6 = "2a01:4f8:c014:b5c5::1";
+  };
+  ns2 = {
+    ipv4 = "157.180.70.82";
+    ipv6 = "2a01:4f9:c014:6d87::1";
+  };
+  mx1 = {
+    ipv4 = "5.223.65.38";
+    ipv6 = "2a01:4ff:2f0:1963::1";
+  };
+  web01 = {
+    ipv4 = "5.223.55.246";
+    ipv6 = "2a01:4ff:2f0:2d8f::1";
+  };
+}

modules/mail-dmarc-cred.nix

@@ -0,0 +1,26 @@
+# Shared credential for the dmarc@cnx.email mailbox.
+#
+# DMARC aggregate/forensic reports are delivered to dmarc@cnx.email on mx1;
+# parsedmarc on control fetches them over IMAPS across the mesh and needs the
+# *plaintext* passphrase, while mx1's mailserver only needs the sha-512 hash.
+# clan vars secrets are per-machine, so this generator is shared (share = true)
+# to make the same value available on both hosts. Files are root-owned: SNM reads
+# the hash as root, and parsedmarc's ExecStartPre reads the passphrase as root.
+# Imported by mx1 (via mail.nix) and control (via monitoring/parsedmarc.nix).
+{ pkgs, ... }:
+{
+  clan.core.vars.generators.mail-dmarc-cred = {
+    share = true;
+    files."passphrase".secret = true; # read by parsedmarc on control
+    files."hash".secret = true; # consumed by the mailserver on mx1
+    runtimeInputs = [
+      pkgs.xkcdpass
+      pkgs.mkpasswd
+    ];
+    script = ''
+      pass="$(xkcdpass --numwords=4 --delimiter=- --case=lower)-$((RANDOM % 90 + 10))"
+      printf '%s' "$pass" > "$out"/passphrase
+      printf '%s' "$pass" | mkpasswd -s -m sha-512 > "$out"/hash
+    '';
+  };
+}

modules/mail.nix

@@ -0,0 +1,163 @@
+# Declarative mail stack for mx1 (Simple NixOS Mailserver: Postfix + Dovecot +
+# Rspamd + OpenDKIM). Imported by machines/mx1 alongside the SNM flake module.
+#
+# Mailboxes are virtual (not system users): each address below is a login account
+# whose password is auto-generated by a clan vars generator as a four-word
+# passphrase with a trailing number (e.g. otter-lantern-cobalt-driftwood-42). The
+# generator stores both the passphrase and its sha-512 hash. To add a mailbox:
+# append the address to `accounts`, run `clan vars generate mx1`, redeploy mx1,
+# then hand the passphrase to the user:
+#   clan vars get mx1 mail-passwd-<addr>/passphrase
+# (addr with @ and . replaced by -at- and -, e.g. mail-passwd-postmaster-at-cnx-email)
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  hosts = import ./hosts.nix;
+  fqdn = "mx1.cnx.email";
+  mtaStsHost = "mta-sts.cnx.email";
+  # Client-facing alias (CNAME -> mx1) so Thunderbird etc. can use mail.cnx.email
+  # for submission/IMAP; added as a cert SAN so TLS validates against that name.
+  clientHost = "mail.cnx.email";
+
+  # MTA-STS policy served at https://mta-sts.cnx.email/.well-known/mta-sts.txt.
+  # enforce = a sending MTA that fetched this must use a valid, MX-matching TLS
+  # cert or refuse to deliver. Bump the _mta-sts TXT id (in the zone) whenever
+  # this changes.
+  mtaStsPolicy = pkgs.writeText "mta-sts.txt" ''
+    version: STSv1
+    mode: enforce
+    mx: ${fqdn}
+    max_age: 604800
+  '';
+
+  # The mailboxes mx1 serves. postmaster is required by RFC 5321.
+  accounts = [
+    "postmaster@cnx.email"
+  ];
+
+  genName = addr: "mail-passwd-" + lib.replaceStrings [ "@" "." ] [ "-at-" "-" ] addr;
+
+  passwdGenerators = lib.listToAttrs (
+    map (addr: {
+      name = genName addr;
+      value = {
+        files."passphrase".secret = true; # retrievable to hand to the user
+        files."hash".secret = true; # consumed by SNM's hashedPasswordFile
+        runtimeInputs = [
+          pkgs.xkcdpass
+          pkgs.mkpasswd
+        ];
+        script = ''
+          pass="$(xkcdpass --numwords=4 --delimiter=- --case=lower)-$((RANDOM % 90 + 10))"
+          printf '%s' "$pass" > "$out"/passphrase
+          printf '%s' "$pass" | mkpasswd -s -m sha-512 > "$out"/hash
+        '';
+      };
+    }) accounts
+  );
+
+  loginAccounts =
+    lib.listToAttrs (
+      map (addr: {
+        name = addr;
+        value.hashedPasswordFile = config.clan.core.vars.generators.${genName addr}.files."hash".path;
+      }) accounts
+    )
+    // {
+      # DMARC report inbox (rua/ruf target in the cnx.email zone). Its password
+      # comes from the *shared* mail-dmarc-cred generator instead of the per-machine
+      # set above, so parsedmarc on control can read the same passphrase over the
+      # mesh. Retrieve it with: clan vars get mx1 mail-dmarc-cred/passphrase
+      "dmarc@cnx.email".hashedPasswordFile =
+        config.clan.core.vars.generators.mail-dmarc-cred.files."hash".path;
+    };
+in
+{
+  imports = [
+    ./dns/acme-mx1-secret.nix
+    ./mail-dmarc-cred.nix
+  ];
+
+  clan.core.vars.generators = passwdGenerators // {
+    # Render the shared acme_mx1 TSIG secret into a lego rfc2136 env file. lego
+    # (via security.acme below) uses it to write the _acme-challenge.mx1.cnx.email
+    # TXT record to ns1, which authorizes the acme_mx1 key for exactly that owner.
+    dns-acme-rfc2136 = {
+      files."rfc2136.env".secret = true; # root-owned; systemd reads it as root
+      dependencies = [ "dns-acme-mx1-secret" ];
+      script = ''
+        printf 'RFC2136_NAMESERVER=${hosts.ns1.ipv4}:53\nRFC2136_TSIG_ALGORITHM=hmac-sha256.\nRFC2136_TSIG_KEY=acme_mx1\nRFC2136_TSIG_SECRET=%s\n' \
+          "$(cat "$in"/dns-acme-mx1-secret/secret)" > "$out"/rfc2136.env
+      '';
+    };
+  };
+
+  mailserver = {
+    enable = true;
+    # Fresh install: declare the latest layout the nixos-26.05 branch ships (3),
+    # so SNM uses the current dovecot mail directory layout with nothing to migrate.
+    stateVersion = 3;
+    inherit fqdn;
+    domains = [ "cnx.email" ];
+    accounts = loginAccounts;
+
+    # Consume a security.acme cert we obtain ourselves via DNS-01 (below); no
+    # web server and no inbound HTTP needed, so port 80 stays closed. The extra
+    # SAN hosts (MTA-STS, client alias) are attached to that cert via
+    # security.acme.certs.${fqdn}.extraDomainNames below.
+    x509.useACMEHost = fqdn;
+
+    dkim.defaults.selector = "mail";
+  };
+
+  security.acme = {
+    acceptTerms = true;
+    defaults.email = "postmaster@cnx.email";
+    certs.${fqdn} = {
+      # The MTA-STS endpoint and client-facing alias ride this one cert as SANs.
+      extraDomainNames = [
+        mtaStsHost
+        clientHost
+      ];
+      dnsProvider = "rfc2136";
+      environmentFile = config.clan.core.vars.generators.dns-acme-rfc2136.files."rfc2136.env".path;
+      # ns1 is the only nameserver that accepts the acme_mx1 UPDATE; check
+      # propagation against it directly rather than a public resolver.
+      dnsResolver = "${hosts.ns1.ipv4}:53";
+      # Keep the private key fixed across renewals so the DANE TLSA "3 1 1"
+      # record (public-key digest, published in the zone) stays valid.
+      extraLegoRenewFlags = [ "--reuse-key" ];
+      # Caddy serves the MTA-STS endpoint from explicit cert file paths, so it
+      # won't notice a renewal on its own — reload it whenever the cert changes.
+      # (Merges with the postfix/dovecot reloads SNM wires up for this cert.)
+      reloadServices = [ "caddy.service" ];
+    };
+  };
+
+  # The mail cert is owned group=acme (SNM adds postfix/dovecot); Caddy serves the
+  # MTA-STS endpoint from the same cert, so it needs to read the key too.
+  users.users.caddy.extraGroups = [ "acme" ];
+
+  # MTA-STS policy endpoint, served by Caddy (same web server as control's docs).
+  # The explicit `tls cert key` points at the lego-issued mail cert (which carries
+  # mta-sts.cnx.email as a SAN) and disables Caddy's automatic ACME, so no extra
+  # issuance happens and the DANE TLSA key stays stable. Only :443 is opened.
+  services.caddy = {
+    enable = true;
+    virtualHosts.${mtaStsHost}.extraConfig = ''
+      tls /var/lib/acme/${fqdn}/cert.pem /var/lib/acme/${fqdn}/key.pem
+      root * ${pkgs.writeTextDir ".well-known/mta-sts.txt" (builtins.readFile mtaStsPolicy)}
+      file_server
+    '';
+  };
+
+  # DKIM private keys are generated on first start under this dir. They're
+  # regenerable (rotate + republish the TXT), but declaring the path as clan
+  # state lets a borg client back it up to avoid a needless DNS round-trip on
+  # restore. Wiring mx1 into the borgbackup instance is a separate step.
+  clan.core.state.mail-dkim.folders = [ config.mailserver.dkimKeyDirectory ];
+}

modules/mesh-hosts.nix

@@ -2,25 +2,35 @@
 # DNS zone transfers and metrics scraping ride this mesh, never the public net.
 #
 # Rather than hardcoding the addresses, we read them from the public clan vars
-# that clan-core's zerotier generator already writes per machine
+# that clan-core's zerotier service writes. As of clan-core 26.05 these are
-# (vars/per-machine/<m>/zerotier/zerotier-ip/value). This keeps the mesh map in
+# shared, instance-scoped generators: the per-machine IP lives at
+# vars/shared/zerotier-ip-<machine>-<instance>/ip and the network id at
+# vars/shared/zerotier-network-<instance>/network-id (instance = "zerotier",
+# the inventory.instances.zerotier name in clan.nix). This keeps the mesh map in
 # lockstep with the actual identities: regenerate or re-key a node and its
 # address here follows automatically. Call as: import ../mesh-hosts.nix { inherit config lib; }.
 { config, lib }:
 let
   dir = config.clan.core.settings.directory;
+  instance = "zerotier";
 
-  readVar =
+  readIp =
-    machine: file:
+    machine: builtins.readFile "${dir}/vars/shared/zerotier-ip-${machine}-${instance}/ip/value";
-    builtins.readFile "${dir}/vars/per-machine/${machine}/zerotier/${file}/value";
 
-  hosts = lib.genAttrs [ "control" "ns1" "ns2" ] (m: readVar m "zerotier-ip");
+  hosts = lib.genAttrs [
+    "control"
+    "ns1"
+    "ns2"
+    "mx1"
+    "web01"
+    "gw-cnx-1"
+  ] readIp;
 
   # RFC 4193 prefix of this ZeroTier network: fd + the 8-byte network id + the
-  # 0x9993 marker. The network id is a public var on the controller (control).
+  # 0x9993 marker. The network id is a public, shared var for the instance.
   # The /88 (11 bytes) covers fd + network id + 0x99 + 0x93, i.e. every mesh peer,
   # and is used to scope mesh-only firewall rules.
-  networkId = readVar "control" "zerotier-network-id";
+  networkId = builtins.readFile "${dir}/vars/shared/zerotier-network-${instance}/network-id/value";
   full = "fd" + networkId + "9993"; # 22 hex chars = 11 bytes
   hextet = i: builtins.substring (i * 4) 4 full;
   subnet = "${hextet 0}:${hextet 1}:${hextet 2}:${hextet 3}:${hextet 4}:${builtins.substring 20 2 full}00::/88";

modules/monitoring/alerts.nix

@@ -59,6 +59,31 @@ in
           }
         ];
       }
+      {
+        # WAN quality on the site gateways, fed by the hourly speedtest timer
+        # (modules/router/speedtest.nix) via the node_exporter textfile
+        # collector. Degradation is judged against the link's own 7-day median,
+        # so no per-site/per-ISP-plan threshold needs maintaining.
+        name = "wan";
+        rules = [
+          {
+            alert = "WANSpeedDegraded";
+            expr = "speedtest_download_mbps < 0.5 * quantile_over_time(0.5, speedtest_download_mbps[7d])";
+            for = "3h";
+            labels.severity = "warning";
+            annotations.summary = "WAN download on {{ $labels.instance }} is below half its 7-day median";
+            annotations.description = "Sustained for 3h (several test runs) — likely ISP degradation, line fault, or saturation. Compare speedtest_upload_mbps/speedtest_ping_ms and run `librespeed-cli` manually on the gateway.";
+          }
+          {
+            alert = "WANSpeedTestFailing";
+            expr = "speedtest_success == 0";
+            for = "3h";
+            labels.severity = "warning";
+            annotations.summary = "Speed tests on {{ $labels.instance }} have been failing for 3h";
+            annotations.description = "librespeed-cli cannot complete a test while other traffic may still flow; check `journalctl -u speedtest` on the gateway.";
+          }
+        ];
+      }
       {
         name = "backup";
         rules = [

modules/monitoring/dashboards/backups.json

@@ -26,7 +26,10 @@
       "fieldConfig": {
         "defaults": {
           "color": { "mode": "thresholds" },
-          "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "green", "value": null }]
+          },
           "noValue": "no data",
           "mappings": [
             {
@@ -41,7 +44,11 @@
         "overrides": []
       },
       "options": {
-        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
         "colorMode": "background",
         "graphMode": "none",
         "textMode": "auto",
@@ -72,7 +79,11 @@
         "overrides": []
       },
       "options": {
-        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
         "colorMode": "none",
         "graphMode": "none",
         "textMode": "auto",
@@ -110,7 +121,11 @@
         "overrides": []
       },
       "options": {
-        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
         "colorMode": "background",
         "graphMode": "none",
         "textMode": "auto",
@@ -168,7 +183,10 @@
       "id": 6,
       "datasource": { "type": "prometheus", "uid": "victoriametrics" },
       "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
-      "fieldConfig": { "defaults": { "unit": "short", "min": 0, "max": 1 }, "overrides": [] },
+      "fieldConfig": {
+        "defaults": { "unit": "short", "min": 0, "max": 1 },
+        "overrides": []
+      },
       "targets": [
         {
           "refId": "A",

modules/monitoring/dashboards/dns.json

@@ -224,7 +224,10 @@
       "options": { "showHeader": true },
       "fieldConfig": {
         "defaults": {
-          "custom": { "align": "auto", "cellOptions": { "type": "color-background" } },
+          "custom": {
+            "align": "auto",
+            "cellOptions": { "type": "color-background" }
+          },
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -245,15 +248,21 @@
         "overrides": [
           {
             "matcher": { "id": "byName", "options": "zone" },
-            "properties": [{ "id": "custom.cellOptions", "value": { "type": "auto" } }]
+            "properties": [
+              { "id": "custom.cellOptions", "value": { "type": "auto" } }
+            ]
           },
           {
             "matcher": { "id": "byName", "options": "query" },
-            "properties": [{ "id": "custom.cellOptions", "value": { "type": "auto" } }]
+            "properties": [
+              { "id": "custom.cellOptions", "value": { "type": "auto" } }
+            ]
           },
           {
             "matcher": { "id": "byName", "options": "instance" },
-            "properties": [{ "id": "custom.cellOptions", "value": { "type": "auto" } }]
+            "properties": [
+              { "id": "custom.cellOptions", "value": { "type": "auto" } }
+            ]
           }
         ]
       },
@@ -283,7 +292,10 @@
       "id": 22,
       "datasource": { "type": "prometheus", "uid": "victoriametrics" },
       "gridPos": { "h": 8, "w": 12, "x": 12, "y": 43 },
-      "fieldConfig": { "defaults": { "unit": "short", "min": 0, "max": 1 }, "overrides": [] },
+      "fieldConfig": {
+        "defaults": { "unit": "short", "min": 0, "max": 1 },
+        "overrides": []
+      },
       "targets": [
         {
           "refId": "A",

modules/monitoring/dashboards/uptime.json

@@ -0,0 +1,194 @@
+{
+  "uid": "cnx-uptime",
+  "title": "CNX Uptime",
+  "tags": ["uptime", "availability", "cnx"],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "version": 1,
+  "refresh": "30s",
+  "time": { "from": "now-24h", "to": "now" },
+  "templating": { "list": [] },
+  "annotations": { "list": [] },
+  "panels": [
+    {
+      "type": "row",
+      "title": "Uptime",
+      "id": 1,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }
+    },
+    {
+      "type": "stat",
+      "title": "Host status",
+      "description": "Whether VictoriaMetrics is currently able to scrape each host's node_exporter. UP means the host (and its mesh path) is reachable; DOWN means the scrape failed. One tile per machine.",
+      "id": 2,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 6, "w": 12, "x": 0, "y": 1 },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "thresholds" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "green", "value": null }]
+          },
+          "noValue": "no data",
+          "mappings": [
+            {
+              "type": "value",
+              "options": {
+                "0": { "text": "DOWN", "color": "red", "index": 0 },
+                "1": { "text": "UP", "color": "green", "index": 1 }
+              }
+            }
+          ]
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "value_and_name",
+        "orientation": "auto"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "up{job=\"node\"}",
+          "legendFormat": "{{instance}}",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "type": "stat",
+      "title": "Current uptime",
+      "description": "Time since each host last booted (now - node_boot_time_seconds). A value that drops back to near zero means the host rebooted.",
+      "id": 3,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 6, "w": 12, "x": 12, "y": 1 },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "dtdurations",
+          "color": { "mode": "fixed", "fixedColor": "text" },
+          "noValue": "no data"
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "none",
+        "graphMode": "none",
+        "textMode": "value_and_name",
+        "orientation": "auto"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "time() - node_boot_time_seconds{job=\"node\"}",
+          "legendFormat": "{{instance}}",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "type": "bargauge",
+      "title": "Availability over window",
+      "description": "Fraction of successful scrapes over the selected time range, per host (avg of up over $__range). 100% means every scrape in the window succeeded; dips reveal flapping or outages. Red below 99%.",
+      "id": 4,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 7 },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "color": { "mode": "thresholds" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "red", "value": null },
+              { "color": "yellow", "value": 99 },
+              { "color": "green", "value": 99.9 }
+            ]
+          },
+          "noValue": "no data"
+        },
+        "overrides": []
+      },
+      "options": {
+        "displayMode": "gradient",
+        "orientation": "horizontal",
+        "showUnfilled": true,
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        }
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "avg_over_time(up{job=\"node\"}[$__range]) * 100",
+          "legendFormat": "{{instance}}",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Uptime over time",
+      "description": "Host uptime across the window. The line should climb steadily; a reset to zero marks a reboot.",
+      "id": 5,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 7 },
+      "fieldConfig": {
+        "defaults": { "unit": "s", "custom": { "fillOpacity": 0 } },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "time() - node_boot_time_seconds{job=\"node\"}",
+          "legendFormat": "{{instance}}"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Up/down history",
+      "description": "1 while a host's node_exporter was scrapeable, 0 while it was not. Gaps to zero are outages or lost mesh connectivity.",
+      "id": 6,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 6, "w": 24, "x": 0, "y": 15 },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "min": 0,
+          "max": 1,
+          "custom": { "fillOpacity": 20, "lineInterpolation": "stepAfter" }
+        },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "up{job=\"node\"}",
+          "legendFormat": "{{instance}}"
+        }
+      ]
+    }
+  ]
+}

modules/monitoring/dashboards/wan.json

@@ -0,0 +1,267 @@
+{
+  "uid": "cnx-wan",
+  "title": "CNX WAN Speed",
+  "tags": ["wan", "speedtest", "gateway", "cnx"],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "version": 1,
+  "refresh": "5m",
+  "time": { "from": "now-7d", "to": "now" },
+  "templating": { "list": [] },
+  "annotations": { "list": [] },
+  "panels": [
+    {
+      "type": "row",
+      "title": "WAN speed (site gateways)",
+      "id": 1,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }
+    },
+    {
+      "type": "stat",
+      "title": "Download (latest)",
+      "description": "Most recent hourly librespeed result per gateway.",
+      "id": 2,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 5, "w": 6, "x": 0, "y": 1 },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bps",
+          "color": { "mode": "fixed", "fixedColor": "green" },
+          "noValue": "no data"
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "value",
+        "graphMode": "area",
+        "textMode": "value_and_name",
+        "orientation": "auto"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "speedtest_download_mbps * 1e6",
+          "legendFormat": "{{instance}}",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "type": "stat",
+      "title": "Upload (latest)",
+      "description": "Most recent hourly librespeed result per gateway.",
+      "id": 3,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 5, "w": 6, "x": 6, "y": 1 },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bps",
+          "color": { "mode": "fixed", "fixedColor": "blue" },
+          "noValue": "no data"
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "value",
+        "graphMode": "area",
+        "textMode": "value_and_name",
+        "orientation": "auto"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "speedtest_upload_mbps * 1e6",
+          "legendFormat": "{{instance}}",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "type": "stat",
+      "title": "Ping (latest)",
+      "description": "Latency to the librespeed test server. Yellow over 50 ms, red over 100 ms.",
+      "id": 4,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 5, "w": 6, "x": 12, "y": 1 },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ms",
+          "color": { "mode": "thresholds" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 50 },
+              { "color": "red", "value": 100 }
+            ]
+          },
+          "noValue": "no data"
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "value",
+        "graphMode": "none",
+        "textMode": "value_and_name",
+        "orientation": "auto"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "speedtest_ping_ms",
+          "legendFormat": "{{instance}}",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "type": "stat",
+      "title": "Test status",
+      "description": "Whether the last speed test on each gateway completed. FAILING also fires the WANSpeedTestFailing alert after 3h.",
+      "id": 5,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 5, "w": 6, "x": 18, "y": 1 },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "thresholds" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "green", "value": null }]
+          },
+          "noValue": "no data",
+          "mappings": [
+            {
+              "type": "value",
+              "options": {
+                "0": { "text": "FAILING", "color": "red", "index": 0 },
+                "1": { "text": "OK", "color": "green", "index": 1 }
+              }
+            }
+          ]
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "value_and_name",
+        "orientation": "auto"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "speedtest_success",
+          "legendFormat": "{{instance}}",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Download",
+      "description": "Hourly download rate per gateway, with each link's own 7-day median as reference — the WANSpeedDegraded alert fires when a link stays below half its median for 3h.",
+      "id": 6,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 9, "w": 12, "x": 0, "y": 6 },
+      "fieldConfig": {
+        "defaults": { "unit": "bps", "min": 0 },
+        "overrides": [
+          {
+            "matcher": { "id": "byRegexp", "options": ".*7d median.*" },
+            "properties": [
+              {
+                "id": "custom.lineStyle",
+                "value": { "fill": "dash", "dash": [10, 10] }
+              },
+              { "id": "custom.fillOpacity", "value": 0 }
+            ]
+          }
+        ]
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "speedtest_download_mbps * 1e6",
+          "legendFormat": "{{instance}}"
+        },
+        {
+          "refId": "B",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "quantile_over_time(0.5, speedtest_download_mbps[7d]) * 1e6",
+          "legendFormat": "{{instance}} 7d median"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Upload",
+      "description": "Hourly upload rate per gateway.",
+      "id": 7,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 9, "w": 12, "x": 12, "y": 6 },
+      "fieldConfig": {
+        "defaults": { "unit": "bps", "min": 0 },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "speedtest_upload_mbps * 1e6",
+          "legendFormat": "{{instance}}"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Latency & jitter",
+      "description": "Ping and jitter towards the librespeed test server. A latency climb with stable throughput usually means bufferbloat or an ISP routing change.",
+      "id": 8,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 15 },
+      "fieldConfig": {
+        "defaults": { "unit": "ms", "min": 0 },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "speedtest_ping_ms",
+          "legendFormat": "{{instance}} ping"
+        },
+        {
+          "refId": "B",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "speedtest_jitter_ms",
+          "legendFormat": "{{instance}} jitter"
+        }
+      ]
+    }
+  ]
+}

modules/monitoring/parsedmarc.nix

@@ -0,0 +1,122 @@
+# DMARC report analyzer, imported by control only. parsedmarc fetches the
+# aggregate/forensic reports that land in the dmarc@cnx.email mailbox on mx1,
+# parses the XML, and stores results in a local Elasticsearch; the official
+# parsedmarc dashboard + an Elasticsearch datasource are auto-provisioned into
+# the Grafana instance that server.nix already runs on this host.
+#
+# IMAP runs over the ZeroTier mesh, not the public net: we pin mx1.cnx.email to
+# its mesh address in /etc/hosts so TLS still validates against the public
+# Let's Encrypt cert (primary domain mx1.cnx.email) while the bytes stay on the
+# overlay. The mailbox passphrase is the shared mail-dmarc-cred secret; parsedmarc
+# reads it as root in its ExecStartPre, so root-owned (clan default) is fine.
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  mesh = import ../mesh-hosts.nix { inherit config lib; };
+
+  # Grafana 13 (since the externalisation of core data sources) no longer bundles
+  # the Elasticsearch data source, and nixpkgs has no grafanaPlugins entry for it,
+  # so a bare elasticsearch datasource reports "Plugin not registered" and every
+  # DMARC panel fails. Pull the official, Grafana-signed plugin from the catalog
+  # and load it declaratively.
+  elasticsearchPlugin = pkgs.grafanaPlugins.grafanaPlugin {
+    pname = "elasticsearch";
+    version = "12.6.4";
+    zipHash.x86_64-linux = "sha256-xyAUprdWyQM0IJSg/oBVZ0ltFAAffgPrphln9+IKcUY=";
+  };
+in
+{
+  imports = [ ../mail-dmarc-cred.nix ];
+
+  # Elasticsearch 7.x is under the (unfree) Elastic License; allow just this one
+  # package rather than opening allowUnfree globally.
+  nixpkgs.config.allowUnfreePredicate = pkg: lib.getName pkg == "elasticsearch";
+
+  services.grafana.declarativePlugins = [ elasticsearchPlugin ];
+
+  # Keep mx1's IMAP traffic on the mesh while presenting the public cert name.
+  networking.hosts.${mesh.hosts.mx1} = [ "mx1.cnx.email" ];
+
+  services.parsedmarc = {
+    enable = true;
+    provision = {
+      # Local Elasticsearch on 127.0.0.1:9200 (loopback; no firewall change).
+      # datasource + dashboard default to true once ES and Grafana are both on.
+      elasticsearch = true;
+      # GeoIP needs a MaxMind account/license key; skip it (reports still parse,
+      # just without source-IP geolocation).
+      geoIp = false;
+      grafana = {
+        # We provision the two Elasticsearch datasources ourselves (below) so we
+        # can pin their UIDs. Leaving this on would add a second, uid-less pair
+        # and collide on the dmarc-ag/dmarc-fo names.
+        datasource = false;
+        dashboard = true;
+      };
+    };
+    settings = {
+      imap = {
+        host = "mx1.cnx.email";
+        port = 993;
+        ssl = true;
+        user = "dmarc@cnx.email";
+        password = {
+          _secret = config.clan.core.vars.generators.mail-dmarc-cred.files."passphrase".path;
+        };
+      };
+      mailbox = {
+        watch = true; # IMAP IDLE: process reports as they arrive
+        delete = false; # archive processed reports, don't delete
+      };
+      general = {
+        save_aggregate = true;
+        save_forensic = true;
+      };
+    };
+  };
+
+  # The bundled parsedmarc dashboard selects its datasource through template
+  # variables ($datasourceag/$datasourcefo) whose stored value is the string
+  # "dmarc-ag"/"dmarc-fo". Grafana 11+ resolves a datasource template variable
+  # by UID, not by name, so without a matching UID every panel renders
+  # "datasource was not found". Pin the UIDs to those names so it resolves.
+  services.grafana.provision.datasources.settings = {
+    # These two were previously provisioned uid-less, so they exist in Grafana's
+    # DB with random UIDs. Provisioning a *new* UID onto an existing datasource
+    # makes Grafana abort with "data source not found" and crash-loop, so delete
+    # the stale records first; the entries below re-create them with pinned UIDs.
+    deleteDatasources = [
+      {
+        name = "dmarc-ag";
+        orgId = 1;
+      }
+      {
+        name = "dmarc-fo";
+        orgId = 1;
+      }
+    ];
+    datasources =
+      let
+        esVersion = lib.getVersion config.services.elasticsearch.package;
+        es = name: {
+          inherit name;
+          uid = name;
+          type = "elasticsearch";
+          access = "proxy";
+          url = "http://localhost:9200";
+          jsonData = {
+            timeField = "date_range";
+            inherit esVersion;
+          };
+        };
+      in
+      [
+        (es "dmarc-ag")
+        (es "dmarc-fo")
+      ];
+  };
+}

modules/monitoring/server.nix

@@ -25,6 +25,7 @@ let
   v6 = addr: "[${addr}]";
 
   adminPasswordFile = config.clan.core.vars.generators.grafana-admin.files."password".path;
+  secretKeyFile = config.clan.core.vars.generators.grafana-secret-key.files."secret-key".path;
 in
 {
   services.victoriametrics = {
@@ -45,6 +46,15 @@ in
             (target "control" "127.0.0.1" 9100)
             (target "ns1" (v6 mesh.hosts.ns1) 9100)
             (target "ns2" (v6 mesh.hosts.ns2) 9100)
+            (target "mx1" (v6 mesh.hosts.mx1) 9100)
+            (target "web01" (v6 mesh.hosts.web01) 9100)
+            (target "gw-cnx-1" (v6 mesh.hosts.gw-cnx-1) 9100)
+          ];
+        }
+        {
+          job_name = "blocky";
+          static_configs = [
+            (target "gw-cnx-1" (v6 mesh.hosts.gw-cnx-1) 4000)
           ];
         }
         {
@@ -93,6 +103,22 @@ in
     '';
   };
 
+  # NixOS 26.05 removed the built-in default for services.grafana's secret_key
+  # (used to sign/encrypt DB secrets). Mint our own instead of hardcoding the old
+  # upstream default. Safe to generate fresh here: the provisioned datasources
+  # carry no credentials, so nothing in Grafana's DB is encrypted with it.
+  clan.core.vars.generators.grafana-secret-key = {
+    files."secret-key" = {
+      secret = true;
+      owner = "grafana";
+      group = "grafana";
+    };
+    runtimeInputs = [ pkgs.openssl ];
+    script = ''
+      openssl rand -base64 32 | tr -d "\n" > "$out"/secret-key
+    '';
+  };
+
   services.grafana = {
     enable = true;
     # VictoriaLogs datasource plugin so journald is greppable from Grafana,
@@ -107,6 +133,7 @@ in
       security = {
         admin_user = "admin";
         admin_password = "$__file{${adminPasswordFile}}";
+        secret_key = "$__file{${secretKeyFile}}";
       };
       "auth.anonymous".enabled = false;
       users.allow_sign_up = false;

modules/router/crowdsec.nix

@@ -0,0 +1,52 @@
+# CrowdSec security engine + nftables bouncer: parses sshd auth attempts from
+# the journal and bans offending source IPs at the firewall. Log-based (no
+# inline DPI) so it costs the N300 next to nothing.
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.cnx.router;
+  mesh = import ../mesh-hosts.nix { inherit config lib; };
+in
+{
+  config = lib.mkIf cfg.enable {
+    services.crowdsec = {
+      enable = true;
+      autoUpdateService = true;
+      hub.collections = [
+        "crowdsecurity/linux"
+        "crowdsecurity/sshd"
+      ];
+      localConfig = {
+        acquisitions = [
+          {
+            source = "journalctl";
+            journalctl_filter = [ "_SYSTEMD_UNIT=sshd.service" ];
+            labels.type = "syslog";
+          }
+        ];
+        # Never ban the ZeroTier mesh — it is the only admin path to these
+        # boxes (no public SSH), so a false positive would lock us out.
+        # Parser-stage whitelist: mesh events are dropped before any scenario.
+        parsers.s02Enrich = [
+          {
+            name = "cnx/mesh-whitelist";
+            description = "Whitelist the ZeroTier management mesh";
+            whitelist = {
+              reason = "ZeroTier mesh is the admin path";
+              cidr = [ mesh.subnet ];
+            };
+          }
+        ];
+      };
+    };
+
+    services.crowdsec-firewall-bouncer = {
+      enable = true;
+      registerBouncer.enable = true;
+      settings.mode = "nftables";
+    };
+  };
+}

modules/router/default.nix

@@ -0,0 +1,325 @@
+# Site gateway (OPNsense replacement) for the Topton 1U boxes: PPPoE WAN,
+# VLAN-filtering bridge over the LAN ports, and per-VLAN L3 interfaces.
+# Imported by machines/gw-<city>-<n>; everything is driven by cnx.router.*.
+#
+# Fleet addressing convention: each site owns 10.<siteId>.0.0/16. A VLAN's
+# subnet defaults to 10.<siteId>.<vlanId>.0/24 with the router at .1 and the
+# DHCP pool at .100-.199. VLANs that need more space (e.g. public-wifi guest)
+# override `subnet`/`address`/`dhcp.pool` and take a wider block from the
+# upper half (10.<siteId>.128.0/17), e.g. guest -> 10.<siteId>.128.0/22.
+# VLAN ids: 10 = mgmt, 20 = lan (mandatory); 30 = guest, 40 = iot (reserved).
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.cnx.router;
+
+  vlanIf = name: "vlan-${name}";
+  site = toString cfg.siteId;
+
+  vlanModule =
+    { name, config, ... }:
+    let
+      octet = toString config.id;
+    in
+    {
+      options = {
+        id = lib.mkOption {
+          type = lib.types.ints.between 1 4094;
+          description = "802.1Q VLAN id (fleet convention: 10 mgmt, 20 lan, 30 guest, 40 iot).";
+        };
+        address = lib.mkOption {
+          type = lib.types.str;
+          default = "10.${site}.${octet}.1";
+          description = "Router address on this VLAN.";
+        };
+        prefixLength = lib.mkOption {
+          type = lib.types.ints.between 8 30;
+          default = 24;
+        };
+        subnet = lib.mkOption {
+          type = lib.types.str;
+          default = "10.${site}.${octet}.0/24";
+          description = "The VLAN's network in CIDR form (must contain `address`).";
+        };
+        dhcp = {
+          enable = lib.mkOption {
+            type = lib.types.bool;
+            default = true;
+          };
+          pool = {
+            from = lib.mkOption {
+              type = lib.types.str;
+              default = "10.${site}.${octet}.100";
+            };
+            to = lib.mkOption {
+              type = lib.types.str;
+              default = "10.${site}.${octet}.199";
+            };
+          };
+          leaseTime = lib.mkOption {
+            type = lib.types.ints.positive;
+            default = 86400;
+            description = ''
+              Lease validity in seconds. Lower it for high-churn networks,
+              e.g. public-WiFi guest VLANs (3600-7200), so the pool recycles.
+            '';
+          };
+          reservations = lib.mkOption {
+            type = lib.types.attrsOf (
+              lib.types.submodule {
+                options = {
+                  hwAddress = lib.mkOption {
+                    type = lib.types.str;
+                    example = "aa:bb:cc:dd:ee:ff";
+                    description = "Client MAC address.";
+                  };
+                  ipAddress = lib.mkOption {
+                    type = lib.types.str;
+                    description = "Fixed address handed to this client (inside the VLAN's subnet, outside the pool).";
+                  };
+                };
+              }
+            );
+            default = { };
+            description = "Static DHCP leases; the attribute name becomes the client's hostname.";
+          };
+        };
+        allowWan = lib.mkOption {
+          type = lib.types.bool;
+          default = true;
+          description = "Whether clients on this VLAN may reach the internet.";
+        };
+      };
+    };
+in
+{
+  imports = [
+    ./pppoe.nix
+    ./ipv6.nix
+    ./firewall.nix
+    ./dns-dhcp.nix
+    ./crowdsec.nix
+    ./omada.nix
+    ./proxy.nix
+    ./iperf.nix
+    ./speedtest.nix
+  ];
+
+  options.cnx.router = {
+    enable = lib.mkEnableOption "site gateway (router) role";
+
+    site = lib.mkOption {
+      type = lib.types.str;
+      description = "City code of the site, e.g. \"cnx\".";
+    };
+
+    siteId = lib.mkOption {
+      type = lib.types.ints.between 1 254;
+      description = "Site number; drives the 10.<siteId>.<vlan>.0/24 addressing.";
+    };
+
+    wan.interface = lib.mkOption {
+      type = lib.types.str;
+      description = "Physical WAN port the PPPoE session runs on.";
+    };
+
+    wan.vlanId = lib.mkOption {
+      type = lib.types.nullOr (lib.types.ints.between 1 4094);
+      default = null;
+      description = ''
+        802.1Q tag the ISP requires for the PPPoE session (AIS Thailand: 10);
+        null for untagged PPPoE directly on the port. Unrelated to the LAN
+        VLANs — this tag exists only on the WAN port.
+      '';
+    };
+
+    wan.macAddress = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      example = "aa:bb:cc:dd:ee:ff";
+      description = ''
+        Spoofed MAC for the WAN port, e.g. to keep the MAC the ISP has
+        pinned (cloned from the old router). null keeps the hardware MAC.
+      '';
+    };
+
+    wan.pppInterface = lib.mkOption {
+      type = lib.types.str;
+      internal = true;
+      readOnly = true;
+      default = if cfg.wan.vlanId == null then cfg.wan.interface else "wan-vlan";
+      description = "Interface pppd dials on (the WAN port or its ISP VLAN).";
+    };
+
+    trunkPorts = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      description = "LAN ports carrying all VLANs tagged (incl. any 10G SFP+ ports).";
+    };
+
+    accessPorts = lib.mkOption {
+      type = lib.types.attrsOf lib.types.str;
+      default = { };
+      example = {
+        enp4s0 = "mgmt";
+      };
+      description = ''
+        Ports acting as untagged access ports on a single VLAN (port name ->
+        VLAN name). Frames are untagged on the wire; the bridge tags them with
+        the VLAN's PVID. Use for an always-available on-site mgmt port.
+      '';
+    };
+
+    vlans = lib.mkOption {
+      type = lib.types.attrsOf (lib.types.submodule vlanModule);
+      description = "VLANs served at this site; `mgmt` and `lan` are mandatory.";
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = cfg.vlans ? mgmt && cfg.vlans ? lan;
+        message = "cnx.router: every site must define the `mgmt` and `lan` VLANs.";
+      }
+      {
+        assertion = lib.all (v: cfg.vlans ? ${v}) (lib.attrValues cfg.accessPorts);
+        message = "cnx.router: every accessPorts value must name a defined VLAN.";
+      }
+      {
+        assertion = lib.all (p: !(cfg.accessPorts ? ${p})) cfg.trunkPorts;
+        message = "cnx.router: a port cannot be both a trunk and an access port.";
+      }
+    ];
+
+    # Router diagnostics toolkit: packets (tcpdump), path (mtr), link
+    # negotiation (ethtool), NAT state (conntrack), DNS (kdig), per-flow
+    # bandwidth (iftop), WAN throughput (librespeed-cli; iperf3 covers LAN).
+    environment.systemPackages = with pkgs; [
+      tcpdump
+      mtr
+      ethtool
+      conntrack-tools
+      knot-dns
+      iftop
+      librespeed-cli
+    ];
+
+    networking.useNetworkd = true;
+    networking.useDHCP = false;
+    systemd.network.enable = true;
+
+    systemd.network.netdevs = {
+      "20-br0" = {
+        netdevConfig = {
+          Name = "br0";
+          Kind = "bridge";
+        };
+        bridgeConfig.VLANFiltering = true;
+      };
+    }
+    // lib.optionalAttrs (cfg.wan.vlanId != null) {
+      "15-wan-vlan" = {
+        netdevConfig = {
+          Name = "wan-vlan";
+          Kind = "vlan";
+        };
+        vlanConfig.Id = cfg.wan.vlanId;
+      };
+    }
+    // lib.mapAttrs' (
+      name: vlan:
+      lib.nameValuePair "30-${vlanIf name}" {
+        netdevConfig = {
+          Name = vlanIf name;
+          Kind = "vlan";
+        };
+        vlanConfig.Id = vlan.id;
+      }
+    ) cfg.vlans;
+
+    systemd.network.networks =
+      let
+        taggedAll = lib.mapAttrsToList (_: vlan: { VLAN = vlan.id; }) cfg.vlans;
+      in
+      {
+        # WAN port carries only the PPPoE session; no IP config of its own.
+        "10-wan" = {
+          matchConfig.Name = cfg.wan.interface;
+          networkConfig.LinkLocalAddressing = "no";
+          vlan = lib.optional (cfg.wan.vlanId != null) "wan-vlan";
+          linkConfig = {
+            RequiredForOnline = "carrier";
+          }
+          # The wan-vlan subinterface (and thus the PPPoE session) inherits
+          # the parent port's MAC, so spoofing here covers both cases.
+          // lib.optionalAttrs (cfg.wan.macAddress != null) {
+            MACAddress = cfg.wan.macAddress;
+          };
+        };
+      }
+      // lib.optionalAttrs (cfg.wan.vlanId != null) {
+        # The ISP-side VLAN subinterface pppd dials on (e.g. AIS tags PPPoE).
+        "15-wan-vlan" = {
+          matchConfig.Name = "wan-vlan";
+          networkConfig.LinkLocalAddressing = "no";
+          linkConfig.RequiredForOnline = "no";
+        };
+      }
+      // {
+        # The bridge itself is L2-only; L3 lives on the vlan-* interfaces,
+        # which hang off the bridge (tagged on the bridge "self" port).
+        "20-br0" = {
+          matchConfig.Name = "br0";
+          networkConfig.LinkLocalAddressing = "no";
+          vlan = lib.mapAttrsToList (name: _: vlanIf name) cfg.vlans;
+          bridgeVLANs = taggedAll;
+          linkConfig.RequiredForOnline = "no";
+        };
+      }
+      // lib.listToAttrs (
+        map (port: {
+          name = "25-trunk-${port}";
+          value = {
+            matchConfig.Name = port;
+            networkConfig.Bridge = "br0";
+            bridgeVLANs = taggedAll;
+            linkConfig.RequiredForOnline = "no";
+          };
+        }) cfg.trunkPorts
+      )
+      // lib.mapAttrs' (
+        port: vlanName:
+        lib.nameValuePair "25-access-${port}" {
+          matchConfig.Name = port;
+          networkConfig.Bridge = "br0";
+          bridgeVLANs = [
+            {
+              VLAN = cfg.vlans.${vlanName}.id;
+              PVID = cfg.vlans.${vlanName}.id;
+              EgressUntagged = cfg.vlans.${vlanName}.id;
+            }
+          ];
+          linkConfig.RequiredForOnline = "no";
+        }
+      ) cfg.accessPorts
+      // lib.mapAttrs' (
+        name: vlan:
+        lib.nameValuePair "40-${vlanIf name}" {
+          matchConfig.Name = vlanIf name;
+          address = [ "${vlan.address}/${toString vlan.prefixLength}" ];
+          networkConfig = {
+            IPv6AcceptRA = false;
+            # Announce a /64 carved from the DHCPv6-PD prefix on ppp0 (SLAAC).
+            IPv6SendRA = true;
+            DHCPPrefixDelegation = true;
+          };
+          dhcpPrefixDelegationConfig.SubnetId = "auto";
+          linkConfig.RequiredForOnline = "no";
+        }
+      ) cfg.vlans;
+  };
+}

modules/router/dns-dhcp.nix

@@ -0,0 +1,84 @@
+# LAN DHCP (Kea) and DNS (Blocky). Fully declarative: one Kea subnet per VLAN
+# with dhcp.enable, Blocky as the blocklist resolver every DHCP lease points
+# at. Blocky's HTTP listener (:4000) serves Prometheus metrics, scraped by
+# control over the mesh (firewall.nix scopes it to the mesh subnet).
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.cnx.router;
+  dhcpVlans = lib.filterAttrs (_: vlan: vlan.dhcp.enable) cfg.vlans;
+in
+{
+  config = lib.mkIf cfg.enable {
+    services.kea.dhcp4 = {
+      enable = true;
+      settings = {
+        interfaces-config.interfaces = lib.mapAttrsToList (name: _: "vlan-${name}") dhcpVlans;
+        lease-database = {
+          type = "memfile";
+          persist = true;
+          name = "/var/lib/kea/dhcp4.leases";
+        };
+        valid-lifetime = 86400;
+        subnet4 = lib.mapAttrsToList (name: vlan: {
+          id = vlan.id;
+          subnet = vlan.subnet;
+          interface = "vlan-${name}";
+          valid-lifetime = vlan.dhcp.leaseTime;
+          pools = [ { pool = "${vlan.dhcp.pool.from} - ${vlan.dhcp.pool.to}"; } ];
+          reservations = lib.mapAttrsToList (host: res: {
+            hostname = host;
+            hw-address = res.hwAddress;
+            ip-address = res.ipAddress;
+          }) vlan.dhcp.reservations;
+          option-data = [
+            {
+              name = "routers";
+              data = vlan.address;
+            }
+            {
+              name = "domain-name-servers";
+              data = vlan.address;
+            }
+          ];
+        }) dhcpVlans;
+      };
+    };
+
+    services.blocky = {
+      enable = true;
+      settings = {
+        ports = {
+          dns = 53;
+          http = 4000;
+        };
+        upstreams.groups.default = [
+          "9.9.9.9"
+          "149.112.112.112"
+          "2620:fe::fe"
+        ];
+        blocking = {
+          denylists.ads = [
+            "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts"
+          ];
+          clientGroupsBlock.default = [ "ads" ];
+        };
+        caching = {
+          minTime = "5m";
+          prefetching = true;
+        };
+        prometheus.enable = true;
+      };
+    };
+
+    # The router itself resolves via public resolvers, not via Blocky, so DNS
+    # for deploys/updates survives a broken local resolver.
+    networking.nameservers = [
+      "9.9.9.9"
+      "1.1.1.1"
+    ];
+  };
+}

modules/router/firewall.nix

@@ -0,0 +1,62 @@
+# Router firewall/NAT policy (nftables). Trust model:
+#   mgmt VLAN  -> trusted: router services, all VLANs, WAN
+#   other VLANs -> DNS/DHCP on the router + WAN (if allowWan); no inter-VLAN
+#   WAN (ppp0) -> nothing inbound beyond established/related
+#   mesh       -> admin SSH + metrics scrapes (same trust boundary as the fleet)
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.cnx.router;
+  mesh = import ../mesh-hosts.nix { inherit config lib; };
+
+  vlanIfs = lib.mapAttrsToList (name: _: "vlan-${name}") cfg.vlans;
+  wanVlanIfs = lib.mapAttrsToList (name: _: "vlan-${name}") (
+    lib.filterAttrs (_: vlan: vlan.allowWan) cfg.vlans
+  );
+  nonMgmtIfs = lib.filter (i: i != "vlan-mgmt") vlanIfs;
+  ifSet = ifs: "{ ${lib.concatStringsSep ", " (map (i: "\"${i}\"") ifs)} }";
+in
+{
+  config = lib.mkIf cfg.enable {
+    networking.nftables.enable = true;
+
+    # SSH reachable only from the mgmt VLAN (trusted) and the mesh — never
+    # from the WAN or the other VLANs.
+    services.openssh.openFirewall = false;
+
+    networking.firewall = {
+      enable = true;
+      filterForward = true;
+      trustedInterfaces = [ "vlan-mgmt" ];
+
+      # Non-mgmt VLANs may only talk to the router's DNS and DHCP.
+      interfaces = lib.genAttrs nonMgmtIfs (_: {
+        allowedTCPPorts = [ 53 ];
+        allowedUDPPorts = [
+          53
+          67
+        ];
+      });
+
+      extraInputRules = ''
+        ip6 saddr ${mesh.subnet} tcp dport 22 accept comment "admin ssh over the mesh"
+        ip6 saddr ${mesh.subnet} tcp dport 4000 accept comment "blocky metrics scrape from control"
+      '';
+
+      extraForwardRules = ''
+        tcp flags syn tcp option maxseg size set rt mtu comment "MSS clamp for PPPoE mtu 1492"
+        iifname "vlan-mgmt" accept comment "mgmt reaches all VLANs and the WAN"
+        iifname ${ifSet wanVlanIfs} oifname "ppp0" accept comment "LAN to internet"
+      '';
+    };
+
+    networking.nat = {
+      enable = true;
+      externalInterface = "ppp0";
+      internalInterfaces = vlanIfs;
+    };
+  };
+}

modules/router/iperf.nix

@@ -0,0 +1,28 @@
+# iperf3 server on every gateway, for throughput testing from any LAN segment
+# (e.g. validating AP/switch links: `iperf3 -c 10.<siteId>.<vlan>.1`) and from
+# admin machines over the mesh. Never reachable from the WAN (default-deny).
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.cnx.router;
+  mesh = import ../mesh-hosts.nix { inherit config lib; };
+  vlanIfs = lib.mapAttrsToList (name: _: "vlan-${name}") cfg.vlans;
+in
+{
+  config = lib.mkIf cfg.enable {
+    services.iperf3.enable = true;
+
+    networking.firewall.interfaces = lib.genAttrs vlanIfs (_: {
+      allowedTCPPorts = [ 5201 ];
+      allowedUDPPorts = [ 5201 ];
+    });
+
+    networking.firewall.extraInputRules = ''
+      ip6 saddr ${mesh.subnet} tcp dport 5201 accept comment "iperf3 over the mesh"
+      ip6 saddr ${mesh.subnet} udp dport 5201 accept comment "iperf3 over the mesh"
+    '';
+  };
+}

modules/router/ipv6.nix

@@ -0,0 +1,31 @@
+# IPv6 on the PPPoE uplink: run networkd's DHCPv6 client on ppp0 to obtain a
+# delegated prefix; each vlan-* interface (default.nix) carves a /64 out of it
+# via DHCPPrefixDelegation and announces it to clients with SLAAC.
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.cnx.router;
+in
+{
+  config = lib.mkIf cfg.enable {
+    systemd.network.networks."45-ppp0" = {
+      matchConfig.Name = "ppp0";
+      networkConfig = {
+        DHCP = "ipv6";
+        # pppd owns the v4 address/route on this link; don't let networkd
+        # tear them down.
+        KeepConfiguration = "static";
+        # Default v6 route comes from the ISP's RA when they send one.
+        IPv6AcceptRA = true;
+      };
+      # Many PPPoE ISPs never send an RA with the M flag; solicit regardless.
+      dhcpV6Config.WithoutRA = "solicit";
+      linkConfig.RequiredForOnline = "no";
+    };
+
+    boot.kernel.sysctl."net.ipv6.conf.all.forwarding" = lib.mkDefault 1;
+  };
+}

modules/router/omada.nix

@@ -0,0 +1,43 @@
+# TP-Link Omada SDN controller for sites with Omada APs/switches. There is no
+# nixpkgs package, so it runs as a podman container (mbentley/omada-controller,
+# the de-facto standard image). Host networking because device adoption relies
+# on L2 broadcast discovery (UDP 29810) on the mgmt VLAN; the default-deny
+# input firewall keeps its ports unreachable from WAN and non-mgmt VLANs.
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.cnx.router;
+  mesh = import ../mesh-hosts.nix { inherit config lib; };
+in
+{
+  options.cnx.router.omada.enable =
+    lib.mkEnableOption "TP-Link Omada SDN controller (podman container)";
+
+  config = lib.mkIf (cfg.enable && cfg.omada.enable) {
+    virtualisation.podman.enable = true;
+    virtualisation.oci-containers = {
+      backend = "podman";
+      containers.omada = {
+        image = "docker.io/mbentley/omada-controller:5.15";
+        extraOptions = [ "--network=host" ];
+        environment.TZ = config.time.timeZone;
+        volumes = [
+          "/var/lib/omada/data:/opt/tplink/EAPController/data"
+          "/var/lib/omada/logs:/opt/tplink/EAPController/logs"
+        ];
+      };
+    };
+
+    # Admin UI (8043) also reachable over the mesh, like Grafana on control.
+    networking.firewall.extraInputRules = ''
+      ip6 saddr ${mesh.subnet} tcp dport 8043 accept comment "omada ui over the mesh"
+    '';
+
+    # Controller state (adopted devices, site config, cert) — declared as clan
+    # state so a borgbackup client can pick it up; backup wiring is a later step.
+    clan.core.state.omada.folders = [ "/var/lib/omada" ];
+  };
+}

modules/router/pppoe.nix

@@ -0,0 +1,64 @@
+# PPPoE WAN session. ISP credentials are entered once at `clan vars generate`
+# (prompts). Both are secret — AIS often uses the same string for username and
+# password — so neither may land in the Nix store: pppd reads the username from
+# an included secret options file and the password from chap/pap-secrets.
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.cnx.router;
+  creds = config.clan.core.vars.generators.pppoe-credentials;
+in
+{
+  config = lib.mkIf cfg.enable {
+    clan.core.vars.generators.pppoe-credentials = {
+      prompts.username = {
+        description = "PPPoE username (from the ISP)";
+        type = "hidden";
+      };
+      prompts.password = {
+        description = "PPPoE password (from the ISP)";
+        type = "hidden";
+      };
+      files."user-opts".secret = true;
+      files."chap-secrets".secret = true;
+      script = ''
+        user="$(cat "$prompts"/username)"
+        pass="$(cat "$prompts"/password)"
+        printf 'user "%s"\n' "$user" > "$out"/user-opts
+        printf '"%s" * "%s"\n' "$user" "$pass" > "$out"/chap-secrets
+      '';
+    };
+
+    services.pppd = {
+      enable = true;
+      peers.wan = {
+        autostart = true;
+        config = ''
+          plugin pppoe.so ${cfg.wan.pppInterface}
+          ifname ppp0
+          file ${creds.files."user-opts".path}
+          noipdefault
+          defaultroute
+          noauth
+          hide-password
+          persist
+          maxfail 0
+          holdoff 5
+          lcp-echo-interval 15
+          lcp-echo-failure 3
+          +ipv6
+          mtu 1492
+          mru 1492
+        '';
+      };
+    };
+
+    # pppd looks up the password for `user` in these files at dial time; both
+    # point at the same generated `"<user>" * "<pass>"` line (PAP and CHAP).
+    environment.etc."ppp/chap-secrets".source = creds.files."chap-secrets".path;
+    environment.etc."ppp/pap-secrets".source = creds.files."chap-secrets".path;
+  };
+}

modules/router/proxy.nix

@@ -0,0 +1,141 @@
+# Internal reverse proxy for the gateway: Caddy terminates TLS for
+# <service>.<site><siteId>.cnx.network (e.g. omada.cnx1.cnx.network) and
+# forwards to backends by their internal address. The cert is a real Let's
+# Encrypt wildcard (*.<site><siteId>.cnx.network) obtained via ACME DNS-01
+# against ns1 with a gateway-scoped TSIG key, so browsers trust it without
+# any CA install; the names only *resolve* internally — Blocky answers them
+# with the router's LAN address, the public zone never carries them.
+#
+# Requires the machine to also import the shared secret generator:
+#   (import ../../modules/dns/acme-gw-secret.nix "<hostname>")
+# and ns1 to hold the matching key + acl (see machines/ns1/configuration.nix).
+{
+  config,
+  lib,
+  ...
+}:
+let
+  cfg = config.cnx.router;
+  hosts = import ../hosts.nix;
+  hostname = config.networking.hostName;
+  tsigKey = "acme_${lib.replaceStrings [ "-" ] [ "_" ] hostname}";
+  certName = "${cfg.site}${toString cfg.siteId}.cnx.network";
+
+  serviceModule = {
+    options = {
+      backend = lib.mkOption {
+        type = lib.types.str;
+        example = "https://127.0.0.1:8043";
+        description = "URL Caddy forwards to (internal/mesh address).";
+      };
+      insecureSkipVerify = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = "Skip TLS verification towards the backend (self-signed upstreams like Omada).";
+      };
+    };
+  };
+in
+{
+  options.cnx.router.proxy = {
+    enable = lib.mkEnableOption "internal reverse proxy (Caddy, wildcard cert via DNS-01)";
+
+    services = lib.mkOption {
+      type = lib.types.attrsOf (lib.types.submodule serviceModule);
+      default = { };
+      description = "Proxied services; attr name becomes <name>.${certName}.";
+    };
+
+    allowVlans = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [
+        "mgmt"
+        "lan"
+      ];
+      description = "VLANs whose clients may reach the proxy (443, plus 80 for the redirect).";
+    };
+  };
+
+  config = lib.mkIf (cfg.enable && cfg.proxy.enable) {
+    assertions = [
+      {
+        assertion = lib.all (v: cfg.vlans ? ${v}) cfg.proxy.allowVlans;
+        message = "cnx.router.proxy.allowVlans must name VLANs defined in cnx.router.vlans.";
+      }
+    ];
+
+    # Render the shared per-gateway TSIG secret into a lego rfc2136 env file;
+    # same pattern as web01 (modules/web-proxy.nix), scoped on ns1 to
+    # _acme-challenge.<site><siteId> TXT records only.
+    clan.core.vars.generators."dns-acme-${hostname}-rfc2136" = {
+      files."rfc2136.env".secret = true; # root-owned; systemd reads it as root
+      dependencies = [ "dns-acme-${hostname}-secret" ];
+      script = ''
+        printf 'RFC2136_NAMESERVER=${hosts.ns1.ipv4}:53\nRFC2136_TSIG_ALGORITHM=hmac-sha256.\nRFC2136_TSIG_KEY=${tsigKey}\nRFC2136_TSIG_SECRET=%s\n' \
+          "$(cat "$in"/dns-acme-${hostname}-secret/secret)" > "$out"/rfc2136.env
+      '';
+    };
+
+    security.acme = {
+      acceptTerms = true;
+      defaults.email = "postmaster@cnx.email";
+      # One wildcard for every proxied service; DNS-01 against ns1, so issuance
+      # works behind PPPoE with no inbound reachability at all.
+      certs.${certName} = {
+        domain = "*.${certName}";
+        dnsProvider = "rfc2136";
+        environmentFile =
+          config.clan.core.vars.generators."dns-acme-${hostname}-rfc2136".files."rfc2136.env".path;
+        # ns1 is the only nameserver that accepts this key's UPDATE; check
+        # propagation against it directly rather than a public resolver.
+        dnsResolver = "${hosts.ns1.ipv4}:53";
+        # Caddy reads the cert from explicit file paths (tls directive below),
+        # so it won't notice a renewal on its own.
+        reloadServices = [ "caddy.service" ];
+      };
+    };
+
+    # The lego-issued cert is owned group=acme; Caddy needs to read the key.
+    users.users.caddy.extraGroups = [ "acme" ];
+
+    # The explicit `tls cert key` points Caddy at the wildcard cert and disables
+    # its automatic ACME, so no extra issuance happens.
+    services.caddy = {
+      enable = true;
+      virtualHosts = lib.mapAttrs' (
+        name: svc:
+        lib.nameValuePair "${name}.${certName}" {
+          extraConfig = ''
+            tls /var/lib/acme/${certName}/cert.pem /var/lib/acme/${certName}/key.pem
+            ${
+              if svc.insecureSkipVerify then
+                ''
+                  reverse_proxy ${svc.backend} {
+                    transport http {
+                      tls_insecure_skip_verify
+                    }
+                  }''
+              else
+                "reverse_proxy ${svc.backend}"
+            }
+          '';
+        }
+      ) cfg.proxy.services;
+    };
+
+    # Blocky answers <anything>.<site><siteId>.cnx.network (customDNS covers
+    # subdomains) with the router's LAN address — clients on any allowed VLAN
+    # reach that address through the router's input path.
+    services.blocky.settings.customDNS.mapping.${certName} = cfg.vlans.lan.address;
+
+    # 443 serves the proxy; 80 only carries Caddy's automatic HTTP->HTTPS
+    # redirect. mgmt is already a trusted interface; listed anyway so shrinking
+    # trustedInterfaces later doesn't silently break the proxy.
+    networking.firewall.interfaces = lib.genAttrs (map (v: "vlan-${v}") cfg.proxy.allowVlans) (_: {
+      allowedTCPPorts = [
+        80
+        443
+      ];
+    });
+  };
+}

modules/router/speedtest.nix

@@ -0,0 +1,67 @@
+# Periodic WAN speed test so ISP degradation shows up as a trend instead of a
+# complaint. A timer runs librespeed-cli and writes the results as Prometheus
+# metrics into node_exporter's textfile collector — they ride the existing
+# 9100 scrape to VictoriaMetrics, where alerts.nix compares each run against
+# the link's own 7-day median (no per-site threshold to maintain).
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.cnx.router;
+  textfileDir = "/var/lib/speedtest";
+in
+{
+  options.cnx.router.speedtest.interval = lib.mkOption {
+    type = lib.types.str;
+    default = "hourly";
+    description = "systemd OnCalendar spec for the WAN speed test.";
+  };
+
+  config = lib.mkIf cfg.enable {
+    services.prometheus.exporters.node.extraFlags = [
+      "--collector.textfile.directory=${textfileDir}"
+    ];
+
+    systemd.services.speedtest = {
+      description = "WAN speed test to Prometheus textfile metrics";
+      after = [ "network-online.target" ];
+      wants = [ "network-online.target" ];
+      path = [
+        pkgs.librespeed-cli
+        pkgs.jq
+      ];
+      serviceConfig = {
+        Type = "oneshot";
+        StateDirectory = "speedtest";
+        # One test at boot would race PPPoE and log a spurious failure.
+        ExecCondition = "${pkgs.iproute2}/bin/ip link show ppp0";
+      };
+      script = ''
+        tmp="${textfileDir}/.speedtest.prom.tmp"
+        if result=$(librespeed-cli --json); then
+          jq -r '.[0]
+            | "speedtest_download_mbps \(.download)",
+              "speedtest_upload_mbps \(.upload)",
+              "speedtest_ping_ms \(.ping)",
+              "speedtest_jitter_ms \(.jitter)",
+              "speedtest_success 1"' <<<"$result" > "$tmp"
+        else
+          echo "speedtest_success 0" > "$tmp"
+        fi
+        mv "$tmp" "${textfileDir}/speedtest.prom"
+      '';
+    };
+
+    systemd.timers.speedtest = {
+      wantedBy = [ "timers.target" ];
+      timerConfig = {
+        OnCalendar = cfg.speedtest.interval;
+        RandomizedDelaySec = "10m";
+        Persistent = true;
+      };
+    };
+  };
+}

modules/web-proxy.nix

@@ -0,0 +1,73 @@
+# Public reverse proxy with TLS termination for web01. Caddy fronts internal
+# services and forwards to them over the ZeroTier mesh, never the public net.
+# The cert is a single wildcard (*.cnx.network) obtained via ACME DNS-01, so
+# adding a vhost needs no new issuance. Public ports: 443 for the proxy and 80
+# only for Caddy's HTTP->HTTPS redirect (issuance never uses inbound HTTP).
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  mesh = import ./mesh-hosts.nix { inherit config lib; };
+  hosts = import ./hosts.nix;
+  certName = "cnx.network";
+in
+{
+  imports = [ ./dns/acme-web01-secret.nix ];
+
+  # Render the shared acme_web01 TSIG secret into a lego rfc2136 env file. lego
+  # (via security.acme below) uses it to write _acme-challenge.cnx.network TXT
+  # records on ns1, which authorizes the acme_web01 key for exactly that owner.
+  clan.core.vars.generators.dns-acme-web01-rfc2136 = {
+    files."rfc2136.env".secret = true; # root-owned; systemd reads it as root
+    dependencies = [ "dns-acme-web01-secret" ];
+    script = ''
+      printf 'RFC2136_NAMESERVER=${hosts.ns1.ipv4}:53\nRFC2136_TSIG_ALGORITHM=hmac-sha256.\nRFC2136_TSIG_KEY=acme_web01\nRFC2136_TSIG_SECRET=%s\n' \
+        "$(cat "$in"/dns-acme-web01-secret/secret)" > "$out"/rfc2136.env
+    '';
+  };
+
+  security.acme = {
+    acceptTerms = true;
+    defaults.email = "postmaster@cnx.email";
+    # One wildcard cert for every vhost this proxy serves, via DNS-01 (so issuance
+    # never depends on inbound HTTP). Port 80 is open only for Caddy's
+    # HTTP->HTTPS redirect, not for ACME.
+    certs.${certName} = {
+      domain = "*.cnx.network";
+      extraDomainNames = [ "cnx.network" ];
+      dnsProvider = "rfc2136";
+      environmentFile = config.clan.core.vars.generators.dns-acme-web01-rfc2136.files."rfc2136.env".path;
+      # ns1 is the only nameserver that accepts the acme_web01 UPDATE; check
+      # propagation against it directly rather than a public resolver.
+      dnsResolver = "${hosts.ns1.ipv4}:53";
+      # Caddy reads the cert from explicit file paths (tls directive below), so it
+      # won't notice a renewal on its own — reload it whenever the cert changes.
+      reloadServices = [ "caddy.service" ];
+    };
+  };
+
+  # The lego-issued cert is owned group=acme; Caddy needs to read the key.
+  users.users.caddy.extraGroups = [ "acme" ];
+
+  # Reverse proxy. The explicit `tls cert key` points Caddy at the wildcard cert
+  # and disables its automatic ACME, so no extra issuance happens. Backends are
+  # dialed over the mesh by their ZeroTier address (mesh.hosts.<name>).
+  services.caddy = {
+    enable = true;
+    virtualHosts."grafana.cnx.network".extraConfig = ''
+      tls /var/lib/acme/${certName}/cert.pem /var/lib/acme/${certName}/key.pem
+      reverse_proxy http://[${mesh.hosts.control}]:3000
+    '';
+  };
+
+  # 443 serves the proxy; 80 only carries Caddy's automatic HTTP->HTTPS redirect
+  # (the Hetzner cloud firewall also scopes these in
+  # modules/hetzner-firewall-rules.nix). Admin still rides the mesh.
+  networking.firewall.allowedTCPPorts = [
+    80
+    443
+  ];
+}

sops/machines/gw-cnx-1/key.json

@@ -0,0 +1,6 @@
+[
+  {
+    "publickey": "age1g8a6fqhqg8q8qu4jyuhxc7hf00t5qvra55chyfm0qk45wlmvc9qs2cft8t",
+    "type": "age"
+  }
+]

sops/machines/mx1/key.json

@@ -0,0 +1,6 @@
+[
+  {
+    "publickey": "age1l5hw95p5h4sthrgn0usms9yfkwwmcvv34tjgrtv9s4e6x39chacshgxavs",
+    "type": "age"
+  }
+]

sops/machines/web01/key.json

@@ -0,0 +1,6 @@
+[
+  {
+    "publickey": "age1yey6gxgsyl4tj6ek0tve2pckt6qersqspk66ukkzum8mrr6zppqsj4jn3m",
+    "type": "age"
+  }
+]

sops/secrets/control-age.key/groups/admins

@@ -0,0 +1 @@
+../../../groups/admins

sops/secrets/control-age.key/secret

@@ -3,8 +3,12 @@
 	"sops": {
 		"age": [
 			{
-				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBNRWlKNFVSOThSWlI3LzAz\ndnQxbnNGSlhoVmZrQmpIVEJieStlUzlseDJJCmdGbldCSU1oTGszZmtzSWhJaWta\nVDhoQU8vVXB0bG1VUkxoWDlyR0I1a0EKLS0tIDFzdklCTTBjeXBoT3FqVW1Pbkxl\nYVZ5Vm55OFp4YWwzS3dVNXRoZmpBT1EK0pNbk20gwPe5Q1bZR3XRHMWz+x+8FzRa\niq5tLcGkgfMO6WTzj8vtB7g0Q3sFgmPXTqb8yMzqAkZd4z9FKJGt6A==\n-----END AGE ENCRYPTED FILE-----\n",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBcmJ4cXNt\nL3k4UmtqQUJZOGRoZFhab0pRSTdBUm9ZZkRxdHNvaCtRc05JVgphbHhHQThiRDNF\neW8xVGhXQ1pvSkRDR2hod2Zpa3pwaWZDVUpGZlpPWkNZCi0tLSB2T3JCcElnbVU2\nekJ5ZTdyQzZwbmR4aHBJUTc3M3loUm0wbHpHTGM0ZW1RCs0kHes9rkjd3uwHzxhi\n8x8AcQcQTR+CGsV5XPzGVMKKUZpwXWeNnvcHfVwYcqOnXucbUdCxLc6d57c9GQpC\nd3Y=\n-----END AGE ENCRYPTED FILE-----\n"
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBcFBHaFpS\nbjJiL0ZnOXlzMVVNdjhvUGxmaEJhTklSR2ZaTnhPWXFCaC8vbgpLdE04aW16dEls\neGw3VllEYVAzbGJ5TVZoOXhkR3dCUkREdUkxMU13clVzCi0tLSA3eW93QnFsNW1r\nK0ZiWEtrWFVqeTJyTk1maGVTMzdBTkJUdk5WN2N3WlprCp5oak+Bh/g4+ikE1LR5\n7kYrWQlzjC0Z+F5GegEtRjGCPnolWllnPUH4dGGjfuyAxy9qfi1J6Bi6m8uzo5rY\n+zs=\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa"
 			}
 		],
 		"lastmodified": "2026-06-14T05:20:21Z",

sops/secrets/gw-cnx-1-age.key/groups/admins

@@ -0,0 +1 @@
+../../../groups/admins

sops/secrets/gw-cnx-1-age.key/secret

@@ -0,0 +1,19 @@
+{
+	"data": "ENC[AES256_GCM,data:bsb98446kjvnDAVWNIqvH64uD6XyfEpohPFZs38jDaxxTp1wqJmFdeDW+p3SGld8Q+4h6rq7rFtTUJnQyslvgsPfYN5tBhJuTbI=,iv:pPUf3RgHqS8LlRIBdZ7jmzxp5VmlP68nrX8TCVP8aGU=,tag:SseZa3Sf5LuMXWN8CNSbxw==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBLRkppdUoza2l0SnFpME5n\nM0xBRVk4c2lhaHMvazFNaGNTdkJDcnJEQkVrCjhFUlZMUitnZ20zQlFEcm5WQmVG\nWmJVeUh6S09MS0xpQUR4aDF1V1dCZFkKLS0tICtOMEJ6QmRRR3BkaU5sZHdzOElO\nSnV4K0dHMHhDOURPTEpucXFuNEUwaU0Kx4xAaKTrw1bKJPQVCHcByTcapFhslwrV\nqssEBnYWXqFgI3vh3CatBXuZY1Lm3eTKhDDQ2ZyPKgc/l0gXyFwH2g==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBMHFKQTNG\nazdEcThReFZGVVBtVFBDY1luQk1UTkl0a0p1bzQ0cUxBZWVyYwo4aU5qaEs0czJB\nUzZFcG9uTUZ1OUFScW9ZeHljbDBEbjkzZ08vS2F1Qjd3Ci0tLSBHTXhVWjNsWHhK\nbWtrMWw3Nm8wSU9MNXZwcWxVZXRJS1dTK0hycnFoVlRJChemDXVb7LgM7OeUUeqa\nTl0IuDHmIIx/lUYnJWGZmWQHz2znk679j36VQQNWcK6eZlKUIqYmBj3fMu+OYir2\nMLI=\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa"
+			}
+		],
+		"lastmodified": "2026-07-28T09:03:13Z",
+		"mac": "ENC[AES256_GCM,data:MKIyykYwtQqpUnXqZQKzKwoRhSon2PH0wYiTDTvogaVREHfpEPs/mdOjQBUHtbr1QGqWNh4AxdBTKfpjXTlncHAEtcIdVH3qPccVxsXqfqRmZPDFzcHZAIw+VSvDsKwD3anixiXDeHmeLawwbyb1rnS0P8KquXbmjvirwGUIQLU=,iv:+hdLXMJMCby/6Q6JGxp4p95GgPKbdA/OTAZloyylr08=,tag:ZZ4dKjhjpE2xqcoNf4qeoA==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.13.1"
+	}
+}

sops/secrets/gw-cnx-1-age.key/users/berwn

@@ -0,0 +1 @@
+../../../users/berwn

sops/secrets/mx1-age.key/groups/admins

@@ -0,0 +1 @@
+../../../groups/admins

sops/secrets/mx1-age.key/secret

@@ -0,0 +1,18 @@
+{
+	"data": "ENC[AES256_GCM,data:lAgbSvyxBl0/NG8rHweKEEqYyDsHa+SrPZnxVubB2x5H3cqydhTCs8NMxJp+RKTPW2nhfmB/lrKgLHFfFMJfMg5jZ2BvZtR1IzA=,iv:1PgMbra/ec7yiCm7K5yo+1lCLJ89ryfP3SQML7uYv4k=,tag:lmaG+dZPCLHtQ59uFeSkVg==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBELzcrdUc0cW9pSXhGbzRY\naTd5aWgzcFpLQ3MyVElaZU5JK2Z4ZGRtMXpJCk14bmZLRlY4Z3pDeUxPM203anVY\nbE1aUEdDZ3NkcGxrcCtWS1ZmRWIrcmMKLS0tIDVTOUJhenR0WVhZSWNWT2VtRFpv\nc1h6Nk95Z3Y4eFQ1NUdmWWJiaWdDdGMKB3whh/RgAePTJnGmeDJ/WFv4NI42vA5O\nB0F6jmSDNa5beP8Um2DjWdPENkJJjv9yv38b7hP8BLDe9Ba4WBNfBA==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBby9pdmFZ\nOElJVCtUQmJWVVljQi9KcHE2a2FaeU5HeUhyUm05Zm0wZDZKVwptY2hHRFJKMi9I\nN3NDY3ZjYVloTjF6R0l4RllDS1dpNzA1TDJDc0hGQnFvCi0tLSB1akhUWk9RU0Rt\naDV5MHdHcExlSzZWbSs0S1A3NGhES0V1U0pEaUd5WE04ChtGuEq0HnRiVTDwhJnO\nIWMhwCYaewHk+k0a0Z9qCwqqKxhfiGS6kg/YTKHTNhQ0bxIA8yqgQoaE8Hl5Zhzs\nKqk=\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2026-06-18T04:33:03Z",
+		"mac": "ENC[AES256_GCM,data:Iffy/eS80qOJLdaGOIxti1otdxSLHA1TV06R8xq4zI1qFklJX8OWRy9CAYwAcTURS88gd5c2VGHsBI8yHOBj7LKiLJiu5/xRgD106hotBOJUy90UZ6MMbM2wwHomhQg5r5kNpRsJaPFaLFzm5YJN9gLUU1kxO4Nnt2L3bxZTogQ=,iv:cqbFD3ZIJnaZU7j7l9Qt1QqiUSQtclV7MGZWEZRZQoY=,tag:F0ZZQSQKYtBI65uutjNxow==,type:str]",
+		"version": "3.12.1"
+	}
+}

sops/secrets/mx1-age.key/users/berwn

@@ -0,0 +1 @@
+../../../users/berwn

sops/secrets/ns1-age.key/groups/admins

@@ -0,0 +1 @@
+../../../groups/admins

sops/secrets/ns1-age.key/secret

@@ -3,8 +3,12 @@
 	"sops": {
 		"age": [
 			{
-				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBcU01TXl0\nRmU5ZlZOK3NZZE8vNktOWXBkRnZtZHlUbnBHYlVQbDdzZlRDbApFeFI0OE1KdndQ\nZWczZGRYZXBONTc2aG4vZTJEZ1hCdGljNFJBcmF0UGpRCi0tLSBGMGJVWUtJMWFP\nQUZwWVZTWUFKZnVJUTJSMHlRMGRNcExPSEpXTFpLSkVNCj1WEuRyATcxp0GYeoqC\nACpQpcqgIBVD1dYB0pPo+I6yDXvx+JbwVdgGl+2Cxj40HQOfyvJecoOO3efOZrB3\nUZU=\n-----END AGE ENCRYPTED FILE-----\n",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBemRVWUs1\naXFpVm80UmloUUpnR1BVaGsxampHV1JFNWFnSUhiY0orOTU0cgppVEJ5QVcvbGpC\nZlErd01XUzh0MnZOekkvTkc5Z3owS3pBbkNCUDJ6SlJRCi0tLSBRVVgrd3hIVEla\nMEhoN3FDSG1HdGk5YU8ydzlVUGd1RW5zM1lka3VuSHFNClrBItoFo7uqpS5fuvaG\nOvh6mTylk4bHgcSRGKTk383srgsvPdZaioJfUs7fbVmLD3+bhfM/Wgv7tI6kO1dj\nmDg=\n-----END AGE ENCRYPTED FILE-----\n"
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBRb0ZvUmN2RWhtQkpXMXJr\nY1o2MXZsQ0dXNFYxa2RLUGlJWnN3Z25zbnp3CmtLbkYvdHR2MUZIY2U1MzhyU1VD\ndm5MTmMveEJLOHZWWnVWdVpDcTFsTmMKLS0tIFEwYzZYeU5uc29BTEVxN2YvN0Zw\nd0VSeTNhQk1pNVdaZVdTVk1oNjFwQWcKwEbHyT354Cbam9wuiIhapre+nbqtMr9h\nbOjF53LR9RZ9/mbpTNDLOkq2QW2aM4tKkRfdUgP1XgboilzcTjba8A==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06"
 			}
 		],
 		"lastmodified": "2026-06-14T05:20:24Z",

sops/secrets/ns2-age.key/groups/admins

@@ -0,0 +1 @@
+../../../groups/admins

sops/secrets/ns2-age.key/secret

@@ -3,8 +3,12 @@
 	"sops": {
 		"age": [
 			{
-				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBaTVGQWRC\nZUJocGx5Zjd0c1JpUEdHbGczNUh1RjRqSERRNHNSbnRndWFUMAoxbFBEU0Ezajhr\nSUlIRStWUUdVTDJMQWxmSkw0c0FNYnFxNEJIdnExTmx3Ci0tLSBMRGRPLzlXTEN3\ncHZCMGF3WDZTcFhTd2srL3Vpc2dRdExrVzFJcWhjQVJrCrA1y2SapkZoppI3z4LD\nH71D/WxIcTHp3fIzICNvgxWHn3+vkDDZcpcgH+gRiPCDqrgDp0NZcZiDlz6zyHGr\nmaM=\n-----END AGE ENCRYPTED FILE-----\n",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBL2doZFpO\na2Y2QzZSckJaZmZsTlFtQU1mckxJWTBiUXA2SUljVWZDemFJYgpwcFl6RHh2ZXdn\nNTBsaEVpWm5oYWtteU5WZmxWK1RRK2NoYW0yWlpCd3dBCi0tLSBsZVNpczR4dUdv\nSmtqQzMvc1ZYY0xUY0JMQmt2YzB5YmZOMEtXcW9vWUcwCnCdgcgiwF2rkWsV0IdO\n/6cs35FBypWpfflGwOwP0GnTrNizc9HphcMcwAi5NMQVi6X90Xgm5aFdpAsSWepX\n/SI=\n-----END AGE ENCRYPTED FILE-----\n"
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB0Rmp1S29hVE5LNkZpLzRV\nMGZJL242YmdHNkFXR2d0TmlVWXViTlJFNGpnClBSdEhaRlNWbVNrb05tTHh0Q3Rk\naVFNdUd1clNVckREajVoRmZCenFEaVkKLS0tIExOaHd1WWxZRFM1UFk4K0E0cWpE\nK2duUE1KeEtmeVorRnc3MFdDZXFGZjAKKw4ykWu+HzBN3WAj/gx0mMYk0RiyRR50\nYv7hg1B/+x4lDCTjAJIWrrBEjLJBZTZRnECmTMTlJ39uOhxeUOizCA==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06"
 			}
 		],
 		"lastmodified": "2026-06-14T05:20:27Z",

sops/secrets/web01-age.key/groups/admins

@@ -0,0 +1 @@
+../../../groups/admins

sops/secrets/web01-age.key/secret

@@ -0,0 +1,18 @@
+{
+	"data": "ENC[AES256_GCM,data:u+O5nWbFlvp6SGyHJggfkCLCT0ZuDy89e/VGGQNdt3yYzgdNmnrtd+2q+Ft3MtoOSSCLvStriGQzfLhcqEgqGgt3PqfIzCO1IG4=,iv:GyrZ1XUiOZe1I1Z/HebTy2NM2tfDHxIH5zGVk7HD+xQ=,tag:js6fbXWajVZSxt0hmnnA5g==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBWenArV1pZMURzNElTeGlC\nRSttUFRwM2E5d2htVDlFcTkwVDIrV2JxQjE0CmE0NUZ4UERHdHpqYk9uK01GOTFQ\ndW5UNlRrY0hvU2pNQmpSanh5RG5YbVkKLS0tIFdYQmkva1NORG80U3dDSWszZTlO\nOXViWUxMTVR1NE00cjdpVXpVS1J5YU0KHBkeKAJZDc+R1GLKwDYLyQBlEW7tPnMh\nf3tsUvtD0flqPAXNeDgyOmKufP7U6oDy/OriFC9+zYQbWyEEc6CZHg==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBL05xT1BJ\nSldCSkFhaFErWDNMeXMvRksvekkwNUZLZFVtSmUydVRrNG9JTwpFaHNsVThZN2gw\nMlNxUXhnN2xYNFluK1hadGxzMFZuaWR3cVFRYW9mWndVCi0tLSBUUmRMSnJCeTRM\nUVdKY3hzWVRkQkFuQ3FaRDVZSEp1b2N1ajg5RnlhT3VRCjX/vWj0We88ATiz808w\nz60RL0BvDGJ6m1BNmqAdtfCCClH33YXQBGrKT2E5elvOTl0iOCrT7HPjzXxJZXuw\nkKg=\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2026-06-20T19:26:29Z",
+		"mac": "ENC[AES256_GCM,data:uD+L3op6ZPtvmLreJ7S4GE6bQuItV87w1LTMFvjI1Kb5+Z0sXlL0TpYO8WLx8X0yaL3HddwlmBKYQGp/OlRPqZMFDbQuK25oeKB54jfm6YDn66rrMQJl1FOw68fLJaHYNjNelEg/bj2WG7YpfZBoWO67MW6F+44Rg4XF85w/1x0=,iv:X9TmEL5JkcC9waLumWpgpBwp3YWLMslZi++dv2HZ0mk=,tag:i7c1CxT1Xv/+T/jY9E0cdg==,type:str]",
+		"version": "3.12.1"
+	}
+}

sops/secrets/web01-age.key/users/berwn

@@ -0,0 +1 @@
+../../../users/berwn

vars/per-machine/control/grafana-secret-key/secret-key/secret

@@ -0,0 +1,22 @@
+{
+	"data": "ENC[AES256_GCM,data:78fZXLuEBqiLerV0HZ5cEKCeoWayeCSJcfhW7fFck0t08I44UqPW+yzXwkM=,iv:P6OX1cwnY0dmu7IRChwfcsWPsEBHmlBHZfnJTFf8FHw=,tag:sBAE2h0rNxFqhtqQIt/nmQ==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBWQ3lYVmhQRkt3TU5mZ1Fr\nR2Z3MXAwU2J2Zi9VSzJ4YlJhN1BZRVVPSVU4ClVSWlNxdjU0NkNXOFNnUG5ZWno3\nUFQxZzE2bVFMNHFiWWZJdDVYU0o3cG8KLS0tIElQbzhoZHh4Tk5MUjNEdVdrZmxF\nckIwYktjb2hZQStDcXU0Z0NTbkdIYlEKQkFi/lgBoC6zyzu08hhEOUvJse0E6YV2\nLwlLcWl+303HWeWob2R4VZhuIkkqk5pE2Do83AFsU8vick8dUC1sQg==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1env5y2eey0p3dggs8tydwmtyhtwqylpg7spuququ4vvax6arzp2qjnm4tx"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKYkl2MmRPbW90Q3doZ3JY\nb2F0TG10RjEyNXdiSGZGNHlvbVpjQUdXVjB3CjVCZ1UwRkRzK2NKMnVxQ0N0MmJp\ncElvdm9ucGNpTEZnSHFld0Fhb1lLU0UKLS0tIGpZWmhXcVJSS1NHbFYwc01JaDN6\nVm9yOFhZYzU2ZjQ0KzNtV201L3ZnUTQKhGGDHPHcizFBuJx1ZhObb+yqDQWROG1k\nLNqoSvLtd2glN7kC6ro8cvAGW/Oj4Baawetmtz4ctew4b0/8xG3PRQ==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBMnJyMEdV\nUndYaWtiMEtIakxXWUxPYTdvSTcxK0NaOThUcmN1d0hBaWpiUQowbTE3Q080UTVQ\nYnRjRVFwQkZPYUE2c0JFZklnL1RILzVET01JbEg1ZVF3Ci0tLSBsNkVRRXR6WDM0\nR3ZQcmE2VDdETURYYThCRzlabWErc1g3UDFFdkxFeW1rCgbVXMv1eD1Bu8QV/2gk\nNgVeQSvH1qEPQuUIeHqps/x3NyNpii/IymvkNvZJerowuD5TeSIeE2tqCA+0d5ph\nUpw=\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa"
+			}
+		],
+		"lastmodified": "2026-06-25T00:27:59Z",
+		"mac": "ENC[AES256_GCM,data:BDQfNDgwJGvExG9jTOs/BULCSpBvMjnpYtlPUP8m9/9kL+2cKBgHukQqqM4Eaju5dm38hGW9345PQvLeqoD53thtUyPR+GRlfAtKwGSqLfur1KlMEM/b50phtiJ2GjzS4bKu+weTYdmfux9Quh+HI+kVuHHLyRs0A13qSsSikvs=,iv:MmXMr2m2or1dlMTa4LvqRjPGndOedcFZdq/fq8fnzBU=,tag:6bBVwKGqRnWYmFY6q6nSnQ==,type:str]",
+		"version": "3.13.1"
+	}
+}

vars/per-machine/control/openssh-cert/.validation-hash

@@ -0,0 +1 @@
+9a9e2f23526941b204dacfe8fc77dad5f4b1c7aff945cdbf301ec61129959228

vars/per-machine/control/openssh-cert/ssh.id_ed25519-cert.pub/value

@@ -0,0 +1 @@
+ssh-ed25519-cert-v01@openssh.com AAAAIHNzaC1lZDI1NTE5LWNlcnQtdjAxQG9wZW5zc2guY29tAAAAIHiQTWR/pRz9vlrtQSe+u2g8XPDZvFpRBCpF7bQ6mBPaAAAAIH+Ix/XbsbqHU87Kg7Y9frnhYgfUEMNKOmQ3DSlb9LCmAAAAAAAAAAAAAAACAAAAB2NvbnRyb2wAAAAgAAAAHGNvbnRyb2wuY254LW5ldHdvcmsuaW50ZXJuYWwAAAAAAAAAAP//////////AAAAAAAAAAAAAAAAAAAAMwAAAAtzc2gtZWQyNTUxOQAAACCboWwGkseX7mqk2GVtM2o1WPRzh8U3X6A3woanYG9xoQAAAFMAAAALc3NoLWVkMjU1MTkAAABAbWgsXjGd/QGdbYHNfHs4LiMgF1weHHirWk4ZLMuTLRDXObwdJ4tbc9IGDfNBZnAfOSxBBy+U/sjGbKQtRWt4Aw== /tmp/vars-irrhty3h/in/openssh/ssh.id_ed25519.pub

vars/per-machine/control/user-password-root/user-password-hash/machines/control

@@ -0,0 +1 @@
+../../../../../../sops/machines/control

vars/per-machine/control/user-password-root/user-password-hash/secret

@@ -0,0 +1,22 @@
+{
+	"data": "ENC[AES256_GCM,data:r0i0hSQpZQfTP/2VhM7DkFAits86qDRlowbnwZqs4JrtvNjmReJe/mkF8xPT4eKusIzB/THAY4udgltrrOP0iFRMBPQqeFsd0w==,iv:+oYEln5OVeaFujy22IGKxKEgsExw0pSK3+bYQ82riPI=,tag:/FP5zwVfUszr3oAv6nwyVA==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBWSkthdHhQMHJ5THMxNUdS\nTGRZUVFDMEd6OVBPWFN3bXVzcFNnT0lrR1hrCkZ2bVRvVnJlM3BLc0ZqNUUwQ1VD\nMlJ1bmpkdzVYYURPOUorNkZKWDk5NGMKLS0tIC80Yy81K2owMDh6SFVMSmNOeERq\nVEJWNFRLV1gxOUx5WWJwWE5uNDR1eTgKdEcfmvFy5E81t2SWWEdPCbQy9nxGsMCT\nsZZxIbGwN0W3pf+QoPtIHl2tAixKy+gHPp8UNTGToDuK6ILSf8OoSQ==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1env5y2eey0p3dggs8tydwmtyhtwqylpg7spuququ4vvax6arzp2qjnm4tx"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBjTU8vOVRNbjBNOWpBWWpJ\nTUtIY1g4R1VVb2krNWxDOFpobkhuSmVkTFV3ClA0azFHVzkrY0xJdERZV0tLZkhN\nbldVYjJObjBIZm1RelB5VzBiU3dqK2sKLS0tIEdxRFRJMTI3RzI3ampRYVM2YWJI\nSU1pTWtRWVU2RDRHcmZtaGxhWElwd2sKK1zZtqqkdRMdITSKbJ/JilpAAflkgG5G\nnaeHwBIw75pZa6lQDM2aRWN+yFUMQtbJ3i7Zw/Du+emikyN1+hz5cg==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBdzdqU0M0\nOGNqMHJLZ0V4RHhnclE2SUY0aVFVMjB1azVlUlE5VXBTbGRsTgpFWjNPaDErU0cy\nNFFGR0VIODRTamc0bWxCTFN4MU9CdllaWjhvOUJaeWU4Ci0tLSBlK09xeGpRenk0\nZjlLTkhtR0NZZ1Jna2xHbFpZeUl3aEZ4cndhcHFOQkk0CuX8TF7mW7GowFZ2X75v\ng8zrxp3xtMzPp/C+dbuu02Lfe9f/1jaBOTtJjR6BnV0ucfYjUhtw7OQ3qCUV7a+Z\nG0c=\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa"
+			}
+		],
+		"lastmodified": "2026-07-28T10:15:43Z",
+		"mac": "ENC[AES256_GCM,data:uq8xDJBop1yPYMb6gNNWhKxJAiBdMjuYHhA3fvpCauDAakKgEGcSmc7/cnSAtds4NbsPkxi6tn32aJvF8QsKlSIycnqDpxGqpGVgYjvt/lasM50Ukwk6W+YGwTx8bnCmFazimJuaNg92G0VZAGI0uybITR3EEHws/9na69470jc=,iv:hJcVMW1Gaf9Rle5K3LJV/P9Eo7AVBss40rZ5csQml1o=,tag:5ql+BvXOYlgOgbNDl5o/vw==,type:str]",
+		"version": "3.13.1"
+	}
+}

vars/per-machine/control/user-password-root/user-password/secret

@@ -0,0 +1,18 @@
+{
+	"data": "ENC[AES256_GCM,data:TyTbzOZ9AOAaP5e96sleUEuuSq/FPjbfHZ6aeR5iOwvDtoc8ZhLYXEO+AF1p+DKS5hDA,iv:9VgMtKiHt8oYnNY8E9+bodU5ei3riw+Gx2KiOQBhoyk=,tag:zJXsCVITlqqrw72QqE/coQ==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBDdWJBOEkwM01BZDlMVS9W\nekFmT0JuMnZHNnBBOVhYK05Ra3VzMW9kK0ZrCkkzRk1JZHdzVlhBeEtTamNHb21x\nU1RkaDZYVEVFV1pyL0lSb1JXeWN6ZDgKLS0tIEVqWDZwZ1RBSS8rVVJLM1pSVkRY\nNEVub1diUytCUHZ1c3NPaWlCUHFYc2MK2lqzBTaw9qIFB6cucfxbMQGhLfSyHYhi\nRJf+pwI0BehAteT87IAbA63Sdx82QNb21mU1AYHPxLdDdrrHWnFxJA==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBOUhYNXIz\nMU9VU3ZMN0NVT1JSZ2N0Q1hxS2dnZm5iY1Z2WUtsaWlXVks0MApxbEtmWWVmQlZ5\nNzZrLy85M1ZiSlc1Y01OM1VEYy9rZ2FhdU5uNTFPa3U0Ci0tLSBkM0ppSUJtMkVP\nWC9jN0NOam9iSy9xakNhOUN3NjBtVllvUU13Tlo4RUdNCtY/2d3ANDMh2Sx1aFlm\n/b3GKbXeeBeeHuFBAsQmuZXqb5EHGJsB9JpjsR0TJJVflCgYYSjn0L1xg9DG/ANa\nn1k=\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa"
+			}
+		],
+		"lastmodified": "2026-07-28T10:15:43Z",
+		"mac": "ENC[AES256_GCM,data:hVEI3iNwXa0O9ZkZWzb0OP8tnsTXWB0znLPg6FIx7Z7D9yPSwCpVu4qdtLwFnB6SEk1tnFykQofCKJ+UQtf6AuziGymwONUQWwUW7pF3zoKtxJ4bJQEKAVChgJqKx7GTp5HXxFkbrp03USJqaHqE9M0leRWIamdBeyXI9rdX9OE=,iv:dn4BqTra2ljHmXfoEoVFwRXXtwiLOs3j5fRNbvndIgA=,tag:zOaRMxl23pJRIvJo3ettEg==,type:str]",
+		"version": "3.13.1"
+	}
+}

vars/per-machine/gw-cnx-1/dns-acme-gw-cnx-1-rfc2136/rfc2136.env/groups/admins

@@ -0,0 +1 @@
+../../../../../../sops/groups/admins

vars/per-machine/gw-cnx-1/dns-acme-gw-cnx-1-rfc2136/rfc2136.env/machines/gw-cnx-1

@@ -0,0 +1 @@
+../../../../../../sops/machines/gw-cnx-1

vars/per-machine/gw-cnx-1/dns-acme-gw-cnx-1-rfc2136/rfc2136.env/secret

@@ -0,0 +1,22 @@
+{
+	"data": "ENC[AES256_GCM,data:4OPDbpmYb7nhHbUvu8dYmF7B/LipnKNE5g8vE33LMzUVg5S+EAyRu2u9dbeUDayqq+ms8WT/oqN/Q0hjsE67+3PCEYBbeAgb5tibsn3tmh1Erp0QUlA4V/erhDwvH+Y8GA3UEJITQ1TtPhCSCKnQbGm85+Tir0nsocjQi3g9Q67TRqUAF7US8H+MVM+YEnIE7HcUH/LKc9z3kg43AaMa2Ey83PjmkMmRdQ==,iv:3VdMulw2DUIReb2werCE4+VLMlxn4FeVUwl8R9Bn3Vw=,tag:TZJZcRiBSCjeTzd10a3rHA==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBBTXlXNzBJRisyeVM3YmRO\nK3IyTTFOcW5VVCtJcnVySFU2RE9PdUdpeWpJCnpscGhsakZkL1hDenhXUW1yY3JH\naHFzUS8velQxV3lYc2hPL0FWcWRpYUEKLS0tIE9BckgrUVRKM1hOWGg0N0V2Z2ds\nUzlhZ0luWitPS3U0UHpWQzZwdG1temMKf5SKmUh3LYD50tObR5tfEvNRoVegg/Tu\n4jXlMKNVg7yOgXET3CNS/Sm4ork4Wz4bSA6gYhLD5JIW356d5WBO4w==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1g8a6fqhqg8q8qu4jyuhxc7hf00t5qvra55chyfm0qk45wlmvc9qs2cft8t"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAybzRIVjhDUHZBVHFSTDF1\nZUNScTRoZzFzdkRISFMzZ0E3M0ZWeFF3bncwCi9rbzg2bUV0OVhBNlROQlhSQnpF\nSm9mVkF2MzhycnNqRUwvTFV2bW5QRWsKLS0tIHJUc2NyOVprMzBjOTgyM1IvZDU2\na0RSenNHYXNqSzR1K0JRVVBpMlpZN0kKk63q9i5zCP3lAZ+JiknOgFcp/quFGYdp\nzoAmkEeGLK8nR5lOVdUutHktG9Rneci+7Q49bk2S4gqeLDt2AnOCsw==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBcXRoTXMy\nOG01R2hEY3R2NTd1bzdPTVUvOWorN1NOYlVLYWk3VThtUEpoUgpzOHNYN1N0bzlv\naWhDVCs0b1ozQ0FBREpiVXpiTkxOeEl5b3YzdkR5YXowCi0tLSBBbTZUZDVpN0I0\ndWRPUXdVaGx5Mk56UzFUbzBvT0xBa0h3dFdmajNXak9RCmDpIFatc4U+k3NDDw7m\nn/D+azJwB6KS2s9YeUy85RoClEqiNSGC9t7qqffVFdiNS8Zww6MTzY++lgj1bYlK\nLMc=\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa"
+			}
+		],
+		"lastmodified": "2026-07-31T03:07:42Z",
+		"mac": "ENC[AES256_GCM,data:aXgLJn6HmSS01GjwYi/lpDil2/FmQitKZOZHUEr1k1r31gTb87RYVJd11MXuGiCCtc5ASKsbZIxoRuPf9DLiiJfuCaVY6Sv4fH/JXkSaLh9YoRCPeX7lrcyYiqficAhmC6+0D3Y0e9tpLSuEdaNB9wHwvslEZ4LSkVscTPSM8VQ=,iv:eY9+6mPJaVDNkZH4LvPBzggEHtyj4H/3QPfN0cjJdx4=,tag:iZyexO/rlt/8XOJ9yR4zgA==,type:str]",
+		"version": "3.13.1"
+	}
+}

vars/per-machine/gw-cnx-1/dns-acme-gw-cnx-1-rfc2136/rfc2136.env/users/berwn

@@ -0,0 +1 @@
+../../../../../../sops/users/berwn

vars/per-machine/gw-cnx-1/emergency-access/password-hash/value

@@ -0,0 +1 @@
+$y$j9T$tSEgEho.KxGPAaQTjU03W1$M6aXoOlLF6B/QaH5XZl/kAGxuC5VClCUpYlr9gOCJt/

vars/per-machine/gw-cnx-1/emergency-access/password/groups/admins

@@ -0,0 +1 @@
+../../../../../../sops/groups/admins

vars/per-machine/gw-cnx-1/emergency-access/password/secret

@@ -0,0 +1,19 @@
+{
+	"data": "ENC[AES256_GCM,data:WtA3B4lfAERXHgc6hyjaBdFxNMlW26VeAYVb6qosXuFGxwqzbf7XXrEWJ0CSFhb5CSJO0SFJ,iv:ATeO9O1uZ3M40Y8q6Udi5qhmGsjSR7GxIRCpGnls8bg=,tag:NKiLATUjXWVJ2SXNI984Ag==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBOWVrRjh2\nZjgxSHdYeUFkRFVLRjc0WVZPNjFoU0wyMnB3cDU3ODJLeU4rQwpxdXRLOWtuMEhx\nR0NROE5xeHdBL1NhQ3NrSlQ5eUE0OGcrcGt6NGdRdTRvCi0tLSA2aDdydHB2TWpR\nQ09kakg3V09iNHpvdVYxck9DeVgwMFhMZ2g5cUxyWXo0CleMojWF4rB5GlAAM1D1\nKEES65mAa/3GPM0vHhd/YBwwyzyv4HeZHSVao5O10FzMWc226Rc029nhNwcTAlA7\nphU=\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB5aFlkMktXRlIxZE8xTXNL\nRzFzWisrQkxEYXJ4STU5VkxZN2d0QjN2V0hrCnZxalRXZno3bk9TUG9oWFhnVkZM\nT3VLK1lLMWNtaHZ3Qk5odm9rRXFRTVEKLS0tIGd2K3ZNZUxoQXROVng0b1c0UU1r\nUWJxWnpCeWxEK3A2SnNBWjAxczZkeWcKOl+DtJQkkKaS1YYaDzqmasnxiY8iEuAd\n57f0zP+peA9Q4U3TZg3UGG94V9kuweNMTWPfj1GZvwBvZP9sZYuFYw==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06"
+			}
+		],
+		"lastmodified": "2026-07-28T09:03:14Z",
+		"mac": "ENC[AES256_GCM,data:8lemWJvcdPy0sg0K+Zn9A7xWd5X4q3XGTb+R5DiJMSo8BWxpP+QGNqxEuni8sNxP2d0bkrsEI3vG89lx37x3iot667Gds0qAaHq7h8l+3iwAvGMXAEzwIms9GLdIcJiUWYlFiAcicfUINWtPQ+adxACFzTOBIBEGqD6iQ5B+KsM=,iv:OuBeq83CopTNeJJM/UHUrI7RKIGVJ3tM+DcckuFxLBc=,tag:JAXpfqwSDRr4e58wiJeMNQ==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.13.1"
+	}
+}

vars/per-machine/gw-cnx-1/emergency-access/password/users/berwn

@@ -0,0 +1 @@
+../../../../../../sops/users/berwn

vars/per-machine/gw-cnx-1/openssh-cert/.validation-hash

@@ -0,0 +1 @@
+210641414b2391ee4659814fb67a1869d8f1920716b53f7b5226be36887ec7ce

vars/per-machine/gw-cnx-1/openssh-cert/ssh.id_ed25519-cert.pub/value

@@ -0,0 +1 @@
+ssh-ed25519-cert-v01@openssh.com AAAAIHNzaC1lZDI1NTE5LWNlcnQtdjAxQG9wZW5zc2guY29tAAAAICg/CnZm/jIFijKdkXTrJS2Fman7vhPtUcpTFVhLAoyLAAAAIP+kt1X3TTJhWQYakYQzrr2lI8tanZ74uEahnD4meuCaAAAAAAAAAAAAAAACAAAACGd3LWNueC0xAAAAIQAAAB1ndy1jbngtMS5jbngtbmV0d29yay5pbnRlcm5hbAAAAAAAAAAA//////////8AAAAAAAAAAAAAAAAAAAAzAAAAC3NzaC1lZDI1NTE5AAAAIJuhbAaSx5fuaqTYZW0zajVY9HOHxTdfoDfChqdgb3GhAAAAUwAAAAtzc2gtZWQyNTUxOQAAAEAhM9fTE9iCnV1wGKunJcRqOmNWbu0iZjPqXNcPp8rpwFJm/8CW0qC/TV/H5im7sX68K+ZOOaOTuOqu+D6U+lIA /tmp/vars-gc02c4xe/in/openssh/ssh.id_ed25519.pub

vars/per-machine/gw-cnx-1/openssh/ssh.id_ed25519.pub/value

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIP+kt1X3TTJhWQYakYQzrr2lI8tanZ74uEahnD4meuCa 

vars/per-machine/gw-cnx-1/openssh/ssh.id_ed25519/groups/admins

@@ -0,0 +1 @@
+../../../../../../sops/groups/admins