Add mx1 to secret vars/shared/mail-dmarc-cred/passphrase

Deliver cnx.email DMARC aggregate/forensic reports to a dedicated dmarc@cnx.email
mailbox on mx1 and analyze them with parsedmarc on control, storing parsed
reports in a local loopback Elasticsearch and visualizing via the auto-provisioned
Grafana dashboard. parsedmarc fetches the mailbox over IMAPS across the mesh
(mx1.cnx.email pinned to its mesh address so TLS still validates), using a shared
mail-dmarc-cred clan var so mx1's mailserver and control see the same password.

TODO.md

@@ -0,0 +1,69 @@
+# Infra roadmap
+
+Prioritized backlog for the cnx-network clan. See `docs/` for how the current
+pieces work.
+
+## 1. Alerting (done — pending deploy)
+
+Rules evaluated by vmalert against VictoriaMetrics on control, declared in
+`modules/monitoring/alerts.nix`:
+
+- [x] SOA serial divergence between ns1 and ns2 (secondary out of sync)
+- [x] Zone-expiry countdown on the secondary approaching zero (transfers failing)
+- [x] Any scrape target down (`up == 0`)
+- [x] Root filesystem nearly full
+
+Delivery stays minimal for now (`notifier.blackhole`): vmalert remote-writes
+alert state back to VM, so firing alerts show up as the `ALERTS` series in
+Grafana. Wiring a real notifier (Matrix) is a later step — drop `blackhole` and
+set `settings."notifier.url"` to an Alertmanager.
+
+## 2. Backups of critical state (DNSSEC done — pending vars + deploy)
+
+clan `borgbackup` instance in `clan.nix`: control is the server (repos under
+`/var/lib/borgbackup/<client>`), ns1 the client. ns1 declares
+`clan.core.state.knot.folders = [ "/var/lib/knot" ]`, so the Knot KASP keystore
+is backed up nightly (01:00) over the mesh with repokey encryption — control
+never holds plaintext. ns1 maps the `control` machine name to its mesh IP via
+`networking.hosts` so the `borg@control` repo resolves.
+
+Before deploy: `clan vars generate ns1` (YubiKey) to mint the borgbackup ssh
+keypair + repokey; control won't evaluate until ns1's public key exists. Then
+deploy ns1 and control.
+
+- [x] DNSSEC key material on ns1 (KSK/ZSK in Knot's KASP store) — losing it forces
+      an emergency DS rollover at the registrar
+- [ ] VictoriaMetrics TSDB on control (optional, retention is 180d) — deferred;
+      regenerable over time and control is the backup server, so this needs a
+      second client→server pair (e.g. control→ns2) rather than the same topology
+
+## 3. Blackbox DNS probing (done — pending deploy)
+
+`blackbox_exporter` on control (loopback `:9115`), probing each nameserver's
+public v4+v6 address for every zone: an SOA query (zone served?) and a DNSKEY
+query (still signed?). Blackbox has no DO-bit option, so signing is checked by
+asking for DNSKEY directly and asserting the RRset is present. Probe defs live
+in `modules/monitoring/blackbox-probes.nix`, shared by the exporter
+(`blackbox.nix`) and the VM scrape jobs (`server.nix`). Verified live against
+ns1/ns2: SOA + DNSKEY succeed on both servers over v4 and v6.
+
+- [x] `blackbox_exporter` on control doing real DNS + DNSSEC-validation queries
+      against ns1/ns2 — catches outside-in resolution failures the Knot stats miss
+- [x] paired with alerts (`DNSResolutionProbeFailed` / `DNSSECProbeFailed` in
+      `alerts.nix`) and a "DNS probes (outside-in)" row on the CNX DNS dashboard
+
+## 4. Third secondary off Hetzner (resilience)
+
+- [ ] A secondary nameserver on a different provider/network so a single-provider
+      outage doesn't take all authoritative DNS down (architectural — new machine)
+
+## 5. Centralized logs (done — pending deploy)
+
+VictoriaLogs on control (`:9428`, 30d retention, mesh-scoped) in
+`modules/monitoring/server.nix`, plus a VictoriaLogs Grafana datasource. All
+three hosts ship journald with systemd's own `services.journald.upload` to the
+`/insert/journald` endpoint (`modules/monitoring/exporters.nix`) — no extra
+agent. control uploads over loopback; ns1/ns2 over the mesh.
+
+- [x] VictoriaLogs on control to grep journald across all three hosts, pairing
+      with the existing VictoriaMetrics setup

clan.nix

@@ -1,3 +1,19 @@
+let
+  hosts = import ./modules/hosts.nix;
+
+  # This clan-core pins the zerotier `allowedIps` interface (admit by network
+  # IPv6), but node IDs are the stable per-device handle (what `zerotier-cli
+  # info` prints). Derive a member's IP on THIS network from the controller's
+  # network id so external members can be listed by node id, as below.
+  ztNetworkId = builtins.readFile ./vars/per-machine/control/zerotier/zerotier-network-id/value;
+  ztMemberIp =
+    nodeId:
+    let
+      full = "fd" + ztNetworkId + "9993" + nodeId;
+      h = i: builtins.substring (i * 4) 4 full;
+    in
+    "${h 0}:${h 1}:${h 2}:${h 3}:${h 4}:${h 5}:${h 6}:${h 7}";
+in
 {
   # Ensure this is unique among all clans you want to use.
   meta.name = "cnx-network-clan";
@@ -7,6 +23,8 @@
     control = { };
     ns1 = { };
     ns2 = { };
+    mx1 = { };
+    web01 = { };
   };
 
   inventory.instances = {
@@ -23,16 +41,41 @@
     zerotier = {
       roles.controller.machines."control" = { };
       roles.peer.tags.all = { };
+      # External members admitted by ZeroTier node id (stable per device).
+      # Inventory machines are auto-accepted; this is only for peers outside the
+      # clan. Node id comes from `zerotier-cli info` on the joining device.
+      roles.controller.settings.allowedIps = map ztMemberIp [
+        "8802c8d7e0" # alex-nixos
+        "2bd36db8cc" # kurogeek-thinkpad
+      ];
     };
 
     tor = {
       roles.server.tags.nixos = { };
     };
 
+    # Direct SSH to public IPs — clan's priority-1 connection path, with the
+    # ZeroTier mesh and Tor kept as automatic fallbacks. Raw IPs (from
+    # modules/hosts.nix, not the ns1/ns2 DNS names) so reaching these hosts never
+    # depends on their own DNS being up.
+    internet.roles.default.machines = builtins.mapAttrs (_: h: {
+      settings.host = h.ipv4;
+    }) hosts;
+
     # Recovery root password for console access when a machine fails to boot.
     emergency-access = {
       roles.default.tags.nixos = { };
     };
+
+    # Encrypted, deduplicating backups. control hosts the repos; ns1 is the
+    # only client, backing up its declared clan.core.state (the Knot DNSSEC
+    # keystore) over the mesh. Repo lives at /var/lib/borgbackup/ns1 on control.
+    # Cross-host so an ns1 loss is recoverable; repokey encryption means control
+    # never holds plaintext. Run `clan vars generate ns1` (YubiKey) before deploy.
+    borgbackup = {
+      roles.server.machines.control = { };
+      roles.client.machines.ns1 = { };
+    };
   };
 
   machines = {

docs/book.toml

@@ -0,0 +1,12 @@
+[book]
+title = "CNX Infra Runbook"
+description = "Operational docs for the cnx-network clan: DNS, ZeroTier mesh, monitoring."
+authors = ["B4L"]
+src = "src"
+language = "en"
+
+[output.html]
+default-theme = "navy"
+preferred-dark-theme = "navy"
+git-repository-url = "https://git.b4l.co.th/B4L/cnx-network-clan"
+edit-url-template = "https://git.b4l.co.th/B4L/cnx-network-clan/_edit/main/docs/{path}"

docs/src/SUMMARY.md

@@ -0,0 +1,7 @@
+# Summary
+
+- [Overview](./overview.md)
+- [ZeroTier mesh](./mesh.md)
+- [DNS](./dns.md)
+- [Monitoring](./monitoring.md)
+- [Backups](./backups.md)

docs/src/backups.md

@@ -0,0 +1,61 @@
+# Backups
+
+Encrypted, deduplicating backups via clan's `borgbackup` service, declared in
+`clan.nix`. The only critical, non-regenerable state is the **Knot DNSSEC
+keystore** on `ns1` (the KSK/ZSK private keys under `/var/lib/knot`); losing it
+forces an emergency DS rollover at the registrar.
+
+## Topology
+
+- **control** is the borgbackup **server** — it hosts the repos under
+  `/var/lib/borgbackup/<client>` (so `ns1`'s repo is `/var/lib/borgbackup/ns1`).
+- **ns1** is the **client**. It backs up everything it declares as clan state
+  (`clan.core.state.knot.folders = [ "/var/lib/knot" ]`) once a day at 01:00,
+  over the ZeroTier mesh.
+
+The backup is cross-host so that losing `ns1` is recoverable, and stays
+self-contained (no third-party storage). Encryption is `repokey` with a
+generated passphrase, so `control` only ever stores ciphertext.
+
+Mesh peers have no name resolution, so `ns1` maps the `control` machine name to
+its ZeroTier address via `networking.hosts`; that is how the `borg@control` repo
+URL resolves.
+
+## Secrets
+
+The borgbackup ssh keypair and repokey passphrase are clan vars, generated once
+(needs the YubiKey). `control` will not evaluate until `ns1`'s public key
+exists, so generate before the first deploy:
+
+```
+clan vars generate ns1
+clan machines update ns1
+clan machines update control
+```
+
+## Operating
+
+Backups are driven by systemd on `ns1` (`borgbackup-job-control.timer`).
+
+```
+# trigger a backup now (on ns1)
+borgbackup-create
+
+# list archives (on ns1)
+borgbackup-list
+
+# restore selected folders from an archive (on ns1)
+NAME='<archive-name>' FOLDERS=/var/lib/knot borgbackup-restore
+```
+
+Retention is pruned automatically: all archives from the last day, then 7 daily
+and 4 weekly.
+
+## Monitoring
+
+The **CNX Backups** Grafana dashboard
+(`modules/monitoring/dashboards/backups.json`) tracks job health, time since the
+last successful run, and per-job state — all from the node_exporter systemd
+collector on the client. There is no dedicated borg metrics exporter; the unit
+state and the timer's last-trigger timestamp are enough to catch a backup that
+stops running or fails.

docs/src/dns.md

@@ -0,0 +1,95 @@
+# DNS
+
+Authoritative DNS for three zones, served by Knot:
+
+- `cnx.network`
+- `buildfor.life`
+- `cnx.email`
+
+Add a zone in `modules/dns/domains.nix` **and** drop a matching `<domain>.zone`
+file in `modules/dns/zones/`.
+
+## Primary / secondary
+
+- **`ns1` = primary (master).** Loads each zone from its file, signs it, and
+  notifies `ns2`. Config in `machines/ns1/configuration.nix`.
+- **`ns2` = secondary (slave).** Pulls every zone from `ns1` (AXFR/IXFR) and
+  accepts its NOTIFY. Config in `machines/ns2/configuration.nix`.
+
+Zone transfers run **over the ZeroTier mesh**, authenticated with a shared TSIG
+key (`dns-tsig`, a clan var copied to both machines).
+
+## Serial handling
+
+`ns1` uses `zonefile-load = difference-no-serial` with `serial-policy = unixtime`:
+edit records without touching the SOA serial — Knot diffs the file, assigns a
+strictly-monotonic unixtime serial, signs, and transfers. `journal-content = all`
+holds the live signed zone (required by `difference-no-serial`).
+
+## DNSSEC
+
+Automatic signing on `ns1` only, policy `cnx`: ECDSA P-256/SHA-256. The ZSK
+auto-rolls; the KSK is kept stable, so the DS at the registrar only changes on a
+manual KSK rollover.
+
+### Registrar DS records
+
+Knot manages all key material itself on `ns1` (the only signer); the KSK/ZSK
+private keys live in the KASP keystore under `/var/lib/knot` (backed up nightly —
+see [Backups](./backups.md)). You never touch the private keys directly.
+
+What a registrar needs is the **DS record** for a zone's KSK, which anchors the
+zone into the parent's chain of trust. Generate it on `ns1` — the `keymgr` wrapper
+is already pointed at Knot's config, and it runs as the `knot` user that owns the
+keystore:
+
+```
+sudo -u knot keymgr <zone> ds
+```
+
+e.g. `sudo -u knot keymgr cnx.email ds`. Paste the printed DS record (key tag,
+algorithm 13, digest type, digest) into the registrar's DNSSEC form for that
+domain. Repeat per signed zone (`cnx.network`, `buildfor.life`, `cnx.email`) at
+whichever registrar holds each delegation. After submitting, confirm the parent
+publishes it with `dig +short DS <zone>`.
+
+The ZSK rolls automatically and needs no registrar action; only a **KSK rollover**
+requires re-submitting the DS.
+
+> **Pending (manual):** submit DS records for `buildfor.life` and `cnx.email`
+> once they're at a DNSSEC-capable registrar.
+
+## ACME DNS-01
+
+Certificates are issued by `_acme-challenge` TXT updates that `ns1` accepts over
+TSIG, signs, and transfers to `ns2` (which never needs these keys). Each consumer
+gets its **own** key, scoped by an ACL to exactly the owner names it needs and
+attached only to the zone it lives in — so a leaked key can write nothing but its
+own challenges.
+
+- **`acme_ddns`** (`acl_acme`) — the general key, scoped to `TXT` at or under
+  `_acme-challenge.<zone>` and attached to every zone. Client config:
+  ```
+  clan vars get ns1 dns-acme-tsig/acme.conf
+  ```
+- **`acme_mx1`** (`acl_acme_mx1`) — held only by `mx1`, scoped to
+  `_acme-challenge.{mx1,mta-sts,mail}` and attached only to `cnx.email` (the mail
+  cert plus its MTA-STS and client-alias SANs). Secret shared via the
+  `dns-acme-mx1-secret` generator.
+- **`acme_web01`** (`acl_acme_web01`) — held only by `web01`, scoped to
+  `_acme-challenge` and attached only to `cnx.network` (where the wildcard
+  `*.cnx.network` challenge lands, at the apex). Secret shared via the
+  `dns-acme-web01-secret` generator.
+
+## Runbook: stale secondary
+
+If `ns2` serves stale records while SOA serials match (e.g. after a manual zone
+edit that didn't bump the serial as expected), force a fresh transfer on `ns2`:
+
+```
+knotc zone-retransfer <zone>
+```
+
+Watch the **CNX DNS** Grafana dashboard: the per-nameserver SOA serial table
+should agree across `ns1`/`ns2`, and "seconds until zone expiry" on the secondary
+should reset on each successful transfer rather than counting toward zero.

docs/src/mesh.md

@@ -0,0 +1,39 @@
+# ZeroTier mesh
+
+A private IPv6 overlay that every machine (and admin laptops) shares. DNS zone
+transfers and metrics scraping ride this mesh, never the public net.
+
+- **Controller:** `control` (the `zerotier` instance in `clan.nix`).
+- **Peers:** every machine (`roles.peer.tags.all`).
+- **Prefix:** `fd06:1bad:ece2:92ad:ba99:9300::/88` (RFC 4193: `fd` + network id + `0x9993`).
+
+## The mesh map
+
+`modules/mesh-hosts.nix` does **not** hardcode addresses. It reads each machine's
+IP from the public clan vars that clan-core's zerotier generator already writes
+(`vars/per-machine/<m>/zerotier/zerotier-ip/value`) and derives the `/88` subnet
+from `control`'s `zerotier-network-id`. Regenerate or re-key a node and the map
+follows automatically.
+
+Consumers: `modules/dns/authoritative.nix` (transfer ACLs), `modules/monitoring/*`
+(scrape targets and firewall scoping).
+
+## Admitting external members
+
+Inventory machines are auto-accepted. External devices (admin laptops) are listed
+in `clan.nix` under the controller's `allowedIps`. Because this clan-core pins the
+`allowedIps` interface (admit by network IPv6), we keep a **node-id** list and a
+`ztMemberIp` helper derives each device's IP on this network:
+
+```nix
+roles.controller.settings.allowedIps = map ztMemberIp [
+  "8802c8d7e0" # alex-nixos
+  "2bd36db8cc" # kurogeek-thinkpad
+];
+```
+
+A device's 10-char node id comes from `zerotier-cli info` on that device. After
+editing, deploy `control`; the controller admits the new member on its next run.
+
+> A newer clan-core exposes `allowedIds` (admit by node id directly), but adopting
+> it means a zerotier vars-schema migration, so we stay on the IP-derivation path.

docs/src/monitoring.md

@@ -0,0 +1,101 @@
+# Monitoring
+
+Metrics and logs live on `control` over the ZeroTier mesh; the Grafana dashboards
+are also published publicly through `web01` (see [Dashboards](#dashboards)).
+
+## Collection
+
+- **node_exporter** (`:9100`) on every machine — CPU, memory, disk, systemd units.
+  Binds all interfaces; the scrape ports are firewall-scoped to the mesh subnet
+  (`modules/monitoring/exporters.nix`).
+- **knot-exporter** (`:9433`) on `ns1`/`ns2` only — reads Knot's control socket,
+  fed by the `mod-stats` module (query/response counters per zone).
+- **blackbox_exporter** (`127.0.0.1:9115`) on `control` only — outside-in DNS
+  probes. For every zone it queries each nameserver's **public** address (v4 and
+  v6) for SOA (is the zone served?) and DNSKEY (is it still signed?). This is the
+  resolver's-eye view that the Knot stats can't see. Probe definitions are shared
+  between the exporter and the scrape jobs in `modules/monitoring/blackbox-probes.nix`.
+
+## Storage & scraping
+
+**VictoriaMetrics** on `control`, bound to `127.0.0.1:8428`, 180-day retention
+(`modules/monitoring/server.nix`). It scrapes `control` over loopback and
+`ns1`/`ns2`/`mx1`/`web01` over the mesh.
+
+> The scraper dials IPv4-only by default, so mesh (IPv6) targets need
+> `extraOptions = [ "-enableTCP6" ]`. Without it, ns1/ns2 are dropped with
+> "no suitable address found". Check live target health on `control`:
+>
+> ```
+> curl -s http://127.0.0.1:8428/api/v1/targets | jq '.data.activeTargets[] | {i:.labels.instance, h:.health, e:.lastError}'
+> ```
+
+## Dashboards
+
+**Grafana** on `control` (`:3000`), anonymous access disabled. Reachable directly
+over the mesh, and publicly at `https://grafana.cnx.network` via `web01`'s reverse
+proxy (TLS termination — see [Overview](./overview.md)). The admin password is a
+clan var:
+
+```
+clan vars get control grafana-admin/password
+```
+
+Dashboards are provisioned from `modules/monitoring/dashboards/` (any JSON file
+there is picked up):
+
+- **CNX DNS** (`dns.json`) — firing alerts, per-nameserver SOA serials, zone
+  expiry countdowns, query/response rates, host CPU/memory/disk/load, and the
+  outside-in DNS probes.
+- **CNX Backups** (`backups.json`) — borgbackup job health, time since the last
+  run, and per-job state. See [Backups](./backups.md).
+- **CNX Uptime** (`uptime.json`) — per-host up/down status, current uptime,
+  availability over the selected window, and up/down history. Label-driven, so
+  every scraped host appears automatically.
+- **parsedmarc** — DMARC aggregate/forensic report viewer. Auto-provisioned by
+  the `parsedmarc` module (not from `dashboards/`); reads its own Elasticsearch
+  datasource, not VictoriaMetrics. See [DMARC reports](#dmarc-reports) below.
+
+## DMARC reports
+
+The `cnx.email` DMARC record (`rua`/`ruf`) points at the `dmarc@cnx.email`
+mailbox on `mx1`. **parsedmarc** on `control` (`modules/monitoring/parsedmarc.nix`)
+polls that mailbox over IMAPS, parses the XML reports, and stores them in a local
+**Elasticsearch** (`127.0.0.1:9200`, loopback-only); Grafana renders them via the
+auto-provisioned parsedmarc dashboard + Elasticsearch datasource.
+
+The IMAP fetch rides the **mesh**, not the public net: `control` pins
+`mx1.cnx.email` to mx1's mesh address in `/etc/hosts`, so TLS still validates
+against the public cert while the bytes stay on the overlay. The mailbox
+passphrase is the shared `mail-dmarc-cred` clan var (so both mx1's mailserver and
+control's parsedmarc see the same value):
+
+```
+clan vars get mx1 mail-dmarc-cred/passphrase
+```
+
+## Logs
+
+**VictoriaLogs** on `control` (`:9428`), 30-day retention
+(`modules/monitoring/server.nix`). All three hosts ship journald to it via
+systemd's own `services.journald.upload` → the `/insert/journald` endpoint
+(`modules/monitoring/exporters.nix`); no extra agent. `control` uploads over
+loopback so its logs survive a mesh outage, the other hosts push over the mesh,
+and 9428 is firewall-scoped to the mesh like everything else.
+
+> Same IPv4-only default as the scraper: VictoriaLogs binds `0.0.0.0:9428` for a
+> bare `:9428`, so mesh (IPv6) pushes from ns1/ns2 are refused until you pass
+> `extraOptions = [ "-enableTCP6" ]` (binds `[::]`). Verify the bind on `control`:
+>
+> ```
+> ss -tlnp | grep 9428   # want [::]:9428, not 0.0.0.0:9428
+> ```
+
+Query logs from Grafana via the provisioned **VictoriaLogs** datasource (Explore
+view, LogsQL), or directly in the built-in UI at `http://[control]:9428/select/vmui`.
+Logs are tagged with `_HOSTNAME` and `_SYSTEMD_UNIT`, so to follow one service
+across hosts:
+
+```
+_SYSTEMD_UNIT:"knot.service"
+```

docs/src/overview.md

@@ -0,0 +1,28 @@
+# Overview
+
+This is the operational runbook for the **cnx-network** clan. Everything here is
+managed declaratively from the [clan repo](https://git.b4l.co.th/B4L/cnx-network-clan);
+this book is built from `docs/` and served on `control` over the ZeroTier mesh.
+
+## Machines
+
+| Machine   | Role                                   | Public IPv4      | Public IPv6             |
+| --------- | -------------------------------------- | ---------------- | ----------------------- |
+| `control` | ZeroTier controller, monitoring, docs  | `77.42.68.181`   | `2a01:4f9:c013:e6d0::1` |
+| `ns1`     | Knot DNS **primary** (master)          | `46.224.170.206` | `2a01:4f8:c014:b5c5::1` |
+| `ns2`     | Knot DNS **secondary** (slave)         | `157.180.70.82`  | `2a01:4f9:c014:6d87::1` |
+| `mx1`     | Mail server (**MX** for cnx.email)     | `5.223.65.38`    | `2a01:4ff:2f0:1963::1`  |
+| `web01`   | Public reverse proxy (TLS termination) | `5.223.55.246`   | `2a01:4ff:2f0:2d8f::1`  |
+
+## Access
+
+- Admin SSH and all internal services ride the **ZeroTier mesh**, not the public
+  net. Public SSH (22) is intentionally closed at the Hetzner cloud firewall.
+- clan reaches machines by their public IPs first (the `internet` instance), with
+  the mesh and Tor as automatic fallbacks.
+
+## Editing these docs
+
+Commit-to-edit: change the markdown under `docs/src/`, commit, and redeploy
+`control`. There is no in-browser editor by design — the docs are versioned and
+reviewed alongside the config that they describe.

flake.lock

@@ -1,5 +1,21 @@
 {
   "nodes": {
+    "blobs": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1604995301,
+        "narHash": "sha256-wcLzgLec6SGJA8fx1OEN1yV/Py5b+U5iyYpksUY/yLw=",
+        "owner": "simple-nixos-mailserver",
+        "repo": "blobs",
+        "rev": "2cccdf1ca48316f2cfd1c9a0017e8de5a7156265",
+        "type": "gitlab"
+      },
+      "original": {
+        "owner": "simple-nixos-mailserver",
+        "repo": "blobs",
+        "type": "gitlab"
+      }
+    },
     "clan-core": {
       "inputs": {
         "data-mesher": "data-mesher",
@@ -73,6 +89,22 @@
         "type": "github"
       }
     },
+    "flake-compat": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1767039857,
+        "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
     "flake-parts": {
       "inputs": {
         "nixpkgs-lib": [
@@ -94,6 +126,54 @@
         "type": "github"
       }
     },
+    "git-hooks": {
+      "inputs": {
+        "flake-compat": [
+          "nixos-mailserver",
+          "flake-compat"
+        ],
+        "gitignore": "gitignore",
+        "nixpkgs": [
+          "nixos-mailserver",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1772893680,
+        "narHash": "sha256-JDqZMgxUTCq85ObSaFw0HhE+lvdOre1lx9iI6vYyOEs=",
+        "owner": "cachix",
+        "repo": "git-hooks.nix",
+        "rev": "8baab586afc9c9b57645a734c820e4ac0a604af9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "git-hooks.nix",
+        "type": "github"
+      }
+    },
+    "gitignore": {
+      "inputs": {
+        "nixpkgs": [
+          "nixos-mailserver",
+          "git-hooks",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1709087332,
+        "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=",
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "rev": "637db329424fd7e46cf4185293b9cc8c88c95394",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "type": "github"
+      }
+    },
     "nix-darwin": {
       "inputs": {
         "nixpkgs": [
@@ -144,6 +224,30 @@
         "type": "github"
       }
     },
+    "nixos-mailserver": {
+      "inputs": {
+        "blobs": "blobs",
+        "flake-compat": "flake-compat",
+        "git-hooks": "git-hooks",
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1773912645,
+        "narHash": "sha256-QHzRqq6gh+t3F/QU9DkP7X63dDDcuIQmaDz12p7ANTg=",
+        "owner": "simple-nixos-mailserver",
+        "repo": "nixos-mailserver",
+        "rev": "25e6dbb8fca3b6e779c5a46fd03bd760b2165bb5",
+        "type": "gitlab"
+      },
+      "original": {
+        "owner": "simple-nixos-mailserver",
+        "ref": "nixos-25.11",
+        "repo": "nixos-mailserver",
+        "type": "gitlab"
+      }
+    },
     "nixpkgs": {
       "locked": {
         "lastModified": 1778003029,
@@ -163,6 +267,7 @@
     "root": {
       "inputs": {
         "clan-core": "clan-core",
+        "nixos-mailserver": "nixos-mailserver",
         "nixpkgs": [
           "clan-core",
           "nixpkgs"

flake.nix

@@ -3,6 +3,9 @@
   inputs.nixpkgs.follows = "clan-core/nixpkgs";
   inputs.treefmt-nix.url = "github:numtide/treefmt-nix";
   inputs.treefmt-nix.inputs.nixpkgs.follows = "nixpkgs";
+  # Simple NixOS Mailserver, pinned to the branch matching clan-core's nixpkgs.
+  inputs.nixos-mailserver.url = "gitlab:simple-nixos-mailserver/nixos-mailserver/nixos-25.11";
+  inputs.nixos-mailserver.inputs.nixpkgs.follows = "nixpkgs";
 
   outputs =
     {

fmt.nix

@@ -24,6 +24,7 @@
 
       # No formatter, or reformatting would corrupt them.
       "*.zone" # Knot zone files
+      "docs/book.toml" # mdBook config; no TOML formatter enabled
       "flake.lock"
       ".envrc"
       ".gitignore"

inventory.json

@@ -8,6 +8,12 @@
     },
     "ns2": {
       "installedAt": 1781418857
+    },
+    "mx1": {
+      "installedAt": 1781757322
+    },
+    "web01": {
+      "installedAt": 1781983723
     }
   }
 }

machines/control/configuration.nix

@@ -1,10 +1,27 @@
+{ config, ... }:
+let
+  hosts = import ../../modules/hosts.nix;
+in
 {
   imports = [
     ../../modules/hetzner-firewall.nix
+    ../../modules/static-ipv6.nix
+    ../../modules/monitoring/exporters.nix
+    ../../modules/monitoring/server.nix
+    ../../modules/monitoring/blackbox.nix
+    ../../modules/monitoring/alerts.nix
+    ../../modules/monitoring/parsedmarc.nix
+    ../../modules/docs.nix
   ];
 
   clan.core.sops.defaultGroups = [ "admins" ];
 
+  # Public IPv6 (from modules/hosts.nix); SLAAC doesn't bring it up here.
+  cnx.staticIPv6 = {
+    enable = true;
+    address = hosts.${config.networking.hostName}.ipv6;
+  };
+
   time.timeZone = "Etc/GMT-3"; # UTC+3 (fixed offset, no DST)
   services.timesyncd.enable = true;
 

machines/mx1/configuration.nix

@@ -0,0 +1,23 @@
+{ config, inputs, ... }:
+let
+  hosts = import ../../modules/hosts.nix;
+in
+{
+  imports = [
+    inputs.nixos-mailserver.nixosModules.default
+    ../../modules/mail.nix
+    ../../modules/static-ipv6.nix
+    ../../modules/monitoring/exporters.nix
+  ];
+
+  clan.core.sops.defaultGroups = [ "admins" ];
+
+  # Public IPv6 (from modules/hosts.nix); SLAAC doesn't bring it up here.
+  cnx.staticIPv6 = {
+    enable = true;
+    address = hosts.${config.networking.hostName}.ipv6;
+  };
+
+  time.timeZone = "Etc/GMT-8"; # UTC+8 (Singapore, fixed offset, no DST)
+  services.timesyncd.enable = true;
+}

machines/mx1/disko.nix

@@ -0,0 +1,50 @@
+# ---
+# schema = "single-disk"
+# [placeholders]
+# mainDisk = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_117494657"
+# ---
+# This file was automatically generated!
+# CHANGING this configuration requires wiping and reinstalling the machine
+{
+
+  boot.loader.grub.efiSupport = true;
+  boot.loader.grub.efiInstallAsRemovable = true;
+  boot.loader.grub.enable = true;
+  disko.devices = {
+    disk = {
+      main = {
+        name = "main-5a0919ffeb6044a39b7d44bba8895ff2";
+        device = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_117494657";
+        type = "disk";
+        content = {
+          type = "gpt";
+          partitions = {
+            "boot" = {
+              size = "1M";
+              type = "EF02"; # for grub MBR
+              priority = 1;
+            };
+            ESP = {
+              type = "EF00";
+              size = "500M";
+              content = {
+                type = "filesystem";
+                format = "vfat";
+                mountpoint = "/boot";
+                mountOptions = [ "umask=0077" ];
+              };
+            };
+            root = {
+              size = "100%";
+              content = {
+                type = "filesystem";
+                format = "ext4";
+                mountpoint = "/";
+              };
+            };
+          };
+        };
+      };
+    };
+  };
+}

machines/ns1/configuration.nix

@@ -1,22 +1,48 @@
-{ config, pkgs, ... }:
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
 let
   domains = import ../../modules/dns/domains.nix;
+  mesh = import ../../modules/mesh-hosts.nix { inherit config lib; };
+  hosts = import ../../modules/hosts.nix;
 in
 {
   imports = [
     ../../modules/dns/authoritative.nix
+    ../../modules/dns/acme-mx1-secret.nix
+    ../../modules/dns/acme-web01-secret.nix
+    ../../modules/static-ipv6.nix
+    ../../modules/monitoring/exporters.nix
   ];
 
   clan.core.sops.defaultGroups = [ "admins" ];
 
+  # Knot's state dir holds the non-regenerable DNSSEC key material (KSK/ZSK
+  # private keys in the KASP keystore). Declaring it as clan state makes the
+  # borgbackup client back it up; losing it forces an emergency DS rollover at
+  # the registrar. mode 0700 owned by knot, but borg runs as root so it reads it.
+  clan.core.state.knot.folders = [ "/var/lib/knot" ];
+
+  # The borgbackup repo is addressed as `borg@control`; mesh peers have no name
+  # resolution, so map the control machine name to its ZeroTier mesh address.
+  networking.hosts.${mesh.hosts.control} = [ "control" ];
+
+  # Public IPv6 (from modules/hosts.nix; matches the ns1 AAAA glue); SLAAC
+  # doesn't bring it up here.
+  cnx.staticIPv6 = {
+    enable = true;
+    address = hosts.${config.networking.hostName}.ipv6;
+  };
+
   time.timeZone = "Etc/GMT-1"; # UTC+1 (fixed offset, no DST)
   services.timesyncd.enable = true;
 
-  # ACME DNS-01 (RFC 2136): a dedicated TSIG key, scoped to ns1 only, that an
+  # ACME DNS-01 (RFC 2136), general key. A dedicated TSIG key scoped by acl_acme
-  # external ACME client uses to write _acme-challenge TXT records. acl_acme
+  # (referenced by every zone below) to TXT updates at or under _acme-challenge.
-  # (referenced by each zone below) limits the key to TXT updates at or under
+  # Retrieve the client config with:
-  # _acme-challenge.<zone>; Knot then signs the record and transfers it to ns2,
-  # which never needs this key. Retrieve the secret for the client with:
   #   clan vars get ns1 dns-acme-tsig/acme.conf
   clan.core.vars.generators.dns-acme-tsig = {
     files."acme.conf" = {
@@ -30,8 +56,47 @@ in
     '';
   };
 
+  # ACME DNS-01, dedicated mx1 key. A *separate* TSIG key (acme_mx1) that only
+  # mx1 holds, rendered from the shared secret (generator dns-acme-mx1-secret,
+  # imported above). acl_acme_mx1 scopes it to TXT updates at exactly
+  # _acme-challenge.{mx1,mta-sts,mail} (the mail cert and its MTA-STS + client-
+  # alias SANs), and it is attached only to the cnx.email zone below — so this
+  # credential can write nothing but mx1's own cert challenges.
+  clan.core.vars.generators.dns-acme-mx1-knot = {
+    files."acme.conf" = {
+      secret = true;
+      owner = "knot";
+      group = "knot";
+    };
+    dependencies = [ "dns-acme-mx1-secret" ];
+    script = ''
+      printf 'key:\n  - id: acme_mx1\n    algorithm: hmac-sha256\n    secret: %s\n' \
+        "$(cat "$in"/dns-acme-mx1-secret/secret)" > "$out"/acme.conf
+    '';
+  };
+
+  # ACME DNS-01, dedicated web01 key. A *separate* TSIG key (acme_web01) that only
+  # web01 holds, rendered from the shared secret (generator dns-acme-web01-secret,
+  # imported above). acl_acme_web01 scopes it to TXT updates at _acme-challenge on
+  # the cnx.network zone — the owner the wildcard *.cnx.network challenge uses — so
+  # this credential can write nothing but web01's own cert challenges.
+  clan.core.vars.generators.dns-acme-web01-knot = {
+    files."acme.conf" = {
+      secret = true;
+      owner = "knot";
+      group = "knot";
+    };
+    dependencies = [ "dns-acme-web01-secret" ];
+    script = ''
+      printf 'key:\n  - id: acme_web01\n    algorithm: hmac-sha256\n    secret: %s\n' \
+        "$(cat "$in"/dns-acme-web01-secret/secret)" > "$out"/acme.conf
+    '';
+  };
+
   services.knot.keyFiles = [
     config.clan.core.vars.generators.dns-acme-tsig.files."acme.conf".path
+    config.clan.core.vars.generators.dns-acme-mx1-knot.files."acme.conf".path
+    config.clan.core.vars.generators.dns-acme-web01-knot.files."acme.conf".path
   ];
 
   services.knot.settings.acl = [
@@ -44,6 +109,30 @@ in
       "update-owner-match" = "sub-or-equal";
       "update-owner-name" = [ "_acme-challenge" ];
     }
+    {
+      id = "acl_acme_mx1";
+      key = "acme_mx1";
+      action = [ "update" ];
+      "update-type" = [ "TXT" ];
+      "update-owner" = "name";
+      "update-owner-match" = "sub-or-equal";
+      "update-owner-name" = [
+        "_acme-challenge.mx1"
+        "_acme-challenge.mta-sts"
+        "_acme-challenge.mail"
+      ];
+    }
+    {
+      id = "acl_acme_web01";
+      key = "acme_web01";
+      action = [ "update" ];
+      "update-type" = [ "TXT" ];
+      "update-owner" = "name";
+      "update-owner-match" = "sub-or-equal";
+      # Wildcard *.cnx.network places its challenge at _acme-challenge.cnx.network,
+      # i.e. _acme-challenge at the cnx.network apex (where this acl is attached).
+      "update-owner-name" = [ "_acme-challenge" ];
+    }
   ];
 
   # Automatic DNSSEC signing policy (primary only). ECDSA P-256/SHA-256 with
@@ -58,21 +147,27 @@ in
 
   # ns1 = primary (master): loads each zone from its file and serves it to ns2.
   # zonefile-load = difference-no-serial lets us edit records without touching the
-  # SOA serial; Knot diffs the file, assigns a date-based serial, signs the zone,
+  # SOA serial; Knot diffs the file, assigns a unixtime serial, signs the zone,
-  # then notifies ns2 and lets it pull the signed zone via AXFR/IXFR.
+  # then notifies ns2 and lets it pull the signed zone via AXFR/IXFR. unixtime is
+  # strictly monotonic per reload, so two zone versions can never share a serial
+  # (the failure mode dateserial's 2-digit daily counter allowed after a journal reset).
   services.knot.settings.zone = map (d: {
     domain = d;
     file = ../../modules/dns/zones + "/${d}.zone";
     "zonefile-load" = "difference-no-serial";
     "zonefile-sync" = "-1";
     "journal-content" = "all"; # required by difference-no-serial; holds the live signed zone
-    "serial-policy" = "dateserial";
+    "serial-policy" = "unixtime";
     "dnssec-signing" = true;
     "dnssec-policy" = "cnx";
     notify = [ "ns2" ];
+    # ns2 transfers; acme_ddns does general DNS-01 updates. The dedicated
+    # acme_mx1 key is attached only to cnx.email, so it can't touch other zones.
     acl = [
       "acl_ns2"
       "acl_acme"
-    ]; # ns2 transfers; acme_ddns key does DNS-01 updates
+    ]
+    ++ lib.optionals (d == "cnx.email") [ "acl_acme_mx1" ]
+    ++ lib.optionals (d == "cnx.network") [ "acl_acme_web01" ];
   }) domains;
 }

machines/ns2/configuration.nix

@@ -1,14 +1,24 @@
-{ ... }:
+{ config, ... }:
 let
   domains = import ../../modules/dns/domains.nix;
+  hosts = import ../../modules/hosts.nix;
 in
 {
   imports = [
     ../../modules/dns/authoritative.nix
+    ../../modules/static-ipv6.nix
+    ../../modules/monitoring/exporters.nix
   ];
 
   clan.core.sops.defaultGroups = [ "admins" ];
 
+  # Public IPv6 (from modules/hosts.nix; matches the ns2 AAAA glue); SLAAC
+  # doesn't bring it up here.
+  cnx.staticIPv6 = {
+    enable = true;
+    address = hosts.${config.networking.hostName}.ipv6;
+  };
+
   time.timeZone = "Etc/GMT-3"; # UTC+3 (fixed offset, no DST)
   services.timesyncd.enable = true;
 

machines/web01/configuration.nix

@@ -0,0 +1,22 @@
+{ config, ... }:
+let
+  hosts = import ../../modules/hosts.nix;
+in
+{
+  imports = [
+    ../../modules/static-ipv6.nix
+    ../../modules/monitoring/exporters.nix
+    ../../modules/web-proxy.nix
+  ];
+
+  clan.core.sops.defaultGroups = [ "admins" ];
+
+  # Public IPv6 (from modules/hosts.nix); SLAAC doesn't bring it up here.
+  cnx.staticIPv6 = {
+    enable = true;
+    address = hosts.${config.networking.hostName}.ipv6;
+  };
+
+  time.timeZone = "Etc/GMT-8"; # UTC+8 (Singapore, fixed offset, no DST)
+  services.timesyncd.enable = true;
+}

machines/web01/disko.nix

@@ -0,0 +1,50 @@
+# ---
+# schema = "single-disk"
+# [placeholders]
+# mainDisk = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_108706511"
+# ---
+# This file was automatically generated!
+# CHANGING this configuration requires wiping and reinstalling the machine
+{
+
+  boot.loader.grub.efiSupport = true;
+  boot.loader.grub.efiInstallAsRemovable = true;
+  boot.loader.grub.enable = true;
+  disko.devices = {
+    disk = {
+      main = {
+        name = "main-ddd46ebf135244608078712d6ec76691";
+        device = "/dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_108706511";
+        type = "disk";
+        content = {
+          type = "gpt";
+          partitions = {
+            "boot" = {
+              size = "1M";
+              type = "EF02"; # for grub MBR
+              priority = 1;
+            };
+            ESP = {
+              type = "EF00";
+              size = "500M";
+              content = {
+                type = "filesystem";
+                format = "vfat";
+                mountpoint = "/boot";
+                mountOptions = [ "umask=0077" ];
+              };
+            };
+            root = {
+              size = "100%";
+              content = {
+                type = "filesystem";
+                format = "ext4";
+                mountpoint = "/";
+              };
+            };
+          };
+        };
+      };
+    };
+  };
+}

modules/dns/acme-mx1-secret.nix

@@ -0,0 +1,19 @@
+# Shared TSIG secret for the dedicated acme_mx1 key.
+#
+# This key lets mx1 — and only mx1 — write _acme-challenge.mx1.cnx.email TXT
+# records on ns1 to obtain its mail TLS cert via ACME DNS-01. ns1 scopes it with
+# acl_acme_mx1 (attached only to the cnx.email zone) so the credential can touch
+# nothing else. ns1 renders this secret into a Knot key file; mx1 into a lego
+# rfc2136 env file; both must carry the same secret, hence one shared generator
+# with a per-host renderer that depends on it. Imported by ns1 and (via mail.nix)
+# mx1.
+{ pkgs, ... }:
+{
+  clan.core.vars.generators.dns-acme-mx1-secret = {
+    share = true;
+    files."secret".secret = true;
+    runtimeInputs = [ pkgs.openssl ];
+    # 32 random bytes, base64 — a valid hmac-sha256 TSIG secret.
+    script = ''openssl rand -base64 32 | tr -d '\n' > "$out"/secret'';
+  };
+}

modules/dns/acme-web01-secret.nix

@@ -0,0 +1,19 @@
+# Shared TSIG secret for the dedicated acme_web01 key.
+#
+# This key lets web01 — and only web01 — write _acme-challenge.cnx.network TXT
+# records on ns1 to obtain its wildcard (*.cnx.network) TLS cert via ACME DNS-01.
+# ns1 scopes it with acl_acme_web01 (attached only to the cnx.network zone) so the
+# credential can touch nothing else. ns1 renders this secret into a Knot key file;
+# web01 into a lego rfc2136 env file; both must carry the same secret, hence one
+# shared generator with a per-host renderer that depends on it. Imported by ns1
+# and (via web-proxy.nix) web01.
+{ pkgs, ... }:
+{
+  clan.core.vars.generators.dns-acme-web01-secret = {
+    share = true;
+    files."secret".secret = true;
+    runtimeInputs = [ pkgs.openssl ];
+    # 32 random bytes, base64 — a valid hmac-sha256 TSIG secret.
+    script = ''openssl rand -base64 32 | tr -d '\n' > "$out"/secret'';
+  };
+}

modules/dns/authoritative.nix

@@ -1,8 +1,14 @@
-{ config, pkgs, ... }:
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
 let
   # ZeroTier addresses — zone transfers run over the mesh, not the public net.
-  ns1zt = "fd06:1bad:ece2:92ad:ba99:939d:766d:8974";
+  mesh = import ../mesh-hosts.nix { inherit config lib; };
-  ns2zt = "fd06:1bad:ece2:92ad:ba99:9323:61be:a09e";
+  ns1zt = mesh.hosts.ns1;
+  ns2zt = mesh.hosts.ns2;
 in
 {
   # Shared TSIG key, generated once and copied to every machine that imports

modules/dns/zones/cnx.email.zone

@@ -11,8 +11,36 @@ $TTL 3600
 @       IN  NS      ns1.cnx.network.
 @       IN  NS      ns2.cnx.network.
 
-; ---- Mail (fill in once the mail host exists) ----
+; ---- Mail ----
-;@      IN  MX 10   mail.cnx.email.
+mx1     IN  A       5.223.65.38
-;mail   IN  A       <mail-ipv4>
+mx1     IN  AAAA    2a01:4ff:2f0:1963::1
-;@      IN  TXT     "v=spf1 mx -all"
+; Client-facing alias for IMAP/submission (Thunderbird etc.); the cert carries
-;_dmarc IN  TXT     "v=DMARC1; p=quarantine; rua=mailto:postmaster@cnx.email"
+; mail.cnx.email as a SAN. The MX must never point here (CNAMEs are illegal MX
+; targets) — server-to-server delivery and DANE stay on mx1.cnx.email.
+mail    IN  CNAME   mx1.cnx.email.
+@       IN  MX 10   mx1.cnx.email.
+@       IN  TXT     "v=spf1 mx -all"
+; Aggregate (rua) + forensic (ruf) reports go to the dmarc@cnx.email mailbox,
+; which parsedmarc on control polls and feeds into Grafana. fo=1 asks reporters
+; to send a forensic report on any SPF/DKIM failure.
+_dmarc  IN  TXT     "v=DMARC1; p=quarantine; rua=mailto:dmarc@cnx.email; ruf=mailto:dmarc@cnx.email; fo=1"
+
+; ---- DANE / TLSA ----
+; "3 1 1" = DANE-EE, SPKI, SHA-256: the digest of mx1's certificate public key.
+; Valid because the zone is DNSSEC-signed and the lego cert uses --reuse-key, so
+; the key (and thus this digest) is stable across renewals. Compute it AFTER the
+; first issuance and paste the hex below:
+;   ssh mx1 'openssl x509 -in /var/lib/acme/mx1.cnx.email/cert.pem -noout -pubkey \
+;     | openssl pkey -pubin -outform DER | openssl dgst -sha256 -binary | xxd -p -c256'
+_25._tcp.mx1  IN  TLSA  3 1 1  bd9a51f60b6d2dd20f18b3553d2795053ac52f87567a46bc892006bb58506404
+
+; ---- MTA-STS ----
+; Policy host (A/AAAA point at mx1); the _mta-sts TXT id MUST be bumped whenever
+; the policy file in modules/mail.nix changes, or senders keep the cached policy.
+mta-sts   IN  A       5.223.65.38
+mta-sts   IN  AAAA    2a01:4ff:2f0:1963::1
+_mta-sts  IN  TXT     "v=STSv1; id=2026061801"
+mail._domainkey IN TXT ( "v=DKIM1; k=rsa; "
+        "p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAr9QxTs5dLtY76bo156+Tp0GUoE554rMwIooIYa2MMYHNs8zPb0thFmaCKGAINdHKNIq2phXAlk51iBTfdqXjx7gVWSrs+ftykqO3b5hUjgImsgqPWGUTzy5/bUgcDELiD9KKEyKYD3+ebZEw6d0uvBvEsA6a1CPzOsufoCDtyKjByCuQzkCBrK25TUHFolGvEYcZexR0LSF+8hMss"
+        "xyw9NYiPpTXVCWQJnrZZpuOBiX0K2l5CAXVyuT/B5RcBXlAUhBTp3390VEhL0wAZMTOnvtvBYK3NnsTIh96fkh6MfWmre7Fi9hEq//xGf40N5/aomMjJrJdqFZJLZpDotb/XwIDAQAB"
+        )

modules/dns/zones/cnx.network.zone

@@ -24,3 +24,11 @@ control IN  AAAA    fd06:1bad:ece2:92ad:ba99:9306:1bad:ece2
 ; ---- Web / apex (fill in once you have a web host) ----
 ;@      IN  A       <web-ipv4>
 ;www    IN  CNAME   cnx.network.
+monitor IN  A       5.223.66.36
+
+; ---- web01 (public reverse proxy / TLS termination) ----
+; Serves a wildcard *.cnx.network TLS cert (ACME DNS-01) and forwards to internal
+; services over the mesh. Add a vhost in modules/web-proxy.nix and a CNAME here.
+web01   IN  A       5.223.55.246
+web01   IN  AAAA    2a01:4ff:2f0:2d8f::1
+grafana IN  CNAME   web01.cnx.network.

modules/docs.nix

@@ -0,0 +1,46 @@
+# Infra runbook (mdBook), built at Nix-build time from ./docs and served by Caddy.
+# Reachable only over the ZeroTier mesh (firewall rule below); the public side is
+# already closed by the Hetzner cloud firewall. Imported by control only.
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  mesh = import ./mesh-hosts.nix { inherit config lib; };
+  port = 8080;
+
+  site = pkgs.stdenvNoCC.mkDerivation {
+    name = "cnx-infra-docs";
+    src = ../docs;
+    nativeBuildInputs = [ pkgs.mdbook ];
+    # mdbook writes a state dir under $HOME; the build sandbox has none.
+    buildPhase = ''
+      export HOME=$TMPDIR
+      mdbook build -d $out
+    '';
+    dontInstall = true;
+  };
+in
+{
+  # ":port" makes Caddy serve plain HTTP (no automatic TLS) on all interfaces;
+  # the mesh-scoped firewall rule below is what constrains reachability.
+  services.caddy = {
+    enable = true;
+    virtualHosts.":${toString port}".extraConfig = ''
+      root * ${site}
+      # mdBook doesn't fingerprint asset filenames, and every file in the Nix
+      # store carries an epoch (1970) mtime, so file_server's only validator never
+      # changes across redeploys — conditional requests would 304 forever and pin
+      # browsers to stale docs. no-store sidesteps caching entirely; pages are tiny
+      # and mesh-only, so refetching on each load is free.
+      header Cache-Control "no-store"
+      file_server
+    '';
+  };
+
+  networking.firewall.extraInputRules = ''
+    ip6 saddr ${mesh.subnet} tcp dport ${toString port} accept
+  '';
+}

modules/hetzner-firewall-rules.nix

@@ -24,6 +24,46 @@ let
     description = "ICMP (ping / PMTUD)";
   };
 
+  # Public mail ports for mx1 (MX for cnx.email). 25 is server-to-server
+  # delivery; 587/465 are client submission; 143/993 are IMAP. 443 serves only the
+  # MTA-STS policy (https://mta-sts.cnx.email/.well-known/mta-sts.txt); the cert
+  # itself uses ACME DNS-01 so port 80 stays closed. Admin still rides the mesh.
+  mailPort = port: description: {
+    direction = "in";
+    protocol = "tcp";
+    inherit port;
+    source_ips = world;
+    inherit description;
+  };
+  mailRules = [
+    (mailPort "25" "SMTP (inbound mail)")
+    (mailPort "587" "Submission (STARTTLS)")
+    (mailPort "465" "Submission (implicit TLS)")
+    (mailPort "143" "IMAP (STARTTLS)")
+    (mailPort "993" "IMAP (implicit TLS)")
+    (mailPort "443" "MTA-STS policy (HTTPS)")
+  ];
+
+  # web01 is a public reverse proxy with TLS termination. 443 serves the proxy;
+  # 80 only carries Caddy's HTTP->HTTPS redirect (the cert uses ACME DNS-01, not
+  # HTTP-01). Admin rides the mesh.
+  webRules = [
+    {
+      direction = "in";
+      protocol = "tcp";
+      port = "80";
+      source_ips = world;
+      description = "HTTP (redirect to HTTPS)";
+    }
+    {
+      direction = "in";
+      protocol = "tcp";
+      port = "443";
+      source_ips = world;
+      description = "HTTPS (reverse proxy / TLS termination)";
+    }
+  ];
+
   dnsRules = [
     {
       direction = "in";
@@ -50,4 +90,12 @@ in
   ];
   "clan-ns1" = dnsRules;
   "clan-ns2" = dnsRules;
+  "clan-mx1" = mailRules ++ [
+    zerotier
+    ping
+  ];
+  "clan-web01" = webRules ++ [
+    zerotier
+    ping
+  ];
 }

modules/hosts.nix

@@ -0,0 +1,32 @@
+# Per-host public network facts: single source of truth for each machine's
+# public IPv4 and its static public IPv6. Consumed by clan.nix's `internet`
+# connection hosts (ipv4) and each machine's `cnx.staticIPv6` (ipv6), so an
+# address is written once instead of being duplicated across configs.
+#
+# NOT a driver for the DNS zone files — those stay hand-edited text, so a record
+# here that also appears as A/AAAA glue still needs a matching manual zone edit.
+#
+# ipv6 is the single address to assign from the host's allocated /64 (we take
+# ::1), without prefix length; cnx.staticIPv6 supplies the /64 default.
+{
+  control = {
+    ipv4 = "77.42.68.181";
+    ipv6 = "2a01:4f9:c013:e6d0::1";
+  };
+  ns1 = {
+    ipv4 = "46.224.170.206";
+    ipv6 = "2a01:4f8:c014:b5c5::1";
+  };
+  ns2 = {
+    ipv4 = "157.180.70.82";
+    ipv6 = "2a01:4f9:c014:6d87::1";
+  };
+  mx1 = {
+    ipv4 = "5.223.65.38";
+    ipv6 = "2a01:4ff:2f0:1963::1";
+  };
+  web01 = {
+    ipv4 = "5.223.55.246";
+    ipv6 = "2a01:4ff:2f0:2d8f::1";
+  };
+}

modules/mail-dmarc-cred.nix

@@ -0,0 +1,26 @@
+# Shared credential for the dmarc@cnx.email mailbox.
+#
+# DMARC aggregate/forensic reports are delivered to dmarc@cnx.email on mx1;
+# parsedmarc on control fetches them over IMAPS across the mesh and needs the
+# *plaintext* passphrase, while mx1's mailserver only needs the sha-512 hash.
+# clan vars secrets are per-machine, so this generator is shared (share = true)
+# to make the same value available on both hosts. Files are root-owned: SNM reads
+# the hash as root, and parsedmarc's ExecStartPre reads the passphrase as root.
+# Imported by mx1 (via mail.nix) and control (via monitoring/parsedmarc.nix).
+{ pkgs, ... }:
+{
+  clan.core.vars.generators.mail-dmarc-cred = {
+    share = true;
+    files."passphrase".secret = true; # read by parsedmarc on control
+    files."hash".secret = true; # consumed by the mailserver on mx1
+    runtimeInputs = [
+      pkgs.xkcdpass
+      pkgs.mkpasswd
+    ];
+    script = ''
+      pass="$(xkcdpass --numwords=4 --delimiter=- --case=lower)-$((RANDOM % 90 + 10))"
+      printf '%s' "$pass" > "$out"/passphrase
+      printf '%s' "$pass" | mkpasswd -s -m sha-512 > "$out"/hash
+    '';
+  };
+}

modules/mail.nix

@@ -0,0 +1,161 @@
+# Declarative mail stack for mx1 (Simple NixOS Mailserver: Postfix + Dovecot +
+# Rspamd + OpenDKIM). Imported by machines/mx1 alongside the SNM flake module.
+#
+# Mailboxes are virtual (not system users): each address below is a login account
+# whose password is auto-generated by a clan vars generator as a four-word
+# passphrase with a trailing number (e.g. otter-lantern-cobalt-driftwood-42). The
+# generator stores both the passphrase and its sha-512 hash. To add a mailbox:
+# append the address to `accounts`, run `clan vars generate mx1`, redeploy mx1,
+# then hand the passphrase to the user:
+#   clan vars get mx1 mail-passwd-<addr>/passphrase
+# (addr with @ and . replaced by -at- and -, e.g. mail-passwd-postmaster-at-cnx-email)
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  hosts = import ./hosts.nix;
+  fqdn = "mx1.cnx.email";
+  mtaStsHost = "mta-sts.cnx.email";
+  # Client-facing alias (CNAME -> mx1) so Thunderbird etc. can use mail.cnx.email
+  # for submission/IMAP; added as a cert SAN so TLS validates against that name.
+  clientHost = "mail.cnx.email";
+
+  # MTA-STS policy served at https://mta-sts.cnx.email/.well-known/mta-sts.txt.
+  # enforce = a sending MTA that fetched this must use a valid, MX-matching TLS
+  # cert or refuse to deliver. Bump the _mta-sts TXT id (in the zone) whenever
+  # this changes.
+  mtaStsPolicy = pkgs.writeText "mta-sts.txt" ''
+    version: STSv1
+    mode: enforce
+    mx: ${fqdn}
+    max_age: 604800
+  '';
+
+  # The mailboxes mx1 serves. postmaster is required by RFC 5321.
+  accounts = [
+    "postmaster@cnx.email"
+  ];
+
+  genName = addr: "mail-passwd-" + lib.replaceStrings [ "@" "." ] [ "-at-" "-" ] addr;
+
+  passwdGenerators = lib.listToAttrs (
+    map (addr: {
+      name = genName addr;
+      value = {
+        files."passphrase".secret = true; # retrievable to hand to the user
+        files."hash".secret = true; # consumed by SNM's hashedPasswordFile
+        runtimeInputs = [
+          pkgs.xkcdpass
+          pkgs.mkpasswd
+        ];
+        script = ''
+          pass="$(xkcdpass --numwords=4 --delimiter=- --case=lower)-$((RANDOM % 90 + 10))"
+          printf '%s' "$pass" > "$out"/passphrase
+          printf '%s' "$pass" | mkpasswd -s -m sha-512 > "$out"/hash
+        '';
+      };
+    }) accounts
+  );
+
+  loginAccounts =
+    lib.listToAttrs (
+      map (addr: {
+        name = addr;
+        value.hashedPasswordFile = config.clan.core.vars.generators.${genName addr}.files."hash".path;
+      }) accounts
+    )
+    // {
+      # DMARC report inbox (rua/ruf target in the cnx.email zone). Its password
+      # comes from the *shared* mail-dmarc-cred generator instead of the per-machine
+      # set above, so parsedmarc on control can read the same passphrase over the
+      # mesh. Retrieve it with: clan vars get mx1 mail-dmarc-cred/passphrase
+      "dmarc@cnx.email".hashedPasswordFile =
+        config.clan.core.vars.generators.mail-dmarc-cred.files."hash".path;
+    };
+in
+{
+  imports = [
+    ./dns/acme-mx1-secret.nix
+    ./mail-dmarc-cred.nix
+  ];
+
+  clan.core.vars.generators = passwdGenerators // {
+    # Render the shared acme_mx1 TSIG secret into a lego rfc2136 env file. lego
+    # (via security.acme below) uses it to write the _acme-challenge.mx1.cnx.email
+    # TXT record to ns1, which authorizes the acme_mx1 key for exactly that owner.
+    dns-acme-rfc2136 = {
+      files."rfc2136.env".secret = true; # root-owned; systemd reads it as root
+      dependencies = [ "dns-acme-mx1-secret" ];
+      script = ''
+        printf 'RFC2136_NAMESERVER=${hosts.ns1.ipv4}:53\nRFC2136_TSIG_ALGORITHM=hmac-sha256.\nRFC2136_TSIG_KEY=acme_mx1\nRFC2136_TSIG_SECRET=%s\n' \
+          "$(cat "$in"/dns-acme-mx1-secret/secret)" > "$out"/rfc2136.env
+      '';
+    };
+  };
+
+  mailserver = {
+    enable = true;
+    # Fresh install: declare the latest layout the nixos-25.11 branch ships (3),
+    # so SNM uses the current dovecot mail directory layout with nothing to migrate.
+    stateVersion = 3;
+    inherit fqdn;
+    domains = [ "cnx.email" ];
+    inherit loginAccounts;
+
+    # Consume a security.acme cert we obtain ourselves via DNS-01 (below); no
+    # web server and no inbound HTTP needed, so port 80 stays closed. Add the
+    # MTA-STS host as a SAN so the one cert also covers the policy endpoint.
+    certificateScheme = "acme";
+    certificateDomains = [
+      mtaStsHost
+      clientHost
+    ];
+
+    dkimSelector = "mail";
+  };
+
+  security.acme = {
+    acceptTerms = true;
+    defaults.email = "postmaster@cnx.email";
+    certs.${fqdn} = {
+      dnsProvider = "rfc2136";
+      environmentFile = config.clan.core.vars.generators.dns-acme-rfc2136.files."rfc2136.env".path;
+      # ns1 is the only nameserver that accepts the acme_mx1 UPDATE; check
+      # propagation against it directly rather than a public resolver.
+      dnsResolver = "${hosts.ns1.ipv4}:53";
+      # Keep the private key fixed across renewals so the DANE TLSA "3 1 1"
+      # record (public-key digest, published in the zone) stays valid.
+      extraLegoRenewFlags = [ "--reuse-key" ];
+      # Caddy serves the MTA-STS endpoint from explicit cert file paths, so it
+      # won't notice a renewal on its own — reload it whenever the cert changes.
+      # (Merges with the postfix/dovecot reloads SNM wires up for this cert.)
+      reloadServices = [ "caddy.service" ];
+    };
+  };
+
+  # The mail cert is owned group=acme (SNM adds postfix/dovecot); Caddy serves the
+  # MTA-STS endpoint from the same cert, so it needs to read the key too.
+  users.users.caddy.extraGroups = [ "acme" ];
+
+  # MTA-STS policy endpoint, served by Caddy (same web server as control's docs).
+  # The explicit `tls cert key` points at the lego-issued mail cert (which carries
+  # mta-sts.cnx.email as a SAN) and disables Caddy's automatic ACME, so no extra
+  # issuance happens and the DANE TLSA key stays stable. Only :443 is opened.
+  services.caddy = {
+    enable = true;
+    virtualHosts.${mtaStsHost}.extraConfig = ''
+      tls /var/lib/acme/${fqdn}/cert.pem /var/lib/acme/${fqdn}/key.pem
+      root * ${pkgs.writeTextDir ".well-known/mta-sts.txt" (builtins.readFile mtaStsPolicy)}
+      file_server
+    '';
+  };
+
+  # DKIM private keys are generated on first start under this dir. They're
+  # regenerable (rotate + republish the TXT), but declaring the path as clan
+  # state lets a borg client back it up to avoid a needless DNS round-trip on
+  # restore. Wiring mx1 into the borgbackup instance is a separate step.
+  clan.core.state.mail-dkim.folders = [ config.mailserver.dkimKeyDirectory ];
+}

modules/mesh-hosts.nix

@@ -0,0 +1,35 @@
+# ZeroTier (clan mesh) addresses — the private IPv6 overlay every machine shares.
+# DNS zone transfers and metrics scraping ride this mesh, never the public net.
+#
+# Rather than hardcoding the addresses, we read them from the public clan vars
+# that clan-core's zerotier generator already writes per machine
+# (vars/per-machine/<m>/zerotier/zerotier-ip/value). This keeps the mesh map in
+# lockstep with the actual identities: regenerate or re-key a node and its
+# address here follows automatically. Call as: import ../mesh-hosts.nix { inherit config lib; }.
+{ config, lib }:
+let
+  dir = config.clan.core.settings.directory;
+
+  readVar =
+    machine: file: builtins.readFile "${dir}/vars/per-machine/${machine}/zerotier/${file}/value";
+
+  hosts = lib.genAttrs [
+    "control"
+    "ns1"
+    "ns2"
+    "mx1"
+    "web01"
+  ] (m: readVar m "zerotier-ip");
+
+  # RFC 4193 prefix of this ZeroTier network: fd + the 8-byte network id + the
+  # 0x9993 marker. The network id is a public var on the controller (control).
+  # The /88 (11 bytes) covers fd + network id + 0x99 + 0x93, i.e. every mesh peer,
+  # and is used to scope mesh-only firewall rules.
+  networkId = readVar "control" "zerotier-network-id";
+  full = "fd" + networkId + "9993"; # 22 hex chars = 11 bytes
+  hextet = i: builtins.substring (i * 4) 4 full;
+  subnet = "${hextet 0}:${hextet 1}:${hextet 2}:${hextet 3}:${hextet 4}:${builtins.substring 20 2 full}00::/88";
+in
+{
+  inherit hosts subnet;
+}

modules/monitoring/alerts.nix

@@ -0,0 +1,108 @@
+# Alerting rules, evaluated by vmalert against VictoriaMetrics on control.
+# Everything is declared here in git. vmalert remote-writes alert state back to
+# VM, so firing alerts surface as the `ALERTS{alertstate="firing"}` series and
+# can be viewed in Grafana. No notifier is wired yet: notifier.blackhole makes
+# that explicit (vmalert evaluates rules but sends nowhere). To deliver alerts
+# later, drop blackhole and set settings."notifier.url" to an Alertmanager.
+{ ... }:
+let
+  vmUrl = "http://127.0.0.1:8428";
+in
+{
+  services.vmalert.instances.cnx = {
+    enable = true;
+    settings = {
+      "datasource.url" = vmUrl;
+      "remoteWrite.url" = vmUrl; # persists ALERTS / ALERTS_FOR_STATE back to VM
+      "notifier.blackhole" = true;
+      "httpListenAddr" = "127.0.0.1:8880"; # vmalert UI/API, loopback only (like VM)
+    };
+    rules.groups = [
+      {
+        name = "dns";
+        rules = [
+          {
+            alert = "DNSSecondaryOutOfSync";
+            expr = "max by (zone) (knot_zone_serial) - min by (zone) (knot_zone_serial) > 0";
+            for = "15m";
+            labels.severity = "warning";
+            annotations.summary = "Zone {{ $labels.zone }} SOA serial differs between nameservers";
+            annotations.description = "The secondary is out of sync with the primary for {{ $labels.zone }}. `knotc zone-retransfer {{ $labels.zone }}` on ns2 forces a fresh pull.";
+          }
+          {
+            alert = "ZoneExpiryLow";
+            expr = "knot_zone_status_expiration < 3600";
+            for = "5m";
+            labels.severity = "critical";
+            annotations.summary = "Zone {{ $labels.zone }} on {{ $labels.instance }} is within 1h of expiry";
+            annotations.description = "Transfers to the secondary appear to be failing; the zone stops being served when the SOA expire timer hits zero.";
+          }
+        ];
+      }
+      {
+        name = "host";
+        rules = [
+          {
+            alert = "ScrapeTargetDown";
+            expr = "up == 0";
+            for = "5m";
+            labels.severity = "critical";
+            annotations.summary = "{{ $labels.job }} exporter on {{ $labels.instance }} is down";
+            annotations.description = "VictoriaMetrics cannot scrape this target; its metrics are missing.";
+          }
+          {
+            alert = "RootFilesystemFull";
+            expr = ''100 * (1 - node_filesystem_avail_bytes{mountpoint="/",fstype!="tmpfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!="tmpfs"}) > 90'';
+            for = "15m";
+            labels.severity = "warning";
+            annotations.summary = "Root filesystem on {{ $labels.instance }} is over 90% full";
+          }
+        ];
+      }
+      {
+        name = "backup";
+        rules = [
+          {
+            alert = "BackupJobFailed";
+            expr = ''node_systemd_unit_state{name=~"borgbackup-job-.+\\.service",state="failed"} == 1'';
+            for = "5m";
+            labels.severity = "warning";
+            annotations.summary = "Backup job {{ $labels.name }} on {{ $labels.instance }} failed";
+            annotations.description = "The borgbackup run did not complete. Check `systemctl status {{ $labels.name }}` and `journalctl -u {{ $labels.name }}` on the client; `borgbackup-create` re-runs it.";
+          }
+          {
+            alert = "BackupStale";
+            expr = ''time() - node_systemd_timer_last_trigger_seconds{name=~"borgbackup-job-.+\\.timer"} > 93600'';
+            for = "30m";
+            labels.severity = "warning";
+            annotations.summary = "No successful backup on {{ $labels.instance }} for over 26h";
+            annotations.description = "The daily backup timer {{ $labels.name }} has not fired within its expected window; the most recent archive is stale. A value far above 26h (or no data) means backups have stopped entirely.";
+          }
+        ];
+      }
+      {
+        # Outside-in DNS probes (blackbox on control). The `for` rides out a
+        # single dropped UDP packet; only a sustained failure fires.
+        name = "dns_probe";
+        rules = [
+          {
+            alert = "DNSResolutionProbeFailed";
+            expr = ''probe_success{query="SOA"} == 0'';
+            for = "5m";
+            labels.severity = "critical";
+            annotations.summary = "{{ $labels.zone }} is not resolving from {{ $labels.instance }}";
+            annotations.description = "The blackbox SOA probe to this public nameserver address is failing; from the outside the zone looks unavailable there, which the Knot stats would not show.";
+          }
+          {
+            alert = "DNSSECProbeFailed";
+            expr = ''probe_success{query="DNSKEY"} == 0'';
+            for = "5m";
+            labels.severity = "critical";
+            annotations.summary = "{{ $labels.zone }} DNSKEY missing from {{ $labels.instance }}";
+            annotations.description = "The DNSKEY probe to this public nameserver address is failing: the zone's signing keys are not being served, so validating resolvers will treat answers as bogus.";
+          }
+        ];
+      }
+    ];
+  };
+}

modules/monitoring/blackbox-probes.nix

@@ -0,0 +1,108 @@
+# Blackbox DNS probe definitions, shared between the exporter module
+# (modules/monitoring/blackbox.nix, which renders these into the blackbox
+# config) and the scraper (modules/monitoring/server.nix, which turns them into
+# VictoriaMetrics scrape jobs). Kept in one place so the module list and the
+# scrape jobs can never drift apart.
+#
+# These query the nameservers' PUBLIC addresses, i.e. the path a real internet
+# resolver takes, not the mesh — the whole point is to catch outside-in
+# resolution failures the Knot stats can't see. For each zone we run two probes
+# per endpoint: an SOA query (is the zone being served at all?) and a DNSKEY
+# query (is it still DNSSEC-signed?). Blackbox has no DO-bit option, so we ask
+# for DNSKEY directly — an authoritative signed zone returns it without EDNS0,
+# and its absence means signing has broken.
+{ lib }:
+let
+  domains = import ../dns/domains.nix;
+
+  blackboxAddr = "127.0.0.1:9115";
+
+  # Public endpoints of the authoritative nameservers. The v4 addresses also
+  # appear in the `internet` instance in clan.nix; the v6 ones in each ns
+  # machine's cnx.staticIPv6. IPv6 literals are bracketed for host:port.
+  endpoints = [
+    {
+      instance = "ns1 v4";
+      target = "46.224.170.206:53";
+    }
+    {
+      instance = "ns1 v6";
+      target = "[2a01:4f8:c014:b5c5::1]:53";
+    }
+    {
+      instance = "ns2 v4";
+      target = "157.180.70.82:53";
+    }
+    {
+      instance = "ns2 v6";
+      target = "[2a01:4f9:c014:6d87::1]:53";
+    }
+  ];
+
+  queries = [
+    {
+      name = "soa";
+      type = "SOA";
+    }
+    {
+      name = "dnskey";
+      type = "DNSKEY";
+    }
+  ];
+
+  sanitize = lib.replaceStrings [ "." ] [ "_" ];
+  moduleName = zone: q: "dns_${q.name}_${sanitize zone}";
+
+  modules = lib.listToAttrs (
+    lib.concatMap (
+      zone:
+      map (
+        q:
+        lib.nameValuePair (moduleName zone q) {
+          prober = "dns";
+          timeout = "5s";
+          dns = {
+            query_name = "${zone}.";
+            query_type = q.type;
+            valid_rcodes = [ "NOERROR" ];
+            # Fail unless at least one answer RR of the queried type is present:
+            # a NOERROR with an empty answer (or a missing DNSKEY) still fails.
+            validate_answer_rrs.fail_if_not_matches_regexp = [ "\\s${q.type}\\s" ];
+          };
+        }
+      ) queries
+    ) domains
+  );
+
+  scrapeConfigs = lib.concatMap (
+    zone:
+    map (q: {
+      job_name = "blackbox_${moduleName zone q}";
+      metrics_path = "/probe";
+      params.module = [ (moduleName zone q) ];
+      static_configs = map (e: {
+        targets = [ e.target ];
+        labels = {
+          instance = e.instance;
+          zone = zone;
+          query = q.type;
+        };
+      }) endpoints;
+      # Hand the real DNS server to blackbox as ?target=, then point the scrape
+      # at the exporter itself.
+      relabel_configs = [
+        {
+          source_labels = [ "__address__" ];
+          target_label = "__param_target";
+        }
+        {
+          target_label = "__address__";
+          replacement = blackboxAddr;
+        }
+      ];
+    }) queries
+  ) domains;
+in
+{
+  inherit modules scrapeConfigs blackboxAddr;
+}

modules/monitoring/blackbox.nix

@@ -0,0 +1,24 @@
+# Blackbox exporter on control: outside-in DNS probes against the public
+# nameserver addresses (see blackbox-probes.nix for what and why). Bound to
+# loopback — only VictoriaMetrics on the same host scrapes its /probe endpoint,
+# and the scrape jobs that drive it live in server.nix. The probes leave control
+# over the public internet to reach ns1/ns2, which is the path we want to test.
+{
+  lib,
+  pkgs,
+  ...
+}:
+let
+  probes = import ./blackbox-probes.nix { inherit lib; };
+in
+{
+  services.prometheus.exporters.blackbox = {
+    enable = true;
+    listenAddress = "127.0.0.1";
+    port = 9115;
+    # JSON is valid YAML; enableConfigCheck runs the exporter's own --config.check
+    # against this file at build time, so a malformed prober is caught here.
+    configFile = pkgs.writeText "blackbox.yml" (builtins.toJSON { inherit (probes) modules; });
+    enableConfigCheck = true;
+  };
+}

modules/monitoring/dashboards/backups.json

@@ -0,0 +1,217 @@
+{
+  "uid": "cnx-backups",
+  "title": "CNX Backups",
+  "tags": ["backup", "borg", "cnx"],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "version": 1,
+  "refresh": "1m",
+  "time": { "from": "now-7d", "to": "now" },
+  "templating": { "list": [] },
+  "annotations": { "list": [] },
+  "panels": [
+    {
+      "type": "row",
+      "title": "Backups",
+      "id": 1,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }
+    },
+    {
+      "type": "stat",
+      "title": "Backup health",
+      "description": "1 if any borgbackup job is in the failed state, 0 otherwise. A successful run leaves the oneshot unit inactive (still OK); only a real failure shows FAILED. Derived from the node_exporter systemd collector on the backup client (ns1).",
+      "id": 2,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 5, "w": 8, "x": 0, "y": 1 },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "thresholds" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "green", "value": null }]
+          },
+          "noValue": "no data",
+          "mappings": [
+            {
+              "type": "value",
+              "options": {
+                "0": { "text": "OK", "color": "green", "index": 0 },
+                "1": { "text": "FAILED", "color": "red", "index": 1 }
+              }
+            }
+          ]
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "auto",
+        "orientation": "auto"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "max(node_systemd_unit_state{name=~\"borgbackup-job-.+\\\\.service\",state=\"failed\"})",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "type": "stat",
+      "title": "Last backup run",
+      "description": "When the most recent backup timer last fired (the daily borgbackup job). 'No data' before the first run.",
+      "id": 3,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 5, "w": 8, "x": 8, "y": 1 },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "dateTimeFromNow",
+          "color": { "mode": "fixed", "fixedColor": "text" },
+          "noValue": "never"
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "none",
+        "graphMode": "none",
+        "textMode": "auto",
+        "orientation": "auto"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "max(node_systemd_timer_last_trigger_seconds{name=~\"borgbackup-job-.+\\\\.timer\"}) * 1000",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "type": "stat",
+      "title": "Time since last backup",
+      "description": "Age of the most recent backup. Backups run daily, so anything past ~25h means a run was missed. Red over 25h.",
+      "id": 4,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 5, "w": 8, "x": 16, "y": 1 },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s",
+          "color": { "mode": "thresholds" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "red", "value": 90000 }
+            ]
+          },
+          "noValue": "never"
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "auto",
+        "orientation": "auto"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "time() - max(node_systemd_timer_last_trigger_seconds{name=~\"borgbackup-job-.+\\\\.timer\"})",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "type": "table",
+      "title": "Backup jobs (current state)",
+      "description": "Every borgbackup job and the systemd unit state it is currently in, per client. 'inactive' is the normal resting state of a oneshot job between runs.",
+      "id": 5,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
+      "options": { "showHeader": true },
+      "fieldConfig": {
+        "defaults": { "custom": { "align": "auto" } },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "node_systemd_unit_state{name=~\"borgbackup-job-.+\\\\.service\"} == 1",
+          "format": "table",
+          "instant": true
+        }
+      ],
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true,
+              "Value": true,
+              "__name__": true,
+              "job": true,
+              "type": true
+            }
+          }
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Failed state over time",
+      "description": "1 while a backup job is in the failed state. A spike here is a backup that did not complete and was not retried before the next scrape.",
+      "id": 6,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
+      "fieldConfig": {
+        "defaults": { "unit": "short", "min": 0, "max": 1 },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "node_systemd_unit_state{name=~\"borgbackup-job-.+\\\\.service\",state=\"failed\"}",
+          "legendFormat": "{{instance}} {{name}}"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Time since last backup (history)",
+      "description": "Age of the latest backup over time. The sawtooth should reset to near zero once a day; a steady climb without a reset means backups stopped running.",
+      "id": 7,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 14 },
+      "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "time() - node_systemd_timer_last_trigger_seconds{name=~\"borgbackup-job-.+\\\\.timer\"}",
+          "legendFormat": "{{instance}} {{name}}"
+        }
+      ]
+    }
+  ]
+}

modules/monitoring/dashboards/dns.json

@@ -0,0 +1,326 @@
+{
+  "uid": "cnx-dns",
+  "title": "CNX DNS",
+  "tags": ["dns", "knot", "cnx"],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "version": 3,
+  "refresh": "30s",
+  "time": { "from": "now-6h", "to": "now" },
+  "templating": { "list": [] },
+  "annotations": { "list": [] },
+  "panels": [
+    {
+      "type": "row",
+      "title": "Alerts",
+      "id": 11,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }
+    },
+    {
+      "type": "table",
+      "title": "Active alerts",
+      "description": "Firing vmalert alerts (the ALERTS series vmalert writes back to VictoriaMetrics). An empty table means all clear.",
+      "id": 12,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 7, "w": 24, "x": 0, "y": 1 },
+      "options": { "showHeader": true },
+      "fieldConfig": {
+        "defaults": { "custom": { "align": "auto" } },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "ALERTS{alertstate=\"firing\"}",
+          "format": "table",
+          "instant": true
+        }
+      ],
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true,
+              "Value": true,
+              "__name__": true,
+              "alertstate": true
+            }
+          }
+        }
+      ]
+    },
+    {
+      "type": "row",
+      "title": "DNS / Zones",
+      "id": 1,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 8 }
+    },
+    {
+      "type": "table",
+      "title": "Zone SOA serial (per nameserver)",
+      "description": "ns1 and ns2 should report the same serial per zone. A divergence here is the secondary-out-of-sync condition.",
+      "id": 2,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 },
+      "options": { "showHeader": true },
+      "fieldConfig": {
+        "defaults": { "custom": { "align": "auto" } },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "knot_zone_serial",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "{{zone}} @ {{instance}}"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Seconds until zone expiry",
+      "description": "On secondaries this counts down between successful transfers; a steady decline toward zero means transfers are failing.",
+      "id": 3,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 9 },
+      "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "knot_zone_status_expiration",
+          "legendFormat": "{{zone}} @ {{instance}}"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Query rate by nameserver",
+      "id": 4,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 17 },
+      "fieldConfig": { "defaults": { "unit": "qps" }, "overrides": [] },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "sum by (instance) (rate(knot_stats_request_protocol_total[5m]))",
+          "legendFormat": "{{instance}}"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Response codes",
+      "id": 5,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 17 },
+      "fieldConfig": { "defaults": { "unit": "qps" }, "overrides": [] },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "sum by (type) (rate(knot_stats_response_code_total[5m]))",
+          "legendFormat": "{{type}}"
+        }
+      ]
+    },
+    {
+      "type": "row",
+      "title": "Hosts",
+      "id": 6,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 25 }
+    },
+    {
+      "type": "timeseries",
+      "title": "CPU busy %",
+      "id": 7,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 26 },
+      "fieldConfig": {
+        "defaults": { "unit": "percent", "min": 0, "max": 100 },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "100 - (avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)",
+          "legendFormat": "{{instance}}"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Memory used %",
+      "id": 8,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 26 },
+      "fieldConfig": {
+        "defaults": { "unit": "percent", "min": 0, "max": 100 },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "100 * (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)",
+          "legendFormat": "{{instance}}"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Root filesystem used %",
+      "id": 9,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 34 },
+      "fieldConfig": {
+        "defaults": { "unit": "percent", "min": 0, "max": 100 },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "100 * (1 - node_filesystem_avail_bytes{mountpoint=\"/\",fstype!=\"tmpfs\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!=\"tmpfs\"})",
+          "legendFormat": "{{instance}}"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Load average (1m)",
+      "id": 10,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 34 },
+      "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "node_load1",
+          "legendFormat": "{{instance}}"
+        }
+      ]
+    },
+    {
+      "type": "row",
+      "title": "DNS probes (outside-in)",
+      "id": 20,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 42 }
+    },
+    {
+      "type": "table",
+      "title": "Probe status (per zone / server)",
+      "description": "blackbox_exporter on control queries each nameserver's public address (v4 + v6) for every zone: an SOA query (zone served) and a DNSKEY query (still signed). UP = the resolver's-eye view is healthy.",
+      "id": 21,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 43 },
+      "options": { "showHeader": true },
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "align": "auto",
+            "cellOptions": { "type": "color-background" }
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "red", "value": null },
+              { "color": "green", "value": 1 }
+            ]
+          },
+          "mappings": [
+            {
+              "type": "value",
+              "options": {
+                "0": { "text": "DOWN", "index": 0 },
+                "1": { "text": "UP", "index": 1 }
+              }
+            }
+          ]
+        },
+        "overrides": [
+          {
+            "matcher": { "id": "byName", "options": "zone" },
+            "properties": [
+              { "id": "custom.cellOptions", "value": { "type": "auto" } }
+            ]
+          },
+          {
+            "matcher": { "id": "byName", "options": "query" },
+            "properties": [
+              { "id": "custom.cellOptions", "value": { "type": "auto" } }
+            ]
+          },
+          {
+            "matcher": { "id": "byName", "options": "instance" },
+            "properties": [
+              { "id": "custom.cellOptions", "value": { "type": "auto" } }
+            ]
+          }
+        ]
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "probe_success",
+          "format": "table",
+          "instant": true
+        }
+      ],
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": { "Time": true, "__name__": true, "job": true },
+            "renameByName": { "Value": "status" }
+          }
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Probe success (1 = ok)",
+      "description": "0 means the probe failed: the zone is not being served or not signed from that public address. Sustained failures fire DNSResolutionProbeFailed / DNSSECProbeFailed.",
+      "id": 22,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 43 },
+      "fieldConfig": {
+        "defaults": { "unit": "short", "min": 0, "max": 1 },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "probe_success",
+          "legendFormat": "{{zone}} {{query}} @ {{instance}}"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "DNS probe latency",
+      "description": "Total round-trip time of each blackbox DNS probe. A climbing trend points at a slow or overloaded nameserver before it starts failing outright.",
+      "id": 23,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 51 },
+      "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "probe_duration_seconds",
+          "legendFormat": "{{zone}} {{query}} @ {{instance}}"
+        }
+      ]
+    }
+  ]
+}

modules/monitoring/dashboards/uptime.json

@@ -0,0 +1,194 @@
+{
+  "uid": "cnx-uptime",
+  "title": "CNX Uptime",
+  "tags": ["uptime", "availability", "cnx"],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "version": 1,
+  "refresh": "30s",
+  "time": { "from": "now-24h", "to": "now" },
+  "templating": { "list": [] },
+  "annotations": { "list": [] },
+  "panels": [
+    {
+      "type": "row",
+      "title": "Uptime",
+      "id": 1,
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }
+    },
+    {
+      "type": "stat",
+      "title": "Host status",
+      "description": "Whether VictoriaMetrics is currently able to scrape each host's node_exporter. UP means the host (and its mesh path) is reachable; DOWN means the scrape failed. One tile per machine.",
+      "id": 2,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 6, "w": 12, "x": 0, "y": 1 },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "thresholds" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "green", "value": null }]
+          },
+          "noValue": "no data",
+          "mappings": [
+            {
+              "type": "value",
+              "options": {
+                "0": { "text": "DOWN", "color": "red", "index": 0 },
+                "1": { "text": "UP", "color": "green", "index": 1 }
+              }
+            }
+          ]
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "value_and_name",
+        "orientation": "auto"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "up{job=\"node\"}",
+          "legendFormat": "{{instance}}",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "type": "stat",
+      "title": "Current uptime",
+      "description": "Time since each host last booted (now - node_boot_time_seconds). A value that drops back to near zero means the host rebooted.",
+      "id": 3,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 6, "w": 12, "x": 12, "y": 1 },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "dtdurations",
+          "color": { "mode": "fixed", "fixedColor": "text" },
+          "noValue": "no data"
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "colorMode": "none",
+        "graphMode": "none",
+        "textMode": "value_and_name",
+        "orientation": "auto"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "time() - node_boot_time_seconds{job=\"node\"}",
+          "legendFormat": "{{instance}}",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "type": "bargauge",
+      "title": "Availability over window",
+      "description": "Fraction of successful scrapes over the selected time range, per host (avg of up over $__range). 100% means every scrape in the window succeeded; dips reveal flapping or outages. Red below 99%.",
+      "id": 4,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 7 },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "color": { "mode": "thresholds" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "red", "value": null },
+              { "color": "yellow", "value": 99 },
+              { "color": "green", "value": 99.9 }
+            ]
+          },
+          "noValue": "no data"
+        },
+        "overrides": []
+      },
+      "options": {
+        "displayMode": "gradient",
+        "orientation": "horizontal",
+        "showUnfilled": true,
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        }
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "avg_over_time(up{job=\"node\"}[$__range]) * 100",
+          "legendFormat": "{{instance}}",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Uptime over time",
+      "description": "Host uptime across the window. The line should climb steadily; a reset to zero marks a reboot.",
+      "id": 5,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 7 },
+      "fieldConfig": {
+        "defaults": { "unit": "s", "custom": { "fillOpacity": 0 } },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "time() - node_boot_time_seconds{job=\"node\"}",
+          "legendFormat": "{{instance}}"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Up/down history",
+      "description": "1 while a host's node_exporter was scrapeable, 0 while it was not. Gaps to zero are outages or lost mesh connectivity.",
+      "id": 6,
+      "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+      "gridPos": { "h": 6, "w": 24, "x": 0, "y": 15 },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "min": 0,
+          "max": 1,
+          "custom": { "fillOpacity": 20, "lineInterpolation": "stepAfter" }
+        },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "prometheus", "uid": "victoriametrics" },
+          "expr": "up{job=\"node\"}",
+          "legendFormat": "{{instance}}"
+        }
+      ]
+    }
+  ]
+}

modules/monitoring/exporters.nix

@@ -0,0 +1,118 @@
+# Per-host observability agents, imported by every machine. Host metrics
+# everywhere; Knot DNS metrics on the nameservers; journald shipped to
+# VictoriaLogs on control. Everything is reachable only over the ZeroTier mesh
+# (see the firewall rule at the bottom); the public side is already closed by the
+# Hetzner cloud firewall.
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  mesh = import ../mesh-hosts.nix { inherit config lib; };
+  knotEnabled = config.services.knot.enable;
+  # node_exporter on every host; knot-exporter only where Knot runs.
+  ports = [ 9100 ] ++ lib.optional knotEnabled 9433;
+in
+{
+  # extraInputRules (the mesh-scoped accept below) needs the nftables firewall
+  # backend. allowedTCP/UDPPorts used elsewhere (53, 9993) translate unchanged.
+  networking.nftables.enable = true;
+
+  # Host metrics: CPU, memory, disk, network, systemd unit state.
+  services.prometheus.exporters.node = {
+    enable = true;
+    # Listen on all interfaces (incl. the v6 mesh). We deliberately do NOT bind
+    # to the ZeroTier ULA: the node module renders --web.listen-address without
+    # IPv6 brackets, and binding a single ULA would also race ZeroTier bring-up
+    # at boot. Reachability is constrained by the firewall rule instead.
+    listenAddress = "";
+    port = 9100;
+    enabledCollectors = [ "systemd" ];
+  };
+
+  # Knot DNS metrics. The exporter reads Knot's control socket, so it runs as the
+  # knot user; mod-stats (below) populates the query/response counters it exports.
+  systemd.services.knot-exporter = lib.mkIf knotEnabled {
+    description = "Prometheus exporter for Knot DNS";
+    after = [ "knot.service" ];
+    wants = [ "knot.service" ];
+    wantedBy = [ "multi-user.target" ];
+    serviceConfig = {
+      ExecStart = lib.concatStringsSep " " [
+        "${pkgs.prometheus-knot-exporter}/bin/knot-exporter"
+        "--web-listen-addr ::"
+        "--web-listen-port 9433"
+        "--knot-library-path ${pkgs.knot-dns.out}/lib/libknot.so"
+        "--knot-socket-path /run/knot/knot.sock"
+      ];
+      User = "knot";
+      Group = "knot";
+      Restart = "on-failure";
+      RestartSec = 5;
+      NoNewPrivileges = true;
+      ProtectSystem = "strict";
+      ProtectHome = true;
+      PrivateTmp = true;
+      ProtectKernelTunables = true;
+      ProtectControlGroups = true;
+      RestrictAddressFamilies = [
+        "AF_INET"
+        "AF_INET6"
+        "AF_UNIX"
+      ];
+    };
+  };
+
+  # mod-stats: per-query/zone counters exposed over the control socket and read
+  # by knot-exporter. Loaded as a global module on the default template so it
+  # applies to every zone. (Merges with the zone/acl/policy settings elsewhere.)
+  services.knot.settings = lib.mkIf knotEnabled {
+    "mod-stats" = [
+      {
+        id = "default";
+        "request-protocol" = true;
+        "server-operation" = true;
+        "response-code" = true;
+        "query-type" = true;
+        "reply-nodata" = true;
+      }
+    ];
+    template = [
+      {
+        id = "default";
+        global-module = [ "mod-stats/default" ];
+      }
+    ];
+  };
+
+  # Ship journald to VictoriaLogs on control (services.victorialogs in
+  # server.nix). control uploads to loopback so its own logs survive a mesh
+  # outage; ns1/ns2 push over the mesh to control's ZeroTier address.
+  services.journald.upload = {
+    enable = true;
+    settings.Upload.URL =
+      let
+        dest =
+          if config.networking.hostName == "control" then
+            "127.0.0.1:9428"
+          else
+            "[${mesh.hosts.control}]:9428";
+      in
+      "http://${dest}/insert/journald";
+  };
+
+  # systemd-journal-upload exits if the sink is unreachable. Upstream already
+  # restarts it (Restart=always/RestartSec=3sec), but the default start-rate limit
+  # (5 tries / 10s) lets it give up permanently — so a transient VictoriaLogs
+  # outage leaves the uploader dead until the next deploy. Disable the limit so it
+  # retries forever and self-heals once the sink returns. (A persistent failure
+  # still surfaces loudly in a deploy, which is what we want.)
+  systemd.services.systemd-journal-upload.startLimitIntervalSec = 0;
+
+  # Scrape ports reachable only from the ZeroTier mesh.
+  networking.firewall.extraInputRules = ''
+    ip6 saddr ${mesh.subnet} tcp dport { ${lib.concatMapStringsSep ", " toString ports} } accept
+  '';
+}

modules/monitoring/parsedmarc.nix

@@ -0,0 +1,60 @@
+# DMARC report analyzer, imported by control only. parsedmarc fetches the
+# aggregate/forensic reports that land in the dmarc@cnx.email mailbox on mx1,
+# parses the XML, and stores results in a local Elasticsearch; the official
+# parsedmarc dashboard + an Elasticsearch datasource are auto-provisioned into
+# the Grafana instance that server.nix already runs on this host.
+#
+# IMAP runs over the ZeroTier mesh, not the public net: we pin mx1.cnx.email to
+# its mesh address in /etc/hosts so TLS still validates against the public
+# Let's Encrypt cert (primary domain mx1.cnx.email) while the bytes stay on the
+# overlay. The mailbox passphrase is the shared mail-dmarc-cred secret; parsedmarc
+# reads it as root in its ExecStartPre, so root-owned (clan default) is fine.
+{ config, lib, ... }:
+let
+  mesh = import ../mesh-hosts.nix { inherit config lib; };
+in
+{
+  imports = [ ../mail-dmarc-cred.nix ];
+
+  # Elasticsearch 7.x is under the (unfree) Elastic License; allow just this one
+  # package rather than opening allowUnfree globally.
+  nixpkgs.config.allowUnfreePredicate = pkg: lib.getName pkg == "elasticsearch";
+
+  # Keep mx1's IMAP traffic on the mesh while presenting the public cert name.
+  networking.hosts.${mesh.hosts.mx1} = [ "mx1.cnx.email" ];
+
+  services.parsedmarc = {
+    enable = true;
+    provision = {
+      # Local Elasticsearch on 127.0.0.1:9200 (loopback; no firewall change).
+      # datasource + dashboard default to true once ES and Grafana are both on.
+      elasticsearch = true;
+      # GeoIP needs a MaxMind account/license key; skip it (reports still parse,
+      # just without source-IP geolocation).
+      geoIp = false;
+      grafana = {
+        datasource = true;
+        dashboard = true;
+      };
+    };
+    settings = {
+      imap = {
+        host = "mx1.cnx.email";
+        port = 993;
+        ssl = true;
+        user = "dmarc@cnx.email";
+        password = {
+          _secret = config.clan.core.vars.generators.mail-dmarc-cred.files."passphrase".path;
+        };
+      };
+      mailbox = {
+        watch = true; # IMAP IDLE: process reports as they arrive
+        delete = false; # archive processed reports, don't delete
+      };
+      general = {
+        save_aggregate = true;
+        save_forensic = true;
+      };
+    };
+  };
+}

modules/monitoring/server.nix

@@ -0,0 +1,156 @@
+# Monitoring server, imported by control only: VictoriaMetrics (TSDB + scraper)
+# and Grafana. VictoriaMetrics binds loopback (only Grafana, on the same host,
+# reads it). Grafana is reachable over the ZeroTier mesh, scoped by the firewall
+# rule at the bottom; the Hetzner cloud firewall keeps it off the public net.
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  mesh = import ../mesh-hosts.nix { inherit config lib; };
+  probes = import ./blackbox-probes.nix { inherit lib; };
+  vmPort = 8428;
+  logsPort = 9428;
+  grafanaPort = 3000;
+  controlV6 = mesh.hosts.control;
+
+  # A single scrape target with a friendly instance label. IPv6 mesh addresses
+  # must be bracketed for Prometheus-style targets.
+  target = name: addr: port: {
+    targets = [ "${addr}:${toString port}" ];
+    labels.instance = name;
+  };
+  v6 = addr: "[${addr}]";
+
+  adminPasswordFile = config.clan.core.vars.generators.grafana-admin.files."password".path;
+in
+{
+  services.victoriametrics = {
+    enable = true;
+    listenAddress = "127.0.0.1:${toString vmPort}";
+    retentionPeriod = "180d";
+    # The scraper dials IPv4-only by default; our ns1/ns2 targets are mesh ULAs,
+    # so without this VM drops them with "no suitable address found (try -enableTCP6)".
+    extraOptions = [ "-enableTCP6" ];
+    prometheusConfig = {
+      global.scrape_interval = "30s";
+      scrape_configs = [
+        {
+          job_name = "node";
+          static_configs = [
+            # control scrapes its own node_exporter over loopback so host metrics
+            # survive even if the mesh is down; ns1/ns2 are scraped over the mesh.
+            (target "control" "127.0.0.1" 9100)
+            (target "ns1" (v6 mesh.hosts.ns1) 9100)
+            (target "ns2" (v6 mesh.hosts.ns2) 9100)
+            (target "mx1" (v6 mesh.hosts.mx1) 9100)
+            (target "web01" (v6 mesh.hosts.web01) 9100)
+          ];
+        }
+        {
+          job_name = "knot";
+          static_configs = [
+            (target "ns1" (v6 mesh.hosts.ns1) 9433)
+            (target "ns2" (v6 mesh.hosts.ns2) 9433)
+          ];
+        }
+      ]
+      # Outside-in DNS probes via the blackbox exporter (blackbox.nix). The job
+      # list is generated from the same probe definitions the exporter uses.
+      ++ probes.scrapeConfigs;
+    };
+  };
+
+  # Centralized logs: VictoriaLogs ingests journald from all three hosts, each
+  # of which runs systemd-journal-upload against /insert/journald (exporters.nix).
+  # Binds all interfaces because ns1/ns2 push over the mesh; the firewall rule at
+  # the bottom scopes 9428 to the mesh subnet and the Hetzner firewall closes the
+  # public side. Retention is set via extraOptions (no dedicated NixOS option).
+  services.victorialogs = {
+    enable = true;
+    listenAddress = ":${toString logsPort}";
+    # -enableTCP6: like the scraper above, VictoriaLogs is IPv4-only by default
+    # for *listening* too — ":9428" binds 0.0.0.0 only, so ns1/ns2 pushing over
+    # the IPv6 mesh get "connection refused". This makes it bind [::] (dual-stack)
+    # so the mesh can reach it. Retention has no dedicated NixOS option.
+    extraOptions = [
+      "-retentionPeriod=30d"
+      "-enableTCP6"
+    ];
+  };
+
+  # Admin password generated once and stored as a clan secret. Retrieve with:
+  #   clan vars get control grafana-admin/password
+  clan.core.vars.generators.grafana-admin = {
+    files."password" = {
+      secret = true;
+      owner = "grafana";
+      group = "grafana";
+    };
+    runtimeInputs = [ pkgs.openssl ];
+    script = ''
+      openssl rand -base64 24 | tr -d "\n" > "$out"/password
+    '';
+  };
+
+  services.grafana = {
+    enable = true;
+    # VictoriaLogs datasource plugin so journald is greppable from Grafana,
+    # alongside the metrics datasource.
+    declarativePlugins = [ pkgs.grafanaPlugins.victoriametrics-logs-datasource ];
+    settings = {
+      server = {
+        http_addr = "::";
+        http_port = grafanaPort;
+        root_url = "http://${v6 controlV6}:${toString grafanaPort}/";
+      };
+      security = {
+        admin_user = "admin";
+        admin_password = "$__file{${adminPasswordFile}}";
+      };
+      "auth.anonymous".enabled = false;
+      users.allow_sign_up = false;
+    };
+    provision = {
+      enable = true;
+      datasources.settings = {
+        apiVersion = 1;
+        datasources = [
+          {
+            name = "VictoriaMetrics";
+            type = "prometheus";
+            uid = "victoriametrics";
+            access = "proxy";
+            url = "http://127.0.0.1:${toString vmPort}";
+            isDefault = true;
+          }
+          {
+            name = "VictoriaLogs";
+            type = "victoriametrics-logs-datasource";
+            uid = "victorialogs";
+            access = "proxy";
+            url = "http://127.0.0.1:${toString logsPort}";
+          }
+        ];
+      };
+      dashboards.settings = {
+        apiVersion = 1;
+        providers = [
+          {
+            name = "cnx";
+            options.path = ./dashboards;
+            options.foldersFromFilesStructure = false;
+          }
+        ];
+      };
+    };
+  };
+
+  # Grafana (admin laptops + servers) and VictoriaLogs ingestion (ns1/ns2 push
+  # journald over the mesh) reachable only from the ZeroTier mesh.
+  networking.firewall.extraInputRules = ''
+    ip6 saddr ${mesh.subnet} tcp dport { ${toString grafanaPort}, ${toString logsPort} } accept
+  '';
+}

modules/static-ipv6.nix

@@ -0,0 +1,48 @@
+# Static public IPv6 for hosts where SLAAC/RA doesn't bring up a global
+# address (e.g. Hetzner, which allocates a /64 but expects you to pick one
+# address yourself and route via the link-local gateway fe80::1).
+{ lib, config, ... }:
+let
+  cfg = config.cnx.staticIPv6;
+in
+{
+  options.cnx.staticIPv6 = {
+    enable = lib.mkEnableOption "static public IPv6 on the primary interface";
+
+    address = lib.mkOption {
+      type = lib.types.str;
+      example = "2a01:4f8:c014:b5c5::1";
+      description = "Public IPv6 address (no prefix length).";
+    };
+
+    prefixLength = lib.mkOption {
+      type = lib.types.int;
+      default = 64;
+      description = "Prefix length of the allocated block.";
+    };
+
+    interface = lib.mkOption {
+      type = lib.types.str;
+      default = "enp1s0";
+      description = "Interface to assign the address to.";
+    };
+
+    gateway = lib.mkOption {
+      type = lib.types.str;
+      default = "fe80::1";
+      description = "IPv6 default gateway (Hetzner uses the link-local fe80::1).";
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    networking.interfaces.${cfg.interface}.ipv6.addresses = [
+      {
+        inherit (cfg) address prefixLength;
+      }
+    ];
+    networking.defaultGateway6 = {
+      address = cfg.gateway;
+      interface = cfg.interface;
+    };
+  };
+}

modules/web-proxy.nix

@@ -0,0 +1,73 @@
+# Public reverse proxy with TLS termination for web01. Caddy fronts internal
+# services and forwards to them over the ZeroTier mesh, never the public net.
+# The cert is a single wildcard (*.cnx.network) obtained via ACME DNS-01, so
+# adding a vhost needs no new issuance. Public ports: 443 for the proxy and 80
+# only for Caddy's HTTP->HTTPS redirect (issuance never uses inbound HTTP).
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  mesh = import ./mesh-hosts.nix { inherit config lib; };
+  hosts = import ./hosts.nix;
+  certName = "cnx.network";
+in
+{
+  imports = [ ./dns/acme-web01-secret.nix ];
+
+  # Render the shared acme_web01 TSIG secret into a lego rfc2136 env file. lego
+  # (via security.acme below) uses it to write _acme-challenge.cnx.network TXT
+  # records on ns1, which authorizes the acme_web01 key for exactly that owner.
+  clan.core.vars.generators.dns-acme-web01-rfc2136 = {
+    files."rfc2136.env".secret = true; # root-owned; systemd reads it as root
+    dependencies = [ "dns-acme-web01-secret" ];
+    script = ''
+      printf 'RFC2136_NAMESERVER=${hosts.ns1.ipv4}:53\nRFC2136_TSIG_ALGORITHM=hmac-sha256.\nRFC2136_TSIG_KEY=acme_web01\nRFC2136_TSIG_SECRET=%s\n' \
+        "$(cat "$in"/dns-acme-web01-secret/secret)" > "$out"/rfc2136.env
+    '';
+  };
+
+  security.acme = {
+    acceptTerms = true;
+    defaults.email = "postmaster@cnx.email";
+    # One wildcard cert for every vhost this proxy serves, via DNS-01 (so issuance
+    # never depends on inbound HTTP). Port 80 is open only for Caddy's
+    # HTTP->HTTPS redirect, not for ACME.
+    certs.${certName} = {
+      domain = "*.cnx.network";
+      extraDomainNames = [ "cnx.network" ];
+      dnsProvider = "rfc2136";
+      environmentFile = config.clan.core.vars.generators.dns-acme-web01-rfc2136.files."rfc2136.env".path;
+      # ns1 is the only nameserver that accepts the acme_web01 UPDATE; check
+      # propagation against it directly rather than a public resolver.
+      dnsResolver = "${hosts.ns1.ipv4}:53";
+      # Caddy reads the cert from explicit file paths (tls directive below), so it
+      # won't notice a renewal on its own — reload it whenever the cert changes.
+      reloadServices = [ "caddy.service" ];
+    };
+  };
+
+  # The lego-issued cert is owned group=acme; Caddy needs to read the key.
+  users.users.caddy.extraGroups = [ "acme" ];
+
+  # Reverse proxy. The explicit `tls cert key` points Caddy at the wildcard cert
+  # and disables its automatic ACME, so no extra issuance happens. Backends are
+  # dialed over the mesh by their ZeroTier address (mesh.hosts.<name>).
+  services.caddy = {
+    enable = true;
+    virtualHosts."grafana.cnx.network".extraConfig = ''
+      tls /var/lib/acme/${certName}/cert.pem /var/lib/acme/${certName}/key.pem
+      reverse_proxy http://[${mesh.hosts.control}]:3000
+    '';
+  };
+
+  # 443 serves the proxy; 80 only carries Caddy's automatic HTTP->HTTPS redirect
+  # (the Hetzner cloud firewall also scopes these in
+  # modules/hetzner-firewall-rules.nix). Admin still rides the mesh.
+  networking.firewall.allowedTCPPorts = [
+    80
+    443
+  ];
+}

sops/machines/mx1/key.json

@@ -0,0 +1,6 @@
+[
+  {
+    "publickey": "age1l5hw95p5h4sthrgn0usms9yfkwwmcvv34tjgrtv9s4e6x39chacshgxavs",
+    "type": "age"
+  }
+]

sops/machines/web01/key.json

@@ -0,0 +1,6 @@
+[
+  {
+    "publickey": "age1yey6gxgsyl4tj6ek0tve2pckt6qersqspk66ukkzum8mrr6zppqsj4jn3m",
+    "type": "age"
+  }
+]

sops/secrets/control-age.key/secret

@@ -3,12 +3,13 @@
 	"sops": {
 		"age": [
 			{
-				"recipient": "age1yubikey1qtc8v5yqy3g5sh4trwwdp7elmavvkvkvzc4tfdnv2g8wyd8y5lc064mpv34",
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IEZXQ2lTZyBBMXZ4QWtr\nQVRaeHJnSlNyMmdMOVN6MmRMTm9Fc3FEbURsY1BLamhqS0J0SAphb1orQjJtNEwy\ncE16ejVRS0c3RXdjdzdUMUh0bFNTTGs5YW5WdS8xQ3FRCi0tLSBHWnlOOGZVVXMx\nQnBKMXd6MktTOTU3TmpRSEgrTGFuM2o4NGVlZlVmZjRjCsBsIrVfs218rztmmAtJ\n0iFU7ZcoUsEixjUogQ7xuoBppiRcY3jWz5CtlvpaMheXRv1DqrHrwK2i1kLHwhSO\n/s0=\n-----END AGE ENCRYPTED FILE-----\n"
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBcmJ4cXNt\nL3k4UmtqQUJZOGRoZFhab0pRSTdBUm9ZZkRxdHNvaCtRc05JVgphbHhHQThiRDNF\neW8xVGhXQ1pvSkRDR2hod2Zpa3pwaWZDVUpGZlpPWkNZCi0tLSB2T3JCcElnbVU2\nekJ5ZTdyQzZwbmR4aHBJUTc3M3loUm0wbHpHTGM0ZW1RCs0kHes9rkjd3uwHzxhi\n8x8AcQcQTR+CGsV5XPzGVMKKUZpwXWeNnvcHfVwYcqOnXucbUdCxLc6d57c9GQpC\nd3Y=\n-----END AGE ENCRYPTED FILE-----\n"
 			}
 		],
 		"lastmodified": "2026-06-14T05:20:21Z",
 		"mac": "ENC[AES256_GCM,data:dnvYlFK81wOynOkj0rfiOpGSjoxMMFs5BIlOc7tt3tnpxRRPu5IsHVIyRM6/spU3Ajwv9WMgbp+RncCorlkFUNZxZ2Xc11UGsDDCW7ddV5CHLNdXGpeAbHnIjehJTJehjitbt/Fan2ac9bnzQlIV1bjydhJ0G1A9cDWMQs6g5VU=,iv:D/YRqIFpMHQ0zJXA/92l3VHgUbhzOW7D4oHJQi0+B/Q=,tag:GjcLVcZ+gyQ9v4yaZeHnTg==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
 		"version": "3.12.1"
 	}
 }

sops/secrets/mx1-age.key/groups/admins

@@ -0,0 +1 @@
+../../../groups/admins

sops/secrets/mx1-age.key/secret

@@ -0,0 +1,18 @@
+{
+	"data": "ENC[AES256_GCM,data:lAgbSvyxBl0/NG8rHweKEEqYyDsHa+SrPZnxVubB2x5H3cqydhTCs8NMxJp+RKTPW2nhfmB/lrKgLHFfFMJfMg5jZ2BvZtR1IzA=,iv:1PgMbra/ec7yiCm7K5yo+1lCLJ89ryfP3SQML7uYv4k=,tag:lmaG+dZPCLHtQ59uFeSkVg==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBELzcrdUc0cW9pSXhGbzRY\naTd5aWgzcFpLQ3MyVElaZU5JK2Z4ZGRtMXpJCk14bmZLRlY4Z3pDeUxPM203anVY\nbE1aUEdDZ3NkcGxrcCtWS1ZmRWIrcmMKLS0tIDVTOUJhenR0WVhZSWNWT2VtRFpv\nc1h6Nk95Z3Y4eFQ1NUdmWWJiaWdDdGMKB3whh/RgAePTJnGmeDJ/WFv4NI42vA5O\nB0F6jmSDNa5beP8Um2DjWdPENkJJjv9yv38b7hP8BLDe9Ba4WBNfBA==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBby9pdmFZ\nOElJVCtUQmJWVVljQi9KcHE2a2FaeU5HeUhyUm05Zm0wZDZKVwptY2hHRFJKMi9I\nN3NDY3ZjYVloTjF6R0l4RllDS1dpNzA1TDJDc0hGQnFvCi0tLSB1akhUWk9RU0Rt\naDV5MHdHcExlSzZWbSs0S1A3NGhES0V1U0pEaUd5WE04ChtGuEq0HnRiVTDwhJnO\nIWMhwCYaewHk+k0a0Z9qCwqqKxhfiGS6kg/YTKHTNhQ0bxIA8yqgQoaE8Hl5Zhzs\nKqk=\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2026-06-18T04:33:03Z",
+		"mac": "ENC[AES256_GCM,data:Iffy/eS80qOJLdaGOIxti1otdxSLHA1TV06R8xq4zI1qFklJX8OWRy9CAYwAcTURS88gd5c2VGHsBI8yHOBj7LKiLJiu5/xRgD106hotBOJUy90UZ6MMbM2wwHomhQg5r5kNpRsJaPFaLFzm5YJN9gLUU1kxO4Nnt2L3bxZTogQ=,iv:cqbFD3ZIJnaZU7j7l9Qt1QqiUSQtclV7MGZWEZRZQoY=,tag:F0ZZQSQKYtBI65uutjNxow==,type:str]",
+		"version": "3.12.1"
+	}
+}

sops/secrets/mx1-age.key/users/berwn

@@ -0,0 +1 @@
+../../../users/berwn

sops/secrets/ns1-age.key/secret

@@ -3,12 +3,13 @@
 	"sops": {
 		"age": [
 			{
-				"recipient": "age1yubikey1qtc8v5yqy3g5sh4trwwdp7elmavvkvkvzc4tfdnv2g8wyd8y5lc064mpv34",
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IEZXQ2lTZyBBOW5zQ01V\nTUsrL2FvVVdvOGwvLzI4TGpqSmdRM2pBWndjMUd6K1BlVFY2SAorT2lTZHh0OS8y\nR0hqRER2Z00vYVFRS0lHaG5CL1BjSEw1MVdrYmFZU3VZCi0tLSBvRjlQendYR1V3\nSjdjcVRIbWIyZ05wNURXQjRnelAzMm9hTkc1Tm0yK0IwClY/Iz/DcGOu5pqq44gH\nx3CP0XY2db0PMkY0jlk2v9Mp32sshiBpmaMuFqz28JH5gwTbI2r0nXwteu4GM0EH\nN5Q=\n-----END AGE ENCRYPTED FILE-----\n"
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBemRVWUs1\naXFpVm80UmloUUpnR1BVaGsxampHV1JFNWFnSUhiY0orOTU0cgppVEJ5QVcvbGpC\nZlErd01XUzh0MnZOekkvTkc5Z3owS3pBbkNCUDJ6SlJRCi0tLSBRVVgrd3hIVEla\nMEhoN3FDSG1HdGk5YU8ydzlVUGd1RW5zM1lka3VuSHFNClrBItoFo7uqpS5fuvaG\nOvh6mTylk4bHgcSRGKTk383srgsvPdZaioJfUs7fbVmLD3+bhfM/Wgv7tI6kO1dj\nmDg=\n-----END AGE ENCRYPTED FILE-----\n"
 			}
 		],
 		"lastmodified": "2026-06-14T05:20:24Z",
 		"mac": "ENC[AES256_GCM,data:+9s1F7y7mE5LCmIT23W2gCwRLagSkxu1wvNqyxt9fGG4Uxj1LOspq9I+spNdiaRo5EfYowNbQLsuq/LD8VMy9RCz9/YZVG41Hskv1MGz2TrNZ9IeXS+VYIGCkxeWNERvKDDy+o6cPcU39JR59lqR0OE1pdEl3tOsLE+9XDNRBHg=,iv:Hsh/iaObBUjTBQiGNubA/Z/b4guQFhE9wln0yLAT3nA=,tag:E7mjMixGNipsdNin2dQAcQ==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
 		"version": "3.12.1"
 	}
 }

sops/secrets/ns2-age.key/secret

@@ -3,12 +3,13 @@
 	"sops": {
 		"age": [
 			{
-				"recipient": "age1yubikey1qtc8v5yqy3g5sh4trwwdp7elmavvkvkvzc4tfdnv2g8wyd8y5lc064mpv34",
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IEZXQ2lTZyBBb2U1Mnc4\nOWxRcngvREJYSXdNdjJpTHZnQkdqZks2RWtUODREaURhdzFubgprTDd0cmxRYXdn\nTHNYblpUcHpBQmkyR0xGTm12VTNTR0lZWlBRMjQ3MlRBCi0tLSBhZ0VHcUVCYzAw\nSkc1MkhKRGNtVm96SlhQQXZwblV0YjdJUzE1VVEyMEQ0CmkQWSOEJOFrDLFagXyd\n2cIneWoyVtUtHZqFkuc0me4h0aA4PZ/JZKA9JnA2zbl34CRhzW8lmwchLJCFWarU\n+V0=\n-----END AGE ENCRYPTED FILE-----\n"
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBL2doZFpO\na2Y2QzZSckJaZmZsTlFtQU1mckxJWTBiUXA2SUljVWZDemFJYgpwcFl6RHh2ZXdn\nNTBsaEVpWm5oYWtteU5WZmxWK1RRK2NoYW0yWlpCd3dBCi0tLSBsZVNpczR4dUdv\nSmtqQzMvc1ZYY0xUY0JMQmt2YzB5YmZOMEtXcW9vWUcwCnCdgcgiwF2rkWsV0IdO\n/6cs35FBypWpfflGwOwP0GnTrNizc9HphcMcwAi5NMQVi6X90Xgm5aFdpAsSWepX\n/SI=\n-----END AGE ENCRYPTED FILE-----\n"
 			}
 		],
 		"lastmodified": "2026-06-14T05:20:27Z",
 		"mac": "ENC[AES256_GCM,data:0EX6VnkasTfYUvJuu7ojg8+T+XjMvMmHtaS8iqzDADaLi0W1/PfwxoAFduSjNVWpZI/Qc+mBmkRSqqVOWsOesyQhDbPRGh6TKBar0IYRb63g/trxri5niAHeMxiMEv/XBYkmRWSViThKz+TgFBEKrnok0AddppxgQ7dlXQhGETc=,iv:hxcmEWg0pmVmnHMzmrjwZcokFBE6WQudkFrtw57GyU8=,tag:d0RhkrwjMtNi3iexA/sYBA==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
 		"version": "3.12.1"
 	}
 }

sops/secrets/web01-age.key/groups/admins

@@ -0,0 +1 @@
+../../../groups/admins

sops/secrets/web01-age.key/secret

@@ -0,0 +1,18 @@
+{
+	"data": "ENC[AES256_GCM,data:u+O5nWbFlvp6SGyHJggfkCLCT0ZuDy89e/VGGQNdt3yYzgdNmnrtd+2q+Ft3MtoOSSCLvStriGQzfLhcqEgqGgt3PqfIzCO1IG4=,iv:GyrZ1XUiOZe1I1Z/HebTy2NM2tfDHxIH5zGVk7HD+xQ=,tag:js6fbXWajVZSxt0hmnnA5g==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBWenArV1pZMURzNElTeGlC\nRSttUFRwM2E5d2htVDlFcTkwVDIrV2JxQjE0CmE0NUZ4UERHdHpqYk9uK01GOTFQ\ndW5UNlRrY0hvU2pNQmpSanh5RG5YbVkKLS0tIFdYQmkva1NORG80U3dDSWszZTlO\nOXViWUxMTVR1NE00cjdpVXpVS1J5YU0KHBkeKAJZDc+R1GLKwDYLyQBlEW7tPnMh\nf3tsUvtD0flqPAXNeDgyOmKufP7U6oDy/OriFC9+zYQbWyEEc6CZHg==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBL05xT1BJ\nSldCSkFhaFErWDNMeXMvRksvekkwNUZLZFVtSmUydVRrNG9JTwpFaHNsVThZN2gw\nMlNxUXhnN2xYNFluK1hadGxzMFZuaWR3cVFRYW9mWndVCi0tLSBUUmRMSnJCeTRM\nUVdKY3hzWVRkQkFuQ3FaRDVZSEp1b2N1ajg5RnlhT3VRCjX/vWj0We88ATiz808w\nz60RL0BvDGJ6m1BNmqAdtfCCClH33YXQBGrKT2E5elvOTl0iOCrT7HPjzXxJZXuw\nkKg=\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2026-06-20T19:26:29Z",
+		"mac": "ENC[AES256_GCM,data:uD+L3op6ZPtvmLreJ7S4GE6bQuItV87w1LTMFvjI1Kb5+Z0sXlL0TpYO8WLx8X0yaL3HddwlmBKYQGp/OlRPqZMFDbQuK25oeKB54jfm6YDn66rrMQJl1FOw68fLJaHYNjNelEg/bj2WG7YpfZBoWO67MW6F+44Rg4XF85w/1x0=,iv:X9TmEL5JkcC9waLumWpgpBwp3YWLMslZi++dv2HZ0mk=,tag:i7c1CxT1Xv/+T/jY9E0cdg==,type:str]",
+		"version": "3.12.1"
+	}
+}

sops/secrets/web01-age.key/users/berwn

@@ -0,0 +1 @@
+../../../users/berwn

sops/users/berwn/key.json

@@ -1,6 +1,6 @@
 [
   {
-    "publickey": "age1yubikey1qtc8v5yqy3g5sh4trwwdp7elmavvkvkvzc4tfdnv2g8wyd8y5lc064mpv34",
+    "publickey": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa",
     "type": "age"
   }
 ]

vars/per-machine/control/emergency-access/password/groups/admins

@@ -0,0 +1 @@
+../../../../../../sops/groups/admins

vars/per-machine/control/emergency-access/password/secret

@@ -3,12 +3,17 @@
 	"sops": {
 		"age": [
 			{
-				"recipient": "age1yubikey1qtc8v5yqy3g5sh4trwwdp7elmavvkvkvzc4tfdnv2g8wyd8y5lc064mpv34",
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IEZXQ2lTZyBBclYvY0RS\nMFpHOEpQU0kreWVlTmhqRkxzZGp4UnEvdGcybFRKSE4yYnZXNQpSSGlOaWRWcHFn\nNTk4S2J0dVdvSXk5RzJEaVZJY1VYa1hMcWdleGpRdFdRCi0tLSB0TUJoK3UxdVll\nWUxnaUFqUEExVFE2VllLanJSM2hJSHpxTlpFUVZqRXJJCuAtDqWR4cbc3snmpF/E\nXZn5hBsdyXaxRR1t5ZUCrKBl3zUBq6h/o+tUR105q3QWdm8WY2itIbc8ZEZ7hqi8\nCtc=\n-----END AGE ENCRYPTED FILE-----\n"
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBcnhlSmZv\neVlFR2JWNjR4UWpTeUY0ZnRtR0IyU2tHc2I3TXFnelpyRGR2QwpRL285aTgvNExj\ncWFyV1ZLbGZoOGVMUERldWYrbjNSUjRkTU1oZ20rTU9FCi0tLSBqdFJlNEJmMVYx\nbGYwWDZ0UlI2WEg2ekdDckRRK1hFNlM1STVIZmtzQ2JNCidXOYx35lyp24WGXydR\nEUHxxlCZuOhUZ6fjKY6p2/mKNQPqt41hq+tvf7PLb7xsJLUVJARnQskqo7NCH3ma\nsV8=\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBGdEt1MTFDSTVKWE9ObTAw\neVUyZzlLUW5WYm1VZE5waWI4czJUOXMxR1RBCjJYT09teGpWK3Fvcml2Rzd6eDhU\nMTNhVDRRaFZnYjJpUUVicytRZlN6d00KLS0tIEMxc2dtbUVsTmI5KzNTTVVrSnBu\nQ2lvTGJQcWNhVHdFck1zaWtXY2RHNXMKHG464L9M0OuL6+uWL6CZa1T6b0S5//fJ\ns4QVIOHeXkSo1l1sU1rMoK1Hx+lipoNCYZiTlB6dhw6WH2B6lXCKiw==\n-----END AGE ENCRYPTED FILE-----\n"
 			}
 		],
 		"lastmodified": "2026-06-14T08:00:24Z",
 		"mac": "ENC[AES256_GCM,data:xbnHfzQnGPW6uwH5e0j47OUuwaaaVh7mOuBJGKtGbPEallPA8kAv4CxKQIwfOlVh2VjcAUoVa9sMq/1+cknbN8ubANQ5b8eBsL+dfz+9hcwtR5hrXxN85DsBgw1PVDLdxtFszqvzuQRjEWColWoUzL46ZGlqZ7luoxbVF1nPSak=,iv:xKHnp9UJDOGLt5YrYnGxFvLOrN9Qaumt8wp2WVYijMQ=,tag:/l2XipPDBiWYhHTU+vsC/Q==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
 		"version": "3.12.1"
 	}
 }

vars/per-machine/control/grafana-admin/password/groups/admins

@@ -0,0 +1 @@
+../../../../../../sops/groups/admins

vars/per-machine/control/grafana-admin/password/machines/control

@@ -0,0 +1 @@
+../../../../../../sops/machines/control

vars/per-machine/control/grafana-admin/password/secret

@@ -0,0 +1,23 @@
+{
+	"data": "ENC[AES256_GCM,data:olwE8Xo5y0hn2jDOgH/9A3iJ0O33ukUnjSjbF19oaY0=,iv:ZptLhMDSCHUSZAlBry+ILcuDOlEYZsGvgDBAxLzL9ms=,tag:nK4Ak54Hzz7epsc0Xrc33g==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1env5y2eey0p3dggs8tydwmtyhtwqylpg7spuququ4vvax6arzp2qjnm4tx",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBzZEpWbm5rWFZkQ2FhSS85\nSm14N0NKNVZJcmNZR1BFWW5ILzZiSS9MMGxrCjc3dGRCQkVwRVFoUWozejFqTTZu\nSHRtZUkxOU1OOFFWMlB1ZHZFUU9lMTQKLS0tIDhXUkx6ZmlLb0tYMERralVTQS8r\nYkdzZmVUV1hCamFVRlZJajFCdzBpeTAKfRbQwWPyDrCrqvfjsUFtRvCc1rEPAakA\n4HkbLC7vTrFqizqTKx+9n0zQCLRAQwEdM8HLb/vsBG0NsVRKOANO/g==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBclJ1eW1u\nb2xNQlZNRllOL2JPSEZLTEpPU1NDeWprWW1Cb1F4RWVoalIycQpWZGc0RG9BTDhV\neVlBc2dhaTFrdXMwT3RqRzBLUDAwTXNjQnFKc0RsLzNRCi0tLSBOam5QNmFlUzNm\nMEZKNm1nQlM5eWxTUnhWOXR2L0VWK2FLRndmT2hVd2lnCieOpQefykg9hfb+Tm5K\nDpgw/1XTw4GlyIzjp3sZAZX7dSLPPtZix290pVX68rxedQFmHWds8c4+UmLhZAe7\nlSI=\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA4dzlDcTc1OE5VajJONUtp\nMUY1ZVRoVmpLVDhpSzBnTVpoOTAvbWloN1F3CjRxb0xhZWVlN1dMK0dqZmJJdngr\nRDRyWlo0aXRJM2RHUmM4L20rQmhrVEkKLS0tIDBKRWZzNlg5NUZBZkRlQjNlSnVq\nVlNHc1E4LzdTdWVmMHZyZEdTdHBEZEkKTcUjUOuEBlIi5IW/JcdCuDxPIWRSdIQM\n/VBQT8p2x7n9dLPFmE4GkOaOsogussAnKTltFCopsDa8uymCFjAzAA==\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2026-06-17T03:17:45Z",
+		"mac": "ENC[AES256_GCM,data:sc6zUWctq3TL7TNzsV7joAc2I6+BkjUHN64e/SL3YVLICkS5Bz5rMFew1WEPxVWA/DSNCri1JxltmcdmpvYTNtIkcs6P0Gs+Mr19aW7HLyvUZ9FHBCx29lvj0RAG2Xx2g2iBCLY4tMYRaaBt3JcvxIgAhfo3+O31bYEF2/m1IRg=,iv:zsWgdszzm0ReFOu0VJOCu2RlJ184EnvMw6Winb9c32Q=,tag:6pr704vsIBo8OrEgyhIskA==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.12.1"
+	}
+}

vars/per-machine/control/grafana-admin/password/users/berwn

@@ -0,0 +1 @@
+../../../../../../sops/users/berwn

vars/per-machine/control/hetzner-firewall/token/groups/admins

@@ -0,0 +1 @@
+../../../../../../sops/groups/admins

vars/per-machine/control/hetzner-firewall/token/secret

@@ -4,15 +4,20 @@
 		"age": [
 			{
 				"recipient": "age1env5y2eey0p3dggs8tydwmtyhtwqylpg7spuququ4vvax6arzp2qjnm4tx",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBLUUJzTE9zemJIVlJ3aW83\ncjJPdXlyVUNUOTB1L29OWWFaK0JZazZYY0NjCngzb0J1OVE5dDZuWmFIRnd0WG1l\nSEx2c2lmYklBOVBzYzJGWHkxVTlMSFkKLS0tIHdmZEZUQjRJMUhubFpIYThwRlNF\nWjZQMzU0b0VzTEMzSXFIcW5GeVcwR3MKcTEEpaf7FzzekVOilBqK7js5wqrPeAx8\nt7bwVlo2/qOZm0N/gew+H9E7G0D/O02V/OdWfDOkWrZNWH5VcQlTZg==\n-----END AGE ENCRYPTED FILE-----\n"
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBBYWdrcWFPQ1NCZFlOK3h3\nR2hsQ0l3ZHRPbUpyNDJ3TUVEeWhWV2V5R3l3CmVKWjQzbWxrODZ0bGNSTGx2enVS\nNU5HZ0dmOHJ1VFdIUC9RRWdmdlk3aFUKLS0tIGxRMzd5dlllWmhoOGdnYUFhaVhO\nRGtRRWNORGtCWmd4SVlsWVJWWWNJdFkKUaM2A71BynfBFTTmuB58ltTCfglFNQAV\nKAulSz663JG1KTGsQwQv0rV4kH7qTBk86KLZsnFOpTwoTXBggBJF2A==\n-----END AGE ENCRYPTED FILE-----\n"
 			},
 			{
-				"recipient": "age1yubikey1qtc8v5yqy3g5sh4trwwdp7elmavvkvkvzc4tfdnv2g8wyd8y5lc064mpv34",
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IEZXQ2lTZyBBNkFSeVRM\nVkVWOU9pUS8yK09yQWRBbWh1MXI5M0gyWEVyaG9QR3JCQnA3UQpyem5pYkQvdWdt\nYy9kaTBQd2tqZGZ1bUhVdTJIajJmUkNYNU9ZU2kzT0JFCi0tLSB3UUEvMzJvdXc0\nWWhMb2dqQ1lqMXpEaUZHeUMwWkZZQW9tZld1b2x1ZFRZCgWvS0oFBZYd8DZmfFUF\n+yqFGppcKoIEaJPn1vQ39LvLaubc53YPJQb9wODLejxc1H9iPE/CMQhDU1W5Ccae\nooI=\n-----END AGE ENCRYPTED FILE-----\n"
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBbmdQbldm\ndnFWQ0pCMGY0Ky9Yd1hTVkNnS3NYRjkxWDBTOVdzbDltNjFvWQpCTVJjbVIrTWF4\nT2U3NHBEZTR2TXFyWWE4N1FZUzFrNHp4YXFCSnZid3djCi0tLSBFOU5sWENUQzJk\nSTgyK3orMVhlTXFzRlBRTjlkNHRLdTZDeUJwVWtYZ3U0CoeANngupnQqFRgmAFzg\n0mI4QhBA7Y6kYRXb5Ra2zalKWET9+YccDkfoNGX0qDYgockH8CBNbRRDeAbMFmTH\nWug=\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBkdGM2V3dsR1l1RkNZNGZM\nYzduYnpXNVRabHozOU5DbGJKTFVrMC9jMlNnClJiZ1Q5V0ZaVVM1VnZlUkpVY2lC\nRkd6cnkrZmNIYU5hYzlFUkdXV3pYYWcKLS0tIHEyRzVoZ3ZOSkkxMytnUlBjQ0dZ\nUVJkT0YxZU9Ib25xd004V3N4M2YrbzgKIAV6yr2mHRcF8EUpLSyDKbhqIUtpxyCo\n55JfLNuvhO2lvg84Y0sGdcRRWmt/tvEsxF9mtCpbtGOBkeZh6CVtMA==\n-----END AGE ENCRYPTED FILE-----\n"
 			}
 		],
 		"lastmodified": "2026-06-14T08:37:25Z",
 		"mac": "ENC[AES256_GCM,data:P/XCNw+t81rsNdtiYgAtcFwF2pdKgmpqw+CjTLLacJvyAsN54WOLhPeBgow3vfZEHkgipHSR+IdRIOROe/eT8q5Aatqodq204IgRcY2MwsoriuZYOMXIgoC0jmPWjpL2CRKyNZLSuAcpsCFsqfT+KYMq5iUTH2MIsibZAOj9vow=,iv:BA6GNzzVlDAV/Fi+Me722oFPustMsFVXFUr7NuQKGkg=,tag:cHi0eck6B23BW+/Uy7dcCg==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
 		"version": "3.12.1"
 	}
 }

vars/per-machine/control/openssh/ssh.id_ed25519/groups/admins

@@ -0,0 +1 @@
+../../../../../../sops/groups/admins

vars/per-machine/control/openssh/ssh.id_ed25519/secret

@@ -4,15 +4,20 @@
 		"age": [
 			{
 				"recipient": "age1env5y2eey0p3dggs8tydwmtyhtwqylpg7spuququ4vvax6arzp2qjnm4tx",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAycVd6ZnJVcG5lc1VNaWFK\ncldWTDcxTHgwNjNEdjJVTng1VnBTdFZwdlhBCnJuTElRVDRDdFNUWU9PZnJEV24y\nenN0WG1rbnJTVGl2aTYwVTdIOGtzYncKLS0tIC9QM3JtSE9jVGRyQTZyb0QxazhG\namg1eitxb3NabGhlRkVodG4wL2lQUDQKM5jv9EmOy0SWrIYmrOOUYeiDPJ1g5QSL\nBk5mCuQz2hH5N0zqbTsef0aIfP3FUpuYqVqzO2LcgH+opKh1kwFs1Q==\n-----END AGE ENCRYPTED FILE-----\n"
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB0QVRoZXRFd2pFUVExTHMv\nN3F4bmdmVlJad3NqSFVZLytiZGl4VDdrNkFVCnpEWjZFOFB0QXU5bVNhTWtiOHc1\nVkNmYUdqZHdKYXgyMWtvWnF6RWs5V2cKLS0tIG5PRzkrLzYvOFkxMW1OZXBTVHM3\nT29vR05NZEtiSzhjcjBnS1dmTHp0c3MKaaUISvysH0sRsBNiZ//koype4DI3uwsS\n+qMFJi6RbBOGSJCsZ+iCuOF/cZ9VMxDJEhgKZ0Op/wcCCmtH68nLMA==\n-----END AGE ENCRYPTED FILE-----\n"
 			},
 			{
-				"recipient": "age1yubikey1qtc8v5yqy3g5sh4trwwdp7elmavvkvkvzc4tfdnv2g8wyd8y5lc064mpv34",
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IEZXQ2lTZyBBM2d2OXR6\ncWEvUW5sRTRtYUZmSHFGOXVvSDBFYXkrcjFhdk85alBLMHJYaQpRRGJPTWNSbHV3\nbS85dVNtZTFVdUIrbmRMT252YUN2c3pzS1pPcGVaN3kwCi0tLSBSRE40KzdoaTNO\nY2dpT0dPNzFTUTFUbVFqTVpUeTFuaTdLTXpVdUdyWVlNCuFKS2v3bB8aOIqG4Lc6\nml7QdEWEWaQqo+pomXjXLeGmiG1dcDm8D+ovVPk9HJZKbws9bR4+oiK3oc+GHylg\nR2I=\n-----END AGE ENCRYPTED FILE-----\n"
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBbVY2VERR\nMFd3ejRLZ0VkMWdNM3N4Q3A5S3krSElNOXJZYTQ1bGZEUWhpVQpkU0pkREhBN2tI\nekpoMkFBVFNucGhEbUVDZ2hCd0IwNXlIcHVCZ2Q1NjRzCi0tLSBpdnZmaXBQOERt\nT3k4STFhY01TYnNTcEJxV3pZYVdYVitYVnJ4QWFndkc0ConYEcAH4fsh7KDvscGd\nr0v275je2GQlWV1NeNe4M/058keWGHqGaHKW5udkbCjqONB+xi4azmHExRi8HUiw\n6/k=\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA3Qlpvd2wrbHphYytrUzZu\nWUkwaXczb0dZNGJzRmlKb1lVM0dFblA5WFI4CkdHeHMzbjluaExsS1A4L2JrdDh0\nT3hQdkl6SUpOWFR2NlphTzBFUWpMVFUKLS0tIGd0S2hTN2hMa0kzMG5RcEp2d3Z1\ncm5MWGdKME5mNE1ldjEyWDNRZEgyUFUKmq3s3RyssFxhYvY2IkHqRyv5GTKu655B\nMKldyNdpprvXwwEHiPw4RZGpqXq8UQo2Humdv93iQ1lp6spByRCgVw==\n-----END AGE ENCRYPTED FILE-----\n"
 			}
 		],
 		"lastmodified": "2026-06-14T05:20:21Z",
 		"mac": "ENC[AES256_GCM,data:ce21GxX15Y49Bpsy49l9OLiVPOliaOtKfPTUXZAAfd+u1b38Ov9hgabIcIeOsX29JgEa2qQNM4sVm0nHsC00MtTzogugyGVRS621Jntl+7A2hY8zOxOjfwr9V1VGf5WN+WWeA6ZQsquwD3QXBh66Royah/YNIyuoXDWK5iaUpuI=,iv:vLhpFNpJAMIIFiZ+Jdw9UsK8MejP1SN3N1SzaLa87Y0=,tag:3zYTQgAb5qRglf4bKJIOrw==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
 		"version": "3.12.1"
 	}
 }

vars/per-machine/control/root-password/password-hash/groups/admins

@@ -0,0 +1 @@
+../../../../../../sops/groups/admins

vars/per-machine/control/root-password/password-hash/secret

@@ -4,15 +4,20 @@
 		"age": [
 			{
 				"recipient": "age1env5y2eey0p3dggs8tydwmtyhtwqylpg7spuququ4vvax6arzp2qjnm4tx",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB5MHNacUtieEZwZXVnUkt2\nYU5pRzM2ZGJwUUxDcUxNcVB1ckhHY3ZHdGwwClJtNXRhWGR0bUUxUjdhbkpneDIz\nVi81Zk1yaitudWJLN1B2T00rSW1rVUUKLS0tIEZjSVNyTElSbFJJVjhpeVUrWTlx\ncnY1elZnZUpDRDVuZ2JGaklvTWZ3ekEKL3gFHBg0kGr+HYUzPmTcNdiZn3AHg3Iv\ntMM1q2iWHlCCmxMFOkNYzEEJV86RpfS/OmGV2IyYt9XF388dOkxwsQ==\n-----END AGE ENCRYPTED FILE-----\n"
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBzMGlKZURGWEFsUjZFb1la\nWjBmM1FWSWtYRUNJcHlLZnRzZklqeGFWTUJ3Ci9JRG9rY0Evb0FNRzhJMzVjR1pH\nRUZ1bVE1V1l6V3c2eGV5U3QxeGdqUlEKLS0tIGp6VVdwWWpIV3R5RndNWHkyL3BB\nK2VwVjdUalJOMlcyS3g3elJiU1MzZGcKzNq2lbs9M9KoHglFO+sMURxk7GfirGi3\nwJDkjzNROad4VNwAz0LknGbjH1q05zUsj9OuvCR4DjMGnf3ZLeVrPQ==\n-----END AGE ENCRYPTED FILE-----\n"
 			},
 			{
-				"recipient": "age1yubikey1qtc8v5yqy3g5sh4trwwdp7elmavvkvkvzc4tfdnv2g8wyd8y5lc064mpv34",
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IEZXQ2lTZyBBbWl1SDVX\nSDh4a0s5WSs3bi83T3NBeitXL2ZvaVpJZmZXOVZpdVlKL3dubwpXaXRUWlZuOFhB\nNzU4THFwWFJYYWpjNWM2Mnd4bXRRMDJLWEQzUlNLV1dZCi0tLSBCTFIydTRsWlg2\nbVVQTlB6YllsU0VMaWVZbFh1YjhzSitITFc2MnFpU3k0ClMPAFJ4EfhJhrsCEIeD\nOtbuT8njIxzB0cLVUsGhKVPz3O2dTerYcHOMMKS8amg36+FlSSVP2tt+Y5lqSkwB\nUQE=\n-----END AGE ENCRYPTED FILE-----\n"
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBcVNoRCt0\nMk5uSGxYVjA5RWZGSmZHL3JLZ2RXTVRYS0ZWNGlxUDVFTkpaMApZRXZNM1Avb3VU\namJCaFJRYzIzcWUwVXhxK1diVFBLMUFGaVhUZTlMNTVJCi0tLSBYbHl3SzI2aGxk\nbGRVdlhnUVlZYWJNK1VtUEQ1S1pUSk02UmM4U2pKcGk0CigQwb9qxgkOQIkBEm0j\n4pYpdnmULWs7H9+H5FNdeU8/gjflN1RTshTZJD9nO1f9V/PP2wxtXNOHGDxxOQnw\n0Yw=\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBDM21qTWdsbXQ2WDZvSmZh\nNjhSYmlCMkQ1WnBwVW1jSGRrWUxvbUhmQ1ZnCnU2dzYzSVlOb1NlanpQRDhNQWU0\naUlyRDkvUmMvajVsOVNjZ2diRVBDSmcKLS0tIHZKZEpWd1RYdFR6RXl1YzA4OEZW\nUkxiLzlvMVQ4Q2pxZlc1aXdpRUViSTQKGv59qG+fgX41DL10NVPbCDHxbmFbjsew\nnWIDuXAVAtdyxjX8xUWYiNFpEhYXe33JsnPys7GdNpTY/rQD5b6Skw==\n-----END AGE ENCRYPTED FILE-----\n"
 			}
 		],
 		"lastmodified": "2026-06-14T05:20:21Z",
 		"mac": "ENC[AES256_GCM,data:CbM6M3qWJNA1ZlmzW1EBDDZy7t12HYoP0rZpLpeYDqu37YL49dSehkkbDB9/WS0FCt69wPTH7nw/vkRzUbBDKRs/uyWmqiBfW5PNJqJQjWVHtzfZzNGrj0RVA5NyxhjPfdivytIOq/5b15zVyUas7IIWOv8yV/IMdDF2DpQcpp4=,iv:mLHVWBwwmJE8G22ceqdLK4t2GnpqeXkg5xu8RoyxLKE=,tag:TfVudyECxfNBjJGmTalo1g==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
 		"version": "3.12.1"
 	}
 }

vars/per-machine/control/root-password/password/groups/admins

@@ -0,0 +1 @@
+../../../../../../sops/groups/admins

vars/per-machine/control/root-password/password/secret

@@ -3,12 +3,17 @@
 	"sops": {
 		"age": [
 			{
-				"recipient": "age1yubikey1qtc8v5yqy3g5sh4trwwdp7elmavvkvkvzc4tfdnv2g8wyd8y5lc064mpv34",
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IEZXQ2lTZyBBbFZ0dnhY\nMHpyVnVneWhDc215OEx4TnhjQlFpQzVSaHFZeVJrTU5XaERpLwpDTlFhNVFZaHh2\nbEFUbERjTzZ5M2dBeFV5c3YzMkxkNGFXZmxGbnRzdFlFCi0tLSBjZ2lveUhBR3RG\nRllwTmpMVXU2WHI5NHo3V2JCRXpKUzRhaXhYWUhYYk53Cj0z22vXB3JnwIlqsax5\neXyBwEP6UOmc+cvE4HSXkwOQh7AwxjcdxagL3NKhD2o54sEn/xX8UYRuEv/XVtpW\ncuA=\n-----END AGE ENCRYPTED FILE-----\n"
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBbGMzaFRn\ndExLQ21oZ3B3OWpTUTcwTTY4bklyamRpT3AxS1FENytTK1UrRQpiTXVza2MvVTB2\nbUF6dkJHWVk3WkhFWURlM0tLekp4NlFZMUphWDVuNlRnCi0tLSBqMGN6Um84ZUVO\nU0JFL2hESnMwYjF3Q2swTGtnK2Z5T0pBT0UwTUJ2STZjCtsGjah4SHAo6ZSYKoD+\nGZDt3XcFeVrFfYd6eYUwR4AixRNOFm9D9SLuSyrZytn9ZvA2Ye36z1js7Tm4zn30\nbMg=\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKYmE4MFA5Y1NCMlVRYzZy\nOGZqVVNveWNWZEtGb1p5MjZzZlk2eFFZWUJZCk1ndyt6aHJEZUJ3dVZMTjNBUnFa\nK0tsVkh1T2dSTDZxUU5xRG1YQ0VVeW8KLS0tIGU4UWYyR3hUV1lXZlZ3Z2swQ0RB\nZXVpTFhoT2cyZEZzVVhjNm0rZkZjaHMKVMN4gErAPaEyNchsrlxfjnUZS+vZMrJz\nvHyUzaTJZZBdMaU85B4YoJP7X3t6HQawLR0AQHAGdG0CcbNh+jjZ5A==\n-----END AGE ENCRYPTED FILE-----\n"
 			}
 		],
 		"lastmodified": "2026-06-14T05:20:21Z",
 		"mac": "ENC[AES256_GCM,data:pgw+fV9bBNLITzCQ9+zrhiy2C2J2aCWG51DGJRifVO4Q/pPeQiWoYAkURhcFOVCVDksTfFAXTGcu6lTIIAGLD6vki62efsIhB0/Qj3QDaDBn/NcoTwP0zhY7iAdQlFMJTs347TkhpX7xpfpDIvHPKHfMrqru6Gx+U3Qg2OYYcxA=,iv:fLP1zMS7SmFSDRQnInsG6geA4ceQOEAaS2m/ecbeAjQ=,tag:BP0XmeH2/C4Cmx2LBwC23w==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
 		"version": "3.12.1"
 	}
 }

vars/per-machine/control/tor_tor/hostname/groups/admins

@@ -0,0 +1 @@
+../../../../../../sops/groups/admins

vars/per-machine/control/tor_tor/hostname/secret

@@ -4,15 +4,20 @@
 		"age": [
 			{
 				"recipient": "age1env5y2eey0p3dggs8tydwmtyhtwqylpg7spuququ4vvax6arzp2qjnm4tx",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKeTB1QVZtb3ZDNE5qZHlH\nVVF2VG04bytnSFVEQm1GdUhrKzNuOUxkbVhFClJEL0F5a1ZzZDV5M3luNE5zRDJi\nREI2MEovRUw1T25VRnA1bWJMbXZKVU0KLS0tIEl2cEwzYlhoNzkydW1LV1lISlM1\nYXNuZ25BanJ4STVKZUR0eStpQStLOHcKE0uzLohvvG9A/jzRPZDrJqoepA/eHVGh\nS4zwhQHbYScfzI9ppL8I1j1V82r8w0K1YzQFC2Hua1XgUZdVsKkxoQ==\n-----END AGE ENCRYPTED FILE-----\n"
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAwaUFjNW5HZ3pKcFlZVzlD\nQkNsR2w5WVpFNUt6eEFPdFZpTHJZZ3ZWT1NVCmlYZkVuLzdDL1g3aFpDcUtua0Jz\nM0EvMGVYdFBOTEJVNVBrc0QycWh2elEKLS0tIGtMUzVPVlllNFRzMllvV3ViN0lX\nYXp3bXBYRUxsVnZXNUtUeFlXdFVWQ1UKM9AbwjLW7U09t8ZPt5UcfGGcN2Kp2zi/\nAj8DEpuqaVAp4WAG6ZJnABunMrUi6nzU0m6kfAApewLLfEGyzHNMhQ==\n-----END AGE ENCRYPTED FILE-----\n"
 			},
 			{
-				"recipient": "age1yubikey1qtc8v5yqy3g5sh4trwwdp7elmavvkvkvzc4tfdnv2g8wyd8y5lc064mpv34",
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IEZXQ2lTZyBBdmtnMXpG\nQzRBeU1kYUlSbnd5M1VXQkRXWHZiREVhS3pIY2FoRTY4V0g3dgpIZHBkQVczRElq\nVktPV21FS1NtQzlDVTFBU0Zsd0JqdjFSelBJNTl2UEtvCi0tLSAycVdlSk1sK0Iy\nS2I0M1JveVdwdkI5WWMvUHJ0ajN6N1JuZHh6STFQUDdFCrLadWKNN+bi40vREbWD\nrJyilZSbh+DbZV2fUT0mbO/wOtr+8IOpaHzuw+zigr8RUfk0/Dae2kvUD0uMwBwa\ncOM=\n-----END AGE ENCRYPTED FILE-----\n"
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBaTFlNGEv\nbDlXMUdqV1dtWlBmcDN5d1dHblFyTE82dTNPYTViMUk1L2ZHTQpGa3FlK3BQdytK\nZmE2UTRiNnVJU2RVdmxwSzFOZlphaXpDblJVYTRxc2VnCi0tLSByWTM1Q2VvWGJT\nOEYranJidEVpWGVZeTZTdExQUFM2czdta2REaVRUUlJFCvY7h2bHU5fPGIPydiW3\nEzuKGbLzdH+hLoCJRF5Xi08cy+K4L1I4mtyx947kUL1M/KJtgpwTmHayXvhSJ1Mj\n5f4=\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBLQ1BveUFReG5heUNFR1Z5\nVlcvM3pKUVdoVHpVRXFiYXRDZXFFZ2o2QVZFClNNcHRNU2F4RmcxSlRqWlViOHVi\nRUhJL3UrSEJ4S2RkQmRYc25OeGdYT0EKLS0tIEgrWUd3N1JrWHFKa0FKa1dCMnVL\nQnA3SWVGQVFFTW13aEx5cGNpOTQ3amcKJQxUBrqj+I7Xd1/WqZoHAu8w7II+N0HB\ndrX89JwfP32HQCvdgTezH9RBfGHY236WfxQPiItShnD8t6pgzj4XNg==\n-----END AGE ENCRYPTED FILE-----\n"
 			}
 		],
 		"lastmodified": "2026-06-14T05:20:24Z",
 		"mac": "ENC[AES256_GCM,data:RdCsjdkin73/NorulDTfl71ykMtIbZROAJNtU8xyk9PmmNtn2gKMlSDCev3Lov45otKJkKwmvI9NKexijLQceiJRLFAnF66fV1JuyC6VYRVM2UCc92/brPIuE6az18q1yTngLxvvFd4kuWiZkbKfGDpmmMKkexuIeQeMQPvboBg=,iv:jqITlySAidiqBuGb0yEkzsQX31mmI8bcYFNiK7sXBkM=,tag:vrpgk/eP/6AlrFQyf3RGjg==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
 		"version": "3.12.1"
 	}
 }

vars/per-machine/control/tor_tor/hs_ed25519_secret_key/groups/admins

@@ -0,0 +1 @@
+../../../../../../sops/groups/admins

vars/per-machine/control/tor_tor/hs_ed25519_secret_key/secret

@@ -4,15 +4,20 @@
 		"age": [
 			{
 				"recipient": "age1env5y2eey0p3dggs8tydwmtyhtwqylpg7spuququ4vvax6arzp2qjnm4tx",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA5dmVKM2VMS1VGMi9sNDBT\nOGkxL05SYXlQQ0QvdnFSZVNjUG94dlRuUXg4ClNZVzhLSGJNcVlRcTAzT2d3UXNJ\ncHZCSVUxSHd0ZFFvNGZpcUNQUHoxdWsKLS0tIFFZbU5YWXVIcXV5M0xHZ2hKRVNO\na21FODd0TTgrRE40Rm5veGI4TnJhU3cKJrSYlrm3QoBvAuIc7wW7dH82aFl7TL34\n6emdyg9PBHcGfshDZ2Srb4696y+yB6u6vhcImAKWR01FKLXEYkegxQ==\n-----END AGE ENCRYPTED FILE-----\n"
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBFSkJiN2h1QkhUMDljZWc4\nSk5XKy9OT0lRalB4UVZNUFViOHFSWjl2cVZRCm1TamdnQkJYdlB3M1Q2VWVGZENt\nZHNqQW0wUUk1bzd3S3g2RzhWcEZFUFkKLS0tIDYvQjdScmJSdW8zaE5sZWZiZCtn\nYUJRcm8xN2wrcTFFMWV6TmVLK0VFWTAKcrAh/d8x2oy4CT4iGGg4jcHNw7ctzTnV\nx97xAGgKqWqt4wLvifT04MsQAby0rnoPMfIRkDsSO1EOrTpymKp6MA==\n-----END AGE ENCRYPTED FILE-----\n"
 			},
 			{
-				"recipient": "age1yubikey1qtc8v5yqy3g5sh4trwwdp7elmavvkvkvzc4tfdnv2g8wyd8y5lc064mpv34",
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IEZXQ2lTZyBBK3FFYldZ\nL0tsTE5PRVdYTk9UY3Yxbk5PV1l3YzdkaGJsRXZsdm90dHMzLwp1MEdCa1dNeW9V\nUHMwVXlxQlNJL2UvS0xEQTlGNlFuWkpWZDhOem5scTNvCi0tLSBrUHBtWVdjdG5W\nTDRwNnJiSmRvZUJQNE9KTEJRUjRtS09VdjZ2eWs1UnlvCuJrTGaB54B/jwVa6f1r\nucfELMqIjedVZW6sjefUYiVGW667HUYbJaR59moyodSmAuNayQF9mPqNaBdbj93J\n7NE=\n-----END AGE ENCRYPTED FILE-----\n"
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBbmRtdmIz\ncmFscXRYN0FPeFdqRm9NUDF1dm5WRXYweEJ1a3NpUWxPaklBRgpuaTVPcEFVMUoz\nK2FBUU9CNzVUd2RwdXlobTZwS2JlT0V6WmgyQ1YvR0h3Ci0tLSBZWHBVaW1SY2ky\nYWFzcGI1WFg3eFk3TUpweFc5TGV5SzlaMlhrUEVOSklzCt/SU2jg/AQo5TB1sT+N\nZzluCIHqhUIDCLGz6dWpeZarw4B3WtqFYTa/phpRmSPOW0RbzdIMj/z8ydmt5AGs\nPOU=\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4SHc4U1hNd0psWHNnTjV1\nV3VXdWM0MDQ2cUJPY2c4bitkdEdPTEZBb0FJCm5teVJSeHZQTnpoM2VUZmxQTnE5\nNUF2eXJHeFRyakgrUGxmcGNRVlllVzAKLS0tIHJCdmd1NXl2Nmpua1kyUkVXUmNz\nbGVvb2Vrc1V6MEFXR29QRVgvd0RIamMKXAHq39YzpRRGrAyqL5r5vigIWrlIXOOr\nDHoFGSglV38OwScIz2OkAELIwQ+B8QVcxbVejE8ntStjF51yJDVtgQ==\n-----END AGE ENCRYPTED FILE-----\n"
 			}
 		],
 		"lastmodified": "2026-06-14T05:20:24Z",
 		"mac": "ENC[AES256_GCM,data:f6aflOmCHfV7h735fH5faZjWFmuddghF8ziIPK+Yp92Y40MKWIY8G46TSmO7sm+xD976d+qYgdG1tPoJCuMOawFoPk2fG2gpsss9m0UcdS6iKK5KzryzKFS4gt1TbmUhmAktTRnpGjm3WZjNiFk/cCXdyWAPfnbdBbb8H6rZJt0=,iv:kfMs8bpNh4mLTccQf9L+YPTP42HxySTgVRwz7zr6CEU=,tag:Bs9oCDBjxb/7IeRp3U8BTg==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
 		"version": "3.12.1"
 	}
 }

vars/per-machine/control/zerotier/zerotier-identity-secret/groups/admins

@@ -0,0 +1 @@
+../../../../../../sops/groups/admins

vars/per-machine/control/zerotier/zerotier-identity-secret/secret

@@ -4,15 +4,20 @@
 		"age": [
 			{
 				"recipient": "age1env5y2eey0p3dggs8tydwmtyhtwqylpg7spuququ4vvax6arzp2qjnm4tx",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBESDVTS2JlMGFRKzl6SW9m\nQVR0cDFOdHlqd2ZTV05vWkVVNHRPbm9rcFhJCnQ2SUZ0TXpwR1BxbzNrSTkxQmpF\neUc1MitrcUIzMmhPb2VJeGxxZ2ZxdkEKLS0tIFBnUmxuTnRaeXdGVHEvTCt1TWlw\nVEZZWmQreXhlYW5tcUpuTzZGWDBhMEkKSsl0+StzSq8emBuobG7LlreKWhyHHVo8\nGGqOIYQz+vpHswHeQX7AqIquBVdFnVtfKKKOPGkQ4rAj7V62WzZRrg==\n-----END AGE ENCRYPTED FILE-----\n"
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBVWVJjQVBhL0pGTEZ6bkt3\nUWlnbkY1M2V4K0VLZEVFR05pQ09qOEtHUEUwCmRtcVhtRE02ZWtwem5MVUxyOE1u\ndVk3TGxFelBvVGhtL2pmUW4rMXNvYzQKLS0tIG54NHZmelJiUUZPUFFMMUJLa3NK\nL1ptcDNES3hyNjBYOXlrZWJJNEFkOFkKOPrWPmPTqxLwXEK9BvNyEx2/FGwZOQrh\nBD4TWIpuRb+GyhbmGGhi53q3nAPzBSGhCFZOV9eZ4U998d50STqDBA==\n-----END AGE ENCRYPTED FILE-----\n"
 			},
 			{
-				"recipient": "age1yubikey1qtc8v5yqy3g5sh4trwwdp7elmavvkvkvzc4tfdnv2g8wyd8y5lc064mpv34",
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IEZXQ2lTZyBBdklFY1lH\ndmgwd0cwWUlmY2dVU1VubUxhSGROSVUrU01YbFB1K2k0V256OQpjR1Q1Ly90M1hO\nYlRoTDNiSjRkd0FmelltajJlYzc0UThhYUMra3hsSUlVCi0tLSBaUE1jREZ3UVNV\naTdpRHppTUdqTDZHOTNEa2lhUFJ0eGZNZEk2Mnd4Y0JBCp4q4FTg8GJowM1VKgmw\nBaD8cTmopi5LqqemM0YchTbfWPLH4du5uGIwPeVi2pp3ooTGBFCMI5w8W8EkajXY\n/1I=\n-----END AGE ENCRYPTED FILE-----\n"
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBMVpUM1J2\naXptaTNZVnhhME5pVkhhWTJnbGJieG9yaU5DOHhqQkJ2akxyMgovQlh5Q1dSbDds\neEs2WHBHR2NwcDQwa005VERPSXBnbiszSjdZaEt6a0RzCi0tLSBJYlFuRzFOcE0z\nVU0xbTR1ZWFDQnFpa2oyOFVsWmZJaGlGOGl6VWMrbnVJCmTY1XBYv1THTRLR8riq\n60LPsfHGCzCY3LAz+G+Ve2Y3LKBfiv/a0QFwcT+PEnfqectzPgnIRpLndazwBMtC\ngZo=\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAvNkFGYlYyUjdqS3orMW80\nbmJvejN3dVIwNG01L1RMT2JidjJlaDNWUWtBCjNKbEpjOVd2cGtZQkFaNVFzRlNt\nREkvdnQ3bHNWUENPclk2c21LUVBZTmcKLS0tIEFnNFNtUVA1citYWlhNUCthNVdj\nYlJ2UTdNdWQ2aWZjS25VQnJyNmN3RGMKt/UVKBZoyKMexhCOAPLUJKKg4SjaNPdU\n9AnkzkPf80jKp0IQD7dkswX10gq8a4fAEBq1cWFXdmlcyXIAn7c3DA==\n-----END AGE ENCRYPTED FILE-----\n"
 			}
 		],
 		"lastmodified": "2026-06-14T05:20:24Z",
 		"mac": "ENC[AES256_GCM,data:uKesL9zfe+XfKveLjdKi7HrPEwN41lcTLxbvUdMbnRrsKeLCwaWLkZbjnqplQzqSRMKxtKplpPslc0TaJ4cvwtelx+YARRdYlBDFB1z6Y3e85PQFrhYXeVL8DSnGJw9HT6XaAmOINH7iLvgpLo920hF0LeDqNxjDLgJ8CMpBKXc=,iv:98mhBSUp3g7JQtEWVUFVNx2P1hILmS1JQ6S3BxMJF0s=,tag:ix3I2kz30GnX5MVdoaG7ww==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
 		"version": "3.12.1"
 	}
 }

vars/per-machine/mx1/dns-acme-rfc2136/rfc2136.env/groups/admins

@@ -0,0 +1 @@
+../../../../../../sops/groups/admins

vars/per-machine/mx1/dns-acme-rfc2136/rfc2136.env/machines/mx1

@@ -0,0 +1 @@
+../../../../../../sops/machines/mx1

vars/per-machine/mx1/dns-acme-rfc2136/rfc2136.env/secret

@@ -0,0 +1,22 @@
+{
+	"data": "ENC[AES256_GCM,data:Lkwniu1Pmu+kGb94ncTteN/CkBYE47+UJKRSij5APKyPa6wQkc33S00WVLSXkG6I/XGeRUAXxNpM7G3WqmkluBtJnuYdQ3+nLVdarDy015Zu207LbpaYBuDyMU4e5pxH2ekIGnyL7diDI/3/GG+fVrO4xxdrFPWaB0YDcD8+5mtpshUqa+4rDrU9CSikuRo+dkAHveX1+MFfpF0aJmRw2MMwXO4=,iv:LIOftLZQ63yEPJ5S08t97jGGkSUK1LxSMnvy9lEm070=,tag:+yYbq9RU91KB+6r9eC9/fg==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBDS0Fra0tVdHIwREVYRC9l\nZFNwek0zMWYvTkRBT1RKR3p1bTN4SmZncGxVClRoOFRNcklEeWllMW4wdzRCaXNX\nbWtNYUJNM2dGaDZqVGtVY0twZVhEY1UKLS0tIFUxZ1QyMWNLRGJrYkJTbTJCT0Fy\nd2xjbWthVG1JUW5XZmVSK3lWc2NLRGMK6/g42P7ZvAk+t2GmZammNxTLFMudK+Qv\nZt3YUF0+EYKlEENgtjku7SSZ7UElQ5NZNrldlk8ZYLIVTul+8XuvrQ==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1l5hw95p5h4sthrgn0usms9yfkwwmcvv34tjgrtv9s4e6x39chacshgxavs",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBJQ1ZMRzEreERmM1V3OFhC\ncEhZYjZ2ZGFWc1kveUpLM0MyYlB6S25jR3cwCkhjaTBpS3dHZG51V0RmN1ZVcmRk\nYnIwSlRKVXVoQ1NqdFQ1M2lPazEyRTgKLS0tIGRFTFRaNUMvdU5Fc3A2Nk10NWVw\nWlZOKzJLUDNwTXY4dm9uZTY0cWg4aDgKYDIEQHgomMuJFHHvtt3BbN4tuBiEcboc\nH6K4NmnDE6wMa/1EGGHrjCFb+tUdZSL/zgf5uVOXnXA6d9BEeCGLNQ==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBNk9Od2FL\neEV2Myt0eGI5emF6TlRGS3A3RGNHZ3o3WCtQUDdpYllCTFY1RwpyU2xyNkFIVmhV\nY0g1NDJHRy95SzRrTnRPTGc5ZXdZYTZtMyswWkZIUVdnCi0tLSBWU0Fsak42TUF1\ncndtNFpDUjVhb3RpeWU1eDd0UGtnUTJXeHNiVmNtUTFZCvUQDFKntKn+mZSuDR00\niTu/TdmOu7s89JvirWtFavSZhBzOoW8eXdX/SJCLVy08wdTjb2ksqDdxn0ceiqgL\nLOU=\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2026-06-18T07:11:40Z",
+		"mac": "ENC[AES256_GCM,data:8hd2LItBnmG06HnUQ0avOOnbA4+JAPkJf/Wneqo/YexT/saEK+roa+iMkL2DxtcZK5UahPkJ+wT4q3MfNkOnrdbHmQHUUIkDSX+RxLatMtOseYxg8h9wT6MZehuxkRpN4Y9RlpQu/+l3zKQNbXfsfUj6i91fjH6lcxSYPHNO8ug=,iv:T8hYVO0houthJhFmV2UoOK8d5Z3sevv1pRvhdf5qaXk=,tag:2Q5bi8y+hq1TSkQ45Cmf5Q==,type:str]",
+		"version": "3.12.1"
+	}
+}

vars/per-machine/mx1/dns-acme-rfc2136/rfc2136.env/users/berwn

@@ -0,0 +1 @@
+../../../../../../sops/users/berwn

vars/per-machine/mx1/emergency-access/password-hash/value

@@ -0,0 +1 @@
+$6$E/6oFzd0dMewfnBC$ppC/HUSkmazEB93xxR21v0YtEfCZRxPgqth3d0LgSbXsQ87akNMhWss8pPGI5Ez3QT93btGmGLQNY.kYFy465/

vars/per-machine/mx1/emergency-access/password/groups/admins

@@ -0,0 +1 @@
+../../../../../../sops/groups/admins

vars/per-machine/mx1/emergency-access/password/secret

@@ -0,0 +1,18 @@
+{
+	"data": "ENC[AES256_GCM,data:hHmU59iiqnyvrQ3IIYihxajInbwAVUKT+al9sGubWTM=,iv:3UONUbekFlIwJj8mTR8tsXvlp8ReIPm5eR0djmHCXUs=,tag:wEfguaEgWW9/dYr0TldTJg==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBzUmN6b1d3bWxMV1hsOUJO\nY3ozTmdmOGVsU1ZTU25FellURHFpUDdiYmcwCktEaFVzVVVjdkozR1lvY2FoVVMx\nZ3U1MVFBcDhQUTQ3bWRWVjNyR1l0dU0KLS0tIEZyRkZYRUdYN3lFMFI2SnE0WUx4\na0szbEpGc2FUSFZ6Z1NQK2RKMjV5NlUKJw8BghcP3qRELI9CuY/gq4SvDYBKDPaI\negNQYAQ9sjF7jY0vAyfkYI6KrwKxBa84WZrQYD0Dng1x9YigllSeyw==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBa3JWRUh1\nZnRqMWNrUFJvVmV3Nk1GbmNBdHJKOXVERU5KenBpYTdqRklrNApBT1RGVHJPNmox\nSWxidTI5dS91UnJyM05jVFFMQUlHS1V5bHhKUHRqNkNNCi0tLSBtQ1l2ajhtcTFo\ncmtWb25wQWIrOGtManc2MTM1b1o4ZGV1VE1ONVZPY05jCuxsR2wFaUHnkQxbqrPh\nAmBBWO0xQexWWdJPS1/nz7uS75Ike233UBSLxNapkac5Obfg9UaRYsTPw6qDkQHT\nyUg=\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2026-06-18T04:33:03Z",
+		"mac": "ENC[AES256_GCM,data:sLjSs7asZNK3qJgUOROE/XepuXPqxn2A2idAkLdH95nstbwjQttxjWgSxeLl4V1WbvlUyxBS8O+6JbiPg3dgfFKTEJ2IIQsQS8OBbJV/uUZrFvvAGU3iOB1mMD7Mk/CEz1wuzfVevREQhnbf7YOSAxb6fIK3sY5REU4bNi2qUxI=,iv:PTSvvssFkU8hDMSe2D8EmhwN6KOKIX5SvqjZJJECwXU=,tag:dWyQR50jAxjJA3BrrrL9Ww==,type:str]",
+		"version": "3.12.1"
+	}
+}

vars/per-machine/mx1/emergency-access/password/users/berwn

@@ -0,0 +1 @@
+../../../../../../sops/users/berwn

vars/per-machine/mx1/mail-passwd-postmaster-at-cnx-email/hash/groups/admins

@@ -0,0 +1 @@
+../../../../../../sops/groups/admins

vars/per-machine/mx1/mail-passwd-postmaster-at-cnx-email/hash/machines/mx1

@@ -0,0 +1 @@
+../../../../../../sops/machines/mx1

vars/per-machine/mx1/mail-passwd-postmaster-at-cnx-email/hash/secret

@@ -0,0 +1,22 @@
+{
+	"data": "ENC[AES256_GCM,data:SZIlr7nzcs5EUMVUXQ1KJhrd9JZY7ZR9EUpJ9Eygkpd2sAyMNtV0jMFyfW82PyvLP9/bqXf/5BUR98NuwvMsRmLrb4emEraNJH4qVfS/4s0kXySIJeA6XeMHB7GSuxoh5K/3pUR4tbdFSCM=,iv:grRynVFombEdRp0LfmPHIximhh4rlbQTjqJCjbGhRlw=,tag:qZZCnN63clfKKsb/WQNkbw==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBFWUFGNDBLM3VCMFh2Sjc2\nY1FkOVhta3REK1Y0MUZlY1IxTHorRCtuOWxJClJqK2hjT2p0VFBEeE1xd1JDUTBk\nNzR4eG5zNGJnalVGb2ovRHVqUUpla1EKLS0tIEJMYTU3MlVKOHdvaUhhM2dsWEoy\nRVh6SHc2cVdTTW4xZlhpc3U1aTRLT0UK+WYlVCCJ1bUsuF/vy6+mSU0gpM8FGHDE\n9HfyAHPZLR30ZtkRHSq8LQ137hxmBKSUjv5ztyHc781tLkx9j3lRwQ==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1l5hw95p5h4sthrgn0usms9yfkwwmcvv34tjgrtv9s4e6x39chacshgxavs",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBYMVlBOWVRRitmcVBINGJK\nakxWZEdNeHh5QWIrSHJmVzhDRDl6MUhhdUVnClRFcXZNTGYrazNxWG5UVi8vVWlu\nK3NaN004MVhrR1R2YVpZcnArRmVVUjAKLS0tIG90aXpEVHZLVDMrQWhYMHg2bWZj\nN3B2SmpHTnEzNFZ4cXBmM3paRmhCMjAKZa1OlhBcW+4J0sR+aWv0lkLqDh+73Gay\nKl6ltN1EQI7ISH2azQFahzoz6XV8cyYUHAQaHaYZuyCZNb/XbG9VmQ==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBcHF1OUdP\nQk1VVTdla3ZSc1VUa2MrT0IwanpmRWNsVHlYc21jM3dXNkg4RwpSSENkYUVJa2lS\nTk1pb3NhNkVPU3hPeVVPUE1MOGxTNHpsYXJGSVRQVmVBCi0tLSBFVE96Q0luZ1Jv\nODd2eUl1aWhOR08wUFpPZHFkZEd5MktNU1R2ajBoTERzCrmM40bnvt2iHQERfrN9\n8724ZmXn4YpAiN/FwKpPJ+iF2atpPDbUb2PFG2s6s2kJISMCrpoblZHBTYbG322M\nGgw=\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2026-06-18T07:11:36Z",
+		"mac": "ENC[AES256_GCM,data:bgu8Pv90Ljc1e6uG5oSDY4IwM85ZqiEQ2Uy0xt6Iu6/f8G/K/sh/+N3ZwcXAV3cpAsa5Cde8vHO8JgmAmcYN1frEgb9+rJyfD/sVvIEmy++WgLKIoUngcRyxX7tk+lLGv03tF09HcfoA+P4d5zNxZq5hR20jkT+2t3Y0nw7XN1M=,iv:gbvg2/5SBg3a2H/bvMFuF1bWSVHTppP59s7ivthe6TI=,tag:6zSYNSG2qHKv9YMLVyvSbA==,type:str]",
+		"version": "3.12.1"
+	}
+}

vars/per-machine/mx1/mail-passwd-postmaster-at-cnx-email/hash/users/berwn

@@ -0,0 +1 @@
+../../../../../../sops/users/berwn

vars/per-machine/mx1/mail-passwd-postmaster-at-cnx-email/passphrase/groups/admins

@@ -0,0 +1 @@
+../../../../../../sops/groups/admins

vars/per-machine/mx1/mail-passwd-postmaster-at-cnx-email/passphrase/machines/mx1

@@ -0,0 +1 @@
+../../../../../../sops/machines/mx1

vars/per-machine/mx1/mail-passwd-postmaster-at-cnx-email/passphrase/secret

@@ -0,0 +1,22 @@
+{
+	"data": "ENC[AES256_GCM,data:FYx5Vll+asvRpC56Wk7ZAB6tdGabWIR25fRD/fw5aTgLz3+wU6K3RkDy,iv:rPD4rRduxodp9e6PGSD9V3zDPaTTAxeSNpC3Q/Umi/k=,tag:DiIahCPXn/DOSgJunryXHg==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBrVGRuNnBYL2NJUFFFZWho\nVDM1d1YxUTJLT2lma1NuYWtaNEVpNnUxV0NrClBOb2tBR0Z2a2lMMTNMeStTK2ln\nZjZ1cEFRZjRTTld5M2pNV1c4aHZlZzAKLS0tIHdRNnB2R1E3M1hVamtJWk9wWE1Y\nWG1yKzF5QkFQbWZZUTVObS9jcUtQVmMKdj+SEz7TcCe5Fk5B65EPBvGC0OWVafax\nws4qgi3O4CNQkVoOx4Jq6aWF0I7a0dR0mG2ERPeVUJKt6Kzap/KJ7w==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1l5hw95p5h4sthrgn0usms9yfkwwmcvv34tjgrtv9s4e6x39chacshgxavs",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBFbXhLYUhXSzZZNm40OUdJ\nZ0x3VTB3WDBPSzQ3RU03dS94RmNRMEM1ekg4Cmt0SDMxY1hsL0t6RXRZV29ObWJt\nUFFiZEFFL3p1RytVdXpXeU9sVlU1THMKLS0tIHptRlZqVS9NU3RydGJzbkZ4a05P\nVkRUNDErdklTV2ZWeVNIdVRmVTdxSWsKTH3olIgtm+rM7CsKeVq3GVYk+Y2JcoZ2\nE6/KdwBOsRDFvQpw6vuNVUD0LDDWh0T3+V4+3f9YBn1qdqWHFDA52Q==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBODdOY293\nWXZYWUNuOUxwcUh1Y05VcEFrQTZ3VnJiT3ArN1JFZTZLOFl6RApib2RnN3JnS3ZQ\nbFhFakNiVFdsR2tRa3JUVmJaZzJTY1ZOaUcyQlMrcm5jCi0tLSA1Z04vRTJ4NzVC\ncG5XYW1ZRUd4QkJMTGlJbEUwMmpTSnk4RXBTclRrOGN3Cj7t6HYbS8kdKaIYWMms\n74vQn/HJvnYnIwUqEsf3z8QzTfsXtPB4ueA1NjftyvlKMRozuKxb+ULFv6YkZNzX\nxvo=\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2026-06-18T07:11:36Z",
+		"mac": "ENC[AES256_GCM,data:Jl/TzaI0c0v3JwJHDSvDUZEGKpgMGgFD1UoWC7qbc6LC+vVOpDjcm3WlfXfy2ljHpaqd74dl2kvy2ra75htI5YuLAisTqrPXhm+8km4tLzQzZOHz3JRIW+0fgnVC8z+GFrOHsz8CCc9SCX03HTyOlugvq0PD2sp4hz6Wgmp4cMc=,iv:Q/qUtb968vhyyRQyDYIP3WE49GLWExkmbarSZz9jPhM=,tag:M0crVWRGl2xvrAJEZvfFaA==,type:str]",
+		"version": "3.12.1"
+	}
+}

vars/per-machine/mx1/mail-passwd-postmaster-at-cnx-email/passphrase/users/berwn

@@ -0,0 +1 @@
+../../../../../../sops/users/berwn

vars/per-machine/mx1/openssh/ssh.id_ed25519.pub/value

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJJWavBJ7+x+VjALDd1UzkAnYkNiHgrkIzzvwT6ZQX4k 

vars/per-machine/mx1/openssh/ssh.id_ed25519/groups/admins

@@ -0,0 +1 @@
+../../../../../../sops/groups/admins

vars/per-machine/mx1/openssh/ssh.id_ed25519/machines/mx1

@@ -0,0 +1 @@
+../../../../../../sops/machines/mx1

vars/per-machine/mx1/openssh/ssh.id_ed25519/secret

@@ -0,0 +1,22 @@
+{
+	"data": "ENC[AES256_GCM,data:k00qZlgD7TOR8kD0nDU3phJgJ+5mNZNGuB0J+VMvn2KC639DshChmU2DMn0J7tVZ1pEc9EpsYJEDni1cBWGvP8lKT0fbGT8vHIMS3GIsSSM0MXeGIGe7TUf6iuI8vxomXR/jQQyBq9dv7tHUKVqmfWT2D5SR2pQimOsUxP/qxK1jtjZqBMFFOIkRl9v3Kl36WSXOpHvSWqC0Y9Sntvhu8UaEEjSE0qNRfADAFiJ7yH/gUhgx4SUqyaMwQHgC66+hqVHrGfkhnC0X0Mcw+2oZHDHn4gTit6RCIJyOXKCzd4D9a5ldUF4m9Ws42N9Ulc20ECkFVRrGfo3b6iI7W1EKo1C4tWEya4/+iYj+H051oM1vUPoMQ0wVoRrc9b7iXj8CDqt8jRBbPsCEeYTHLmXfbYuV8s0Q54byNN3gF0oXuP7sutkQl6bEgCkRIQYSqOAeF2ngr+sJHCuTLm5hnvFQVTvTEKIUd3H78+qta+q0J/nWU4Z6yEca+/dPnNOQr/U1aLw/,iv:eAGFaqHaY3Eu/BPGvx76MyJvUuyuC+XMRAjRNItuuu8=,tag:d2MeamgG3W22LK2GQLM2Sw==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBsYmllbWh0NE93TFJ1b0NV\nNSt3a3VkOThFTVFxaWx3VzY1RnBOQndSaENnClg4eXh3cnJvSkhKdGs2dHAxa2tz\nZ1FqSkNyRzF0MUp4bnhqSWVrWUJGaUUKLS0tIFl1RU9qemMzS2ZjM3QrbWJZTFMr\nd1dNSVlLcWVEZ1hRWnQwR1pST0tVWGMKStjFxZgbz4GgVaNHH85O5Gtpgmpju8qU\n6Woe1nTo0LZjoHKCgvXNPetE25iKeG1SoPVV4LbARZ1tXDnnq6fmcA==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1l5hw95p5h4sthrgn0usms9yfkwwmcvv34tjgrtv9s4e6x39chacshgxavs",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAvK08wODlrN21qaFRTMGdz\nYlhrRGc4M3VnWCs1ajMzZkMvektqZVZRL0ZjCjhWb2hLYm5vMjNYeGFUYlBJMG5R\nNURTQnF3aG90aTFaQ210NElkTWlock0KLS0tIHRCM2Y4djZGYjRDOURXazZSN1hu\nb2l5VS9aNmdJQjBjckpseXhoVzc4M2sKqOw6XiDRlFxpDrhCGPxZG9RDKxtLgaJb\nwTdZjB3C1cF4y7ANwmkbegqLaMXfhRI3/QLwkvYR/bZiWsYdOKUnKg==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBNGNoRzhD\nN21ZcktHemNPallPMzlsRFl5bVNPcEJpUmJ3L00yVWJUT044KwpqRjdHbVNyeUEy\nbTRqRDJqZU82WmNQb1FsRjI3MUUxTjd5ekNqU1J1MnFJCi0tLSBBUEJxeXROWERi\nMlpuSDFxcklwZThXUElhWkUyTnUyMC9DWmJIQU1jbGVFCqIAsovpsuzHJJSQEeXB\naq8nha2lVQQk5aaCRmltzoAGaMM9axrh8bgNT2TrSYv4ZAfV09e4Vn9s/dGm3X5O\ni4M=\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2026-06-18T04:33:03Z",
+		"mac": "ENC[AES256_GCM,data:tYw/e79GLeZI34PbiB3AsWVUP28kVad2NqdxvPjNinKF2yRmcML0LNDMSbVwSYUuGJWvyc0mbF4QFThtEiSPVcKqgYXcNm94fmfeouTrZKAvVxjGXE8pa7wDtsoV9yNcpLiQxqtMCZxy7mJPgNhHiMQjiPOqU7tb1O7+o13bCJU=,iv:DgSpKNC49edVc9kWGVY1hZIya40QP/0b52mbx58I0mU=,tag:qB3zc//bUSMbCK5dufPqzg==,type:str]",
+		"version": "3.12.1"
+	}
+}

vars/per-machine/mx1/openssh/ssh.id_ed25519/users/berwn

@@ -0,0 +1 @@
+../../../../../../sops/users/berwn

vars/per-machine/mx1/root-password/password-hash/groups/admins

@@ -0,0 +1 @@
+../../../../../../sops/groups/admins

vars/per-machine/mx1/root-password/password-hash/machines/mx1

@@ -0,0 +1 @@
+../../../../../../sops/machines/mx1

vars/per-machine/mx1/root-password/password-hash/secret

@@ -0,0 +1,22 @@
+{
+	"data": "ENC[AES256_GCM,data:5rAULXYJAzvJISpwR/HmZFRT47W3J312LlQLibsg42gq1Ddfskn+njf3CrHzkYHppRyNuGdo9NoX5Abd8PLrSucTyyNfTmNRZLP4YQ4DG7AIZJrDD9SosltNC0BaE7J25WRna0lNQDWwEA==,iv:mwNjVtr91DAL4B+s7DjZJM3t+5EmnwgaqixASjVstSI=,tag:wWy1wqvZS+zXdcmq4z4tuQ==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1hlzrpqqgndcthq5m5yj9egfgyet2fzrxwa6ynjzwx2r22uy6m3hqr3rd06",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBYcGtPSnpLODF5U0NIQk5H\neERBcm11NkpkanhLd0pKYVllbHV0SndyaVVVCkQ3T3A1OTRXb3Y4V3NpcWorQ1Fa\nenJMU3UxZnYzMmNraXk0YWRVZjF5T3MKLS0tIFZVcGd2d3BwQjVta1RpSzJMTzhQ\nVWpQL2ZkOXFXa0RhejBDZUlMS2VVOFkKfB+wbosTER+rcLGCeAWMcalT9DsvYMQf\n+k4RUA57zSzNFdX/2azltJ3L58IK9UeSxyEgcYobWSnpZmvqr160Jg==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1l5hw95p5h4sthrgn0usms9yfkwwmcvv34tjgrtv9s4e6x39chacshgxavs",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAxOE84VVkzV3EvRFo1M0Fa\ncnhZa0NxajZFdnZFOXI1N05tOHUzVVB3cUIwCktvVXU5NGh3eW9wa2NoRWNxU2lP\nUDdiU3hURGFvSHJ6ZEN5QnlMNzkzNHMKLS0tIHRjdXdkMEhQZmlTN2JOV3R5WllE\nV1FnYmhFcnZ1MU44TzJqb2J3cmRtcjgKts2tmDm1F5AYNn34UUdEw0wyqMB0OTwx\ne+yuiSYM3WogVbve6c/fMR84k5Wm+yp/PN77JigzM7SvRwLNmAJqgw==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1yubikey1qd859y9ehz2ya8j2cftwrtmdeqhuk7r7yc52zp64wpff6068gwrac3q6nsa",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHBpdi1wMjU2IHFWcTVydyBBcFNIdXVL\nYmlXUnV1T3Q3V3pETUVsY29lcy84RU41UmR0OE9DK3RrbStGQgpxbFVpNlQyS0o4\nVXIrcG1XREc2bFVvODZxTmx3ODBNelI3V3ZLUjhmallvCi0tLSA3emNPb1FLdSs4\neXdrSzA4aVB2ZmVoZ1F1MHVqcVFpbjhoVExmbklKUWR3Cu1BH93XQt5sHPuNJPNR\n0pQ4qS/a1iXbz4A9FSpoerc6esLz7s1r4W2i/Vpc13QKhCr+/q5/n7URFJAPAjFn\nXzo=\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2026-06-18T04:33:04Z",
+		"mac": "ENC[AES256_GCM,data:L1KdxE5KVrwbqa6LkpJNicDqOND69rNZKJCwW8+wYKAcxBwFrXF3fs584hFYydLdSlEWDTO2BzVvi9L7BB10gJuRUmw9Pk1rplCe+EkrrTl8ePbfjfGDK5agijsiNYdgRbF6JwHIYOQGVznYKIp28HZnwAi8eXuXzMDo7Fd3nqg=,iv:Xz0c4phNz8a+r/qDO8qf+0IHsTNkV96xBMgH9nh1hAs=,tag:lhxuokbeoPLQrdapMdmneA==,type:str]",
+		"version": "3.12.1"
+	}
+}