diff --git a/docs/src/gateways.md b/docs/src/gateways.md index 6bfd885..d3c2e86 100644 --- a/docs/src/gateways.md +++ b/docs/src/gateways.md @@ -20,22 +20,22 @@ mgmt-only trust model, SSH exposure and the Wi-Fi bridge ports. Run it with ## What each gateway runs -| Function | Implementation | -| ------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| WAN | PPPoE (`pppd`), per-site ISP credentials via clan vars prompts; `wan.vlanId` when the ISP tags the session (AIS: 10); `wan.macAddress` to clone the old router's MAC if the ISP has it pinned | -| LAN | VLAN-filtering bridge `br0` over the trunk ports (networkd); `accessPorts` pin a port untagged to one VLAN — convention: the last copper port is an untagged `mgmt` recovery port | -| Firewall/NAT | nftables: default-deny WAN, no inter-VLAN, MSS clamp, v4 NAT | -| DHCP | Kea, one subnet per VLAN | -| DNS | Blocky (blocklist resolver), metrics on :4000 scraped by control | -| IPv6 | DHCPv6-PD on ppp0, /64 per VLAN via SLAAC | -| Bans | CrowdSec + nftables bouncer (sshd log parsing) | -| Omada | Optional per site: TP-Link Omada controller as a podman container | -| Wi-Fi | Optional: hostapd on the router's radios; each SSID (`wifi.networks`) is an untagged access port of its VLAN, passphrases via vars prompts — see `modules/clan/router/README.md` | -| Proxy | Optional: Caddy reverse proxy for internal services under `*..cnx.network` with a real Let's Encrypt wildcard (DNS-01 against ns1) | -| Diagnostics | iperf3 server on 5201, reachable from every VLAN and the mesh (`iperf3 -c `); CLI toolkit: tcpdump, mtr, ethtool, conntrack, kdig, iftop, librespeed-cli | -| Speed test | Hourly librespeed run (`speedtest.timer`) → `speedtest_*` metrics via node_exporter; vmalert flags download < 50% of the link's own 7-day median (`WANSpeedDegraded`) | -| Management | ZeroTier mesh: SSH, node_exporter, journald upload — like the fleet | -| Storage | Single-disk ZFS (zstd, `/var` its own dataset for snapshots) | +| Function | Implementation | +| ------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| WAN | PPPoE (`pppd`), per-site ISP credentials via clan vars prompts; `wan.vlanId` when the ISP tags the session (AIS: 10); `wan.macAddress` to clone the old router's MAC if the ISP has it pinned | +| LAN | VLAN-filtering bridge `br0` over the trunk ports (networkd); `accessPorts` pin a port untagged to one VLAN — convention: the last copper port is an untagged `mgmt` recovery port; `stagingPort` turns a spare port into a DHCP uplink into the old LAN while the box runs alongside the router it replaces (drop at cutover) | +| Firewall/NAT | nftables: default-deny WAN, no inter-VLAN, MSS clamp, v4 NAT | +| DHCP | Kea, one subnet per VLAN | +| DNS | Blocky (blocklist resolver), metrics on :4000 scraped by control | +| IPv6 | DHCPv6-PD on ppp0, /64 per VLAN via SLAAC | +| Bans | CrowdSec + nftables bouncer (sshd log parsing) | +| Omada | Optional per site: TP-Link Omada controller as a podman container | +| Wi-Fi | Optional: hostapd on the router's radios; each SSID (`wifi.networks`) is an untagged access port of its VLAN, passphrases via vars prompts — see `modules/clan/router/README.md` | +| Proxy | Optional: Caddy reverse proxy for internal services under `*..cnx.network` with a real Let's Encrypt wildcard (DNS-01 against ns1) | +| Diagnostics | iperf3 server on 5201, reachable from every VLAN and the mesh (`iperf3 -c `); CLI toolkit: tcpdump, mtr, ethtool, conntrack, kdig, iftop, librespeed-cli | +| Speed test | Hourly librespeed run (`speedtest.timer`) → `speedtest_*` metrics via node_exporter; vmalert flags download < 50% of the link's own 7-day median (`WANSpeedDegraded`) | +| Management | ZeroTier mesh: SSH, node_exporter, journald upload — like the fleet | +| Storage | Single-disk ZFS (zstd, `/var` its own dataset for snapshots) | ## Addressing plan diff --git a/inventory.nix b/inventory.nix index 90d44cb..2899d27 100644 --- a/inventory.nix +++ b/inventory.nix @@ -40,10 +40,11 @@ in wan.interface = "enp1s0"; wan.vlanId = null; # this ISP runs PPPoE untagged on the port wan.macAddress = "a8:b8:e0:01:06:87"; - trunkPorts = [ - "enp2s0" - # "enp3s0" # STAGING: serves as the uplink until cutover (see the machine config) - ]; + trunkPorts = [ "enp2s0" ]; + # STAGING (remove at cutover, move enp3s0 back into trunkPorts): + # DHCP-client uplink into the existing OPNsense LAN so the box has + # internet + mesh while it runs alongside the old router. + stagingPort = "enp3s0"; # Dedicated on-site recovery port: untagged mgmt, always available even # if the switch config is broken. accessPorts.enp4s0 = "mgmt"; diff --git a/machines/gw-cnx-1/configuration.nix b/machines/gw-cnx-1/configuration.nix index 0e3faee..55f4af5 100644 --- a/machines/gw-cnx-1/configuration.nix +++ b/machines/gw-cnx-1/configuration.nix @@ -16,18 +16,6 @@ builtins.hashString "sha256" config.networking.hostName ); - # STAGING (remove at cutover, and restore enp3s0 to trunkPorts in clan.nix): - # DHCP-client uplink into the existing OPNsense LAN so the box has internet + - # mesh while it runs alongside the old router. Default-deny firewall on this - # interface (it's in no VLAN zone); PPPoE simply retries until the WAN port - # is cabled. Do NOT connect the trunk ports to the production switch while - # staging — Kea on tag 10 would fight the OPNsense LAN DHCP in one broadcast - # domain. - systemd.network.networks."05-staging" = { - matchConfig.Name = "enp3s0"; - networkConfig.DHCP = "ipv4"; - }; - time.timeZone = "Etc/GMT-7"; # UTC+7 (Thailand, fixed offset, no DST) services.chrony.enable = true; } diff --git a/modules/clan/router/README.md b/modules/clan/router/README.md index b130c71..d47c14d 100644 --- a/modules/clan/router/README.md +++ b/modules/clan/router/README.md @@ -32,6 +32,7 @@ inventory.instances.router = { wan.vlanId = 10; # or null for untagged PPPoE trunkPorts = [ "enp2s0" ]; accessPorts.enp4s0 = "mgmt"; # untagged on-site recovery port + # stagingPort = "enp3s0"; # DHCP uplink into the old LAN until cutover vlans = { mgmt.id = 10; lan.id = 20; diff --git a/modules/clan/router/interface.nix b/modules/clan/router/interface.nix index ba8a217..be1ce50 100644 --- a/modules/clan/router/interface.nix +++ b/modules/clan/router/interface.nix @@ -250,6 +250,22 @@ in ''; }; + stagingPort = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "enp3s0"; + description = '' + Temporary DHCPv4-client uplink into the existing LAN while the box + runs alongside the router it replaces: gives it internet + mesh + before the WAN port is cabled (PPPoE simply retries until then). The + port is in no VLAN zone; the firewall admits only SSH on it. Do NOT + connect the trunk ports to the production switch while staging — + Kea on the mgmt tag would fight the old router's DHCP in one + broadcast domain. Set to null at cutover (and usually hand the port + back to `trunkPorts`). + ''; + }; + vlans = lib.mkOption { type = lib.types.attrsOf (lib.types.submodule vlanModule); description = "VLANs served at this site; `mgmt` and `lan` are mandatory."; diff --git a/modules/clan/router/network.nix b/modules/clan/router/network.nix index 4202530..dd43e06 100644 --- a/modules/clan/router/network.nix +++ b/modules/clan/router/network.nix @@ -21,6 +21,16 @@ in assertion = lib.all (p: !(cfg.accessPorts ? ${p})) cfg.trunkPorts; message = "router: a port cannot be both a trunk and an access port."; } + { + assertion = + cfg.stagingPort == null + || !( + cfg.stagingPort == cfg.wan.interface + || lib.elem cfg.stagingPort cfg.trunkPorts + || cfg.accessPorts ? ${cfg.stagingPort} + ); + message = "router: stagingPort ${toString cfg.stagingPort} is also the WAN, a trunk or an access port."; + } ]; # Router diagnostics toolkit: packets (tcpdump), path (mtr), link @@ -73,7 +83,15 @@ in let taggedAll = lib.mapAttrsToList (_: vlan: { VLAN = vlan.id; }) cfg.vlans; in - { + lib.optionalAttrs (cfg.stagingPort != null) { + # Staging uplink (see interface.nix): plain DHCPv4 client on a spare + # port, no bridge/VLAN membership, so the firewall treats it as untrusted. + "05-staging" = { + matchConfig.Name = cfg.stagingPort; + networkConfig.DHCP = "ipv4"; + }; + } + // { # WAN port carries only the PPPoE session; no IP config of its own. "10-wan" = { matchConfig.Name = cfg.wan.interface;