router/tests: cover the staging uplink as NATed fallback exit

Neither regression fixed by the previous commit was observable: the VM
test had no staging port, and pinging the ISP's PPPoE address only needs
ppp0's connected route, not the default route pppd refused to install.

Add a fifth node, `oldlan`: a networkd DHCP server on vlan 4 handing gw
its staging lease, with a second address (203.0.113.1) that gw can only
reach through that lease's default route. It has no route back to the
VLANs, so client pings only work if gw masquerades.

Proven: pppd's metric-0 default wins over the DHCP one while ppp0 is up;
lan reaches the old LAN via the staging port and iot (allowWan = false)
does not; after stopping pppd the staging route carries WAN traffic with
the same allowWan split; restarting pppd makes ppp0 preferred again.

modules/clan/router/dns-dhcp.nix

@@ -12,7 +12,19 @@ in
   services.kea.dhcp4 = {
     enable = true;
     settings = {
-      interfaces-config.interfaces = lib.mapAttrsToList (name: _: "vlan-${name}") dhcpVlans;
+      interfaces-config = {
+        interfaces = lib.mapAttrsToList (name: _: "vlan-${name}") dhcpVlans;
+        # The unit orders after network-online.target, which under networkd
+        # only waits for the WAN carrier (the vlan-* links are
+        # RequiredForOnline=no), so Kea can start before vlan-* have their
+        # addresses. By default it then logs the failed bind and runs with no
+        # socket at all: clients' DISCOVERs reach vlan-lan and nobody answers.
+        # Insist on every socket and keep retrying while networkd catches up;
+        # if it still cannot bind, exit and let systemd restart the unit.
+        service-sockets-require-all = true;
+        service-sockets-max-retries = 60;
+        service-sockets-retry-wait-time = 1000;
+      };
       lease-database = {
         type = "memfile";
         persist = true;
@@ -44,6 +56,10 @@ in
     };
   };
 
+  # The nixpkgs unit already has Restart=on-failure; space the restarts out so
+  # a persistent bind failure does not trip the start-rate limit.
+  systemd.services.kea-dhcp4-server.serviceConfig.RestartSec = 5;
+
   services.blocky = {
     enable = true;
     settings = {

modules/clan/router/firewall.nix

@@ -3,7 +3,8 @@
 #   other VLANs -> DNS/DHCP on the router + WAN (if allowWan); no inter-VLAN
 #   WAN (ppp0) -> nothing inbound beyond established/related
 #   mesh       -> admin SSH + metrics scrapes (same trust boundary as the fleet)
-#   staging    -> admin SSH only (pre-cutover uplink into the old LAN)
+#   staging    -> admin SSH only inbound (pre-cutover uplink into the old LAN);
+#                 allowWan VLANs are NATed out through it while ppp0 is down
 { settings }:
 { lib, ... }:
 let
@@ -14,6 +15,13 @@ let
     lib.filterAttrs (_: vlan: vlan.allowWan) cfg.vlans
   );
   nonMgmtIfs = lib.filter (i: i != "vlan-mgmt") vlanIfs;
+
+  # allowWan VLANs may also leave through the staging uplink. Same set as
+  # networking.nat.internalInterfaces below, so `allowWan` holds on both
+  # exits. The kernel picks the exit: ppp0 (metric 0, see pppoe.nix) while
+  # the session is up, the staging DHCP route (metric 1024) otherwise.
+  stagingExit = cfg.stagingPort != null && wanVlanIfs != [ ];
+  wanVlanSet = "{ ${lib.concatMapStringsSep ", " (i: ''"${i}"'') wanVlanIfs} }";
 in
 {
   networking.nftables.enable = true;
@@ -50,6 +58,9 @@ in
     extraForwardRules = ''
       tcp flags syn tcp option maxseg size set rt mtu comment "MSS clamp for PPPoE mtu 1492"
       iifname "vlan-mgmt" accept comment "mgmt reaches all VLANs and the WAN"
+    ''
+    + lib.optionalString stagingExit ''
+      iifname ${wanVlanSet} oifname "${cfg.stagingPort}" accept comment "allowWan VLANs out via the staging uplink"
     '';
   };
 
@@ -62,4 +73,19 @@ in
     externalInterface = "ppp0";
     internalInterfaces = wanVlanIfs;
   };
+
+  # networking.nat only masquerades on its single externalInterface; the
+  # staging uplink needs its own postrouting chain (nixos-nat's is
+  # oifname-scoped to ppp0, so the two never both apply).
+  networking.nftables.tables = lib.optionalAttrs stagingExit {
+    router-staging-nat = {
+      family = "ip";
+      content = ''
+        chain post {
+          type nat hook postrouting priority srcnat;
+          iifname ${wanVlanSet} oifname "${cfg.stagingPort}" masquerade comment "allowWan VLANs out via the staging uplink"
+        }
+      '';
+    };
+  };
 }

modules/clan/router/interface.nix

@@ -256,13 +256,15 @@ in
       example = "enp3s0";
       description = ''
         Temporary DHCPv4-client uplink into the existing LAN while the box
-        runs alongside the router it replaces: gives it internet + mesh
+        runs alongside the router it replaces: gives it (and, NATed, the
-        before the WAN port is cabled (PPPoE simply retries until then). The
+        allowWan VLANs) internet + mesh before the WAN port is cabled; once
-        port is in no VLAN zone; the firewall admits only SSH on it. Do NOT
+        the PPPoE session is up its default route wins, and the staging
-        connect the trunk ports to the production switch while staging —
+        route only carries traffic again if the session drops (PPPoE simply
-        Kea on the mgmt tag would fight the old router's DHCP in one
+        retries until then). The port is in no VLAN zone; inbound, the
-        broadcast domain. Set to null at cutover (and usually hand the port
+        firewall admits only SSH on it. Do NOT connect the trunk ports to
-        back to `trunkPorts`).
+        the production switch while staging — Kea on the mgmt tag would
+        fight the old router's DHCP in one broadcast domain. Set to null at
+        cutover (and usually hand the port back to `trunkPorts`).
       '';
     };
 

modules/clan/router/pppoe.nix

@@ -30,6 +30,11 @@ in
     '';
   };
 
+  # defaultroute-metric 0: pppd refuses `defaultroute` while any other
+  # default route exists (e.g. the staging uplink's DHCP route, metric 1024,
+  # network.nix) unless given a metric; with 0 it only checks for a metric-0
+  # route, installs its own as the preferred exit, and removes it again on
+  # hangup so the staging route takes over.
   services.pppd = {
     enable = true;
     peers.wan = {
@@ -40,6 +45,7 @@ in
         file ${creds.files."user-opts".path}
         noipdefault
         defaultroute
+        defaultroute-metric 0
         noauth
         hide-password
         persist

modules/clan/router/tests/vm/default.nix

@@ -1,14 +1,17 @@
 # End-to-end VM test of the router service: a PPPoE access concentrator plays
 # the ISP on the WAN port, a trunk carries tagged lan/iot VLANs to `client`,
-# and an untagged access port carries mgmt to `admin`.
+# an untagged access port carries mgmt to `admin`, and `oldlan` is the DHCP
+# network the box is staged in before cutover.
 #
 #   isp ---(vlan 1: PPPoE)--- wan [gw] trunk ---(vlan 2: tagged 20/40)--- client
 #                                    access --(vlan 3: untagged mgmt)--- admin
+#                                    staging -(vlan 4: DHCP client)--- oldlan
 #
 # What is proven: PPPoE dial-in with the vars-provided credentials, bridge
 # VLAN tagging/untagging, Kea leases and reservations per VLAN, Blocky
-# answering on the VLAN with the blocklist active, NAT to the WAN, and the
+# answering on the VLAN with the blocklist active, NAT to the WAN, the
-# firewall trust model (allowWan, mgmt-only SSH, no inter-VLAN forwarding).
+# firewall trust model (allowWan, mgmt-only SSH, no inter-VLAN forwarding),
+# and the staging uplink as NATed fallback exit behind ppp0.
 { pkgs, lib, ... }:
 let
   # The vars mock answers every prompt with "mock-prompt-value-<name>"; the
@@ -20,6 +23,10 @@ let
   clientAddress = "10.9.20.50";
   adminMac = "02:00:00:00:00:10";
   adminAddress = "10.9.10.50";
+  oldlanAddress = "192.168.88.1";
+  # Only reachable through oldlan's router role, i.e. via gw's staging
+  # default route (metric 1024); ppp0's metric-0 default must win while up.
+  beyondStaging = "203.0.113.1";
 in
 {
   name = "router";
@@ -36,6 +43,7 @@ in
         isp = { };
         client = { };
         admin = { };
+        oldlan = { };
       };
 
       instances.router = {
@@ -48,6 +56,7 @@ in
           wan.interface = "wan";
           trunkPorts = [ "trunk" ];
           accessPorts.access = "mgmt";
+          stagingPort = "staging";
           vlans = {
             mgmt = {
               id = 10;
@@ -111,6 +120,10 @@ in
           vlan = 3;
           assignIP = false;
         };
+        staging = {
+          vlan = 4;
+          assignIP = false;
+        };
       };
 
       # Something must listen on 22 for the mgmt-only SSH rule to be observable
@@ -251,6 +264,32 @@ in
       };
       environment.systemPackages = [ pkgs.netcat ];
     };
+
+    # The LAN the box is staged in: a DHCP server handing gw its uplink
+    # lease, plus an address that is only reachable via that uplink's
+    # default route. No route back to 10.9.0.0/16: replies only reach the
+    # clients if gw masquerades them.
+    oldlan = {
+      virtualisation.interfaces.staging = {
+        vlan = 4;
+        assignIP = false;
+      };
+      networking.useDHCP = false;
+      networking.useNetworkd = true;
+      systemd.network.networks."10-staging" = {
+        matchConfig.Name = "staging";
+        address = [
+          "${oldlanAddress}/24"
+          "${beyondStaging}/32"
+        ];
+        networkConfig.DHCPServer = true;
+        dhcpServerConfig = {
+          PoolOffset = 100;
+          PoolSize = 50;
+        };
+      };
+      networking.firewall.allowedUDPPorts = [ 67 ];
+    };
   };
 
   testScript = ''
@@ -302,5 +341,21 @@ in
         gw.wait_until_succeeds("ip netns exec sta wpa_cli -i wlan1 status | grep -q wpa_state=COMPLETED")
         gw.succeed("timeout 60 sta-dhcp")
         gw.succeed("ip netns exec sta ip -4 addr show wlan1 | grep -q 'inet 10.9.20.1[0-9][0-9]/24'")
+
+    with subtest("Staging uplink: NATed exit for allowWan VLANs, behind ppp0 while it is up"):
+        gw.wait_until_succeeds("ip -4 route show default dev staging | grep -q 'via ${oldlanAddress}'")
+        # pppd installs its default route despite the DHCP one (defaultroute-metric 0).
+        gw.succeed("ip route get ${beyondStaging} | grep -q 'dev ppp0'")
+        # On-link old-LAN hosts are reached through the staging port regardless.
+        client.succeed("ping -c1 -W2 -I lan0 ${oldlanAddress}")
+        client.fail("ping -c1 -W2 -I iot0 ${oldlanAddress}")
+        # ppp0 down: the staging route carries the WAN traffic, allowWan still holds.
+        gw.systemctl("stop pppd-wan.service")
+        gw.wait_until_succeeds("ip route get ${beyondStaging} | grep -q 'dev staging'")
+        client.succeed("ping -c1 -W2 -I lan0 ${beyondStaging}")
+        client.fail("ping -c1 -W2 -I iot0 ${beyondStaging}")
+        # ppp0 back: preferred again.
+        gw.systemctl("start pppd-wan.service")
+        gw.wait_until_succeeds("ip route get ${beyondStaging} | grep -q 'dev ppp0'")
   '';
 }