From 8090ab3e6dce3b1ef356e7f25df3e09e057453d5 Mon Sep 17 00:00:00 2001
From: kurogeek <kurogeek@lmvhaus.com>
Date: Mon, 21 Sep 2026 03:41:48 +0000
Subject: [PATCH] router: let allowWan VLANs out through the staging uplink

With stagingPort set, the box itself had internet over the staging DHCP
uplink but LAN/Wi-Fi clients had none: forward and masquerade were scoped
to ppp0 only. Worse, pppd's `defaultroute` refuses to install its route
while the staging DHCP default route (metric 1024) exists ("not replacing
existing default route"), so even a live PPPoE session was never used.

- firewall: forward-allow + masquerade allowWan VLANs -> stagingPort in a
  separate `router-staging-nat` postrouting chain (networking.nat only
  takes one external interface). Same allowWan set as nixos-nat.
- pppoe: `defaultroute-metric 0`, so pppd only checks for a metric-0
  default route, installs ppp0 as the preferred exit and removes it on
  hangup, leaving the staging route as the fallback.
---
 modules/clan/router/firewall.nix  | 28 +++++++++++++++++++++++++++-
 modules/clan/router/interface.nix | 16 +++++++++-------
 modules/clan/router/pppoe.nix     |  6 ++++++
 3 files changed, 42 insertions(+), 8 deletions(-)

diff --git a/modules/clan/router/firewall.nix b/modules/clan/router/firewall.nix
index f41b0a8..ba8ff63 100644
--- a/modules/clan/router/firewall.nix
+++ b/modules/clan/router/firewall.nix
@@ -3,7 +3,8 @@
 #   other VLANs -> DNS/DHCP on the router + WAN (if allowWan); no inter-VLAN
 #   WAN (ppp0) -> nothing inbound beyond established/related
 #   mesh       -> admin SSH + metrics scrapes (same trust boundary as the fleet)
-#   staging    -> admin SSH only (pre-cutover uplink into the old LAN)
+#   staging    -> admin SSH only inbound (pre-cutover uplink into the old LAN);
+#                 allowWan VLANs are NATed out through it while ppp0 is down
 { settings }:
 { lib, ... }:
 let
@@ -14,6 +15,13 @@ let
     lib.filterAttrs (_: vlan: vlan.allowWan) cfg.vlans
   );
   nonMgmtIfs = lib.filter (i: i != "vlan-mgmt") vlanIfs;
+
+  # allowWan VLANs may also leave through the staging uplink. Same set as
+  # networking.nat.internalInterfaces below, so `allowWan` holds on both
+  # exits. The kernel picks the exit: ppp0 (metric 0, see pppoe.nix) while
+  # the session is up, the staging DHCP route (metric 1024) otherwise.
+  stagingExit = cfg.stagingPort != null && wanVlanIfs != [ ];
+  wanVlanSet = "{ ${lib.concatMapStringsSep ", " (i: ''"${i}"'') wanVlanIfs} }";
 in
 {
   networking.nftables.enable = true;
@@ -50,6 +58,9 @@ in
     extraForwardRules = ''
       tcp flags syn tcp option maxseg size set rt mtu comment "MSS clamp for PPPoE mtu 1492"
       iifname "vlan-mgmt" accept comment "mgmt reaches all VLANs and the WAN"
+    ''
+    + lib.optionalString stagingExit ''
+      iifname ${wanVlanSet} oifname "${cfg.stagingPort}" accept comment "allowWan VLANs out via the staging uplink"
     '';
   };
 
@@ -62,4 +73,19 @@ in
     externalInterface = "ppp0";
     internalInterfaces = wanVlanIfs;
   };
+
+  # networking.nat only masquerades on its single externalInterface; the
+  # staging uplink needs its own postrouting chain (nixos-nat's is
+  # oifname-scoped to ppp0, so the two never both apply).
+  networking.nftables.tables = lib.optionalAttrs stagingExit {
+    router-staging-nat = {
+      family = "ip";
+      content = ''
+        chain post {
+          type nat hook postrouting priority srcnat;
+          iifname ${wanVlanSet} oifname "${cfg.stagingPort}" masquerade comment "allowWan VLANs out via the staging uplink"
+        }
+      '';
+    };
+  };
 }
diff --git a/modules/clan/router/interface.nix b/modules/clan/router/interface.nix
index be1ce50..94ccfa0 100644
--- a/modules/clan/router/interface.nix
+++ b/modules/clan/router/interface.nix
@@ -256,13 +256,15 @@ in
       example = "enp3s0";
       description = ''
         Temporary DHCPv4-client uplink into the existing LAN while the box
-        runs alongside the router it replaces: gives it internet + mesh
-        before the WAN port is cabled (PPPoE simply retries until then). The
-        port is in no VLAN zone; the firewall admits only SSH on it. Do NOT
-        connect the trunk ports to the production switch while staging —
-        Kea on the mgmt tag would fight the old router's DHCP in one
-        broadcast domain. Set to null at cutover (and usually hand the port
-        back to `trunkPorts`).
+        runs alongside the router it replaces: gives it (and, NATed, the
+        allowWan VLANs) internet + mesh before the WAN port is cabled; once
+        the PPPoE session is up its default route wins, and the staging
+        route only carries traffic again if the session drops (PPPoE simply
+        retries until then). The port is in no VLAN zone; inbound, the
+        firewall admits only SSH on it. Do NOT connect the trunk ports to
+        the production switch while staging — Kea on the mgmt tag would
+        fight the old router's DHCP in one broadcast domain. Set to null at
+        cutover (and usually hand the port back to `trunkPorts`).
       '';
     };
 
diff --git a/modules/clan/router/pppoe.nix b/modules/clan/router/pppoe.nix
index 9848a0f..7859ac9 100644
--- a/modules/clan/router/pppoe.nix
+++ b/modules/clan/router/pppoe.nix
@@ -30,6 +30,11 @@ in
     '';
   };
 
+  # defaultroute-metric 0: pppd refuses `defaultroute` while any other
+  # default route exists (e.g. the staging uplink's DHCP route, metric 1024,
+  # network.nix) unless given a metric; with 0 it only checks for a metric-0
+  # route, installs its own as the preferred exit, and removes it again on
+  # hangup so the staging route takes over.
   services.pppd = {
     enable = true;
     peers.wan = {
@@ -40,6 +45,7 @@ in
         file ${creds.files."user-opts".path}
         noipdefault
         defaultroute
+        defaultroute-metric 0
         noauth
         hide-password
         persist