Compare commits
2
Commits
c0c2193429
...
4c589b8d0c
| Author | SHA1 | Date | |
|---|---|---|---|
 kurogeek
|
4c589b8d0c | ||
|
kurogeek
|
c9b04711c9 |
+16
-16
@@ -20,22 +20,22 @@ mgmt-only trust model, SSH exposure and the Wi-Fi bridge ports. Run it with
|
||||
|
||||
## What each gateway runs
|
||||
|
||||
| Function | Implementation |
|
||||
| ------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| WAN | PPPoE (`pppd`), per-site ISP credentials via clan vars prompts; `wan.vlanId` when the ISP tags the session (AIS: 10); `wan.macAddress` to clone the old router's MAC if the ISP has it pinned |
|
||||
| LAN | VLAN-filtering bridge `br0` over the trunk ports (networkd); `accessPorts` pin a port untagged to one VLAN — convention: the last copper port is an untagged `mgmt` recovery port |
|
||||
| Firewall/NAT | nftables: default-deny WAN, no inter-VLAN, MSS clamp, v4 NAT |
|
||||
| DHCP | Kea, one subnet per VLAN |
|
||||
| DNS | Blocky (blocklist resolver), metrics on :4000 scraped by control |
|
||||
| IPv6 | DHCPv6-PD on ppp0, /64 per VLAN via SLAAC |
|
||||
| Bans | CrowdSec + nftables bouncer (sshd log parsing) |
|
||||
| Omada | Optional per site: TP-Link Omada controller as a podman container |
|
||||
| Wi-Fi | Optional: hostapd on the router's radios; each SSID (`wifi.networks`) is an untagged access port of its VLAN, passphrases via vars prompts — see `modules/clan/router/README.md` |
|
||||
| Proxy | Optional: Caddy reverse proxy for internal services under `*.<site><n>.cnx.network` with a real Let's Encrypt wildcard (DNS-01 against ns1) |
|
||||
| Diagnostics | iperf3 server on 5201, reachable from every VLAN and the mesh (`iperf3 -c <router-vlan-addr>`); CLI toolkit: tcpdump, mtr, ethtool, conntrack, kdig, iftop, librespeed-cli |
|
||||
| Speed test | Hourly librespeed run (`speedtest.timer`) → `speedtest_*` metrics via node_exporter; vmalert flags download < 50% of the link's own 7-day median (`WANSpeedDegraded`) |
|
||||
| Management | ZeroTier mesh: SSH, node_exporter, journald upload — like the fleet |
|
||||
| Storage | Single-disk ZFS (zstd, `/var` its own dataset for snapshots) |
|
||||
| Function | Implementation |
|
||||
| ------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| WAN | PPPoE (`pppd`), per-site ISP credentials via clan vars prompts; `wan.vlanId` when the ISP tags the session (AIS: 10); `wan.macAddress` to clone the old router's MAC if the ISP has it pinned |
|
||||
| LAN | VLAN-filtering bridge `br0` over the trunk ports (networkd); `accessPorts` pin a port untagged to one VLAN — convention: the last copper port is an untagged `mgmt` recovery port; `stagingPort` turns a spare port into a DHCP uplink into the old LAN while the box runs alongside the router it replaces (drop at cutover) |
|
||||
| Firewall/NAT | nftables: default-deny WAN, no inter-VLAN, MSS clamp, v4 NAT |
|
||||
| DHCP | Kea, one subnet per VLAN |
|
||||
| DNS | Blocky (blocklist resolver), metrics on :4000 scraped by control |
|
||||
| IPv6 | DHCPv6-PD on ppp0, /64 per VLAN via SLAAC |
|
||||
| Bans | CrowdSec + nftables bouncer (sshd log parsing) |
|
||||
| Omada | Optional per site: TP-Link Omada controller as a podman container |
|
||||
| Wi-Fi | Optional: hostapd on the router's radios; each SSID (`wifi.networks`) is an untagged access port of its VLAN, passphrases via vars prompts — see `modules/clan/router/README.md` |
|
||||
| Proxy | Optional: Caddy reverse proxy for internal services under `*.<site><n>.cnx.network` with a real Let's Encrypt wildcard (DNS-01 against ns1) |
|
||||
| Diagnostics | iperf3 server on 5201, reachable from every VLAN and the mesh (`iperf3 -c <router-vlan-addr>`); CLI toolkit: tcpdump, mtr, ethtool, conntrack, kdig, iftop, librespeed-cli |
|
||||
| Speed test | Hourly librespeed run (`speedtest.timer`) → `speedtest_*` metrics via node_exporter; vmalert flags download < 50% of the link's own 7-day median (`WANSpeedDegraded`) |
|
||||
| Management | ZeroTier mesh: SSH, node_exporter, journald upload — like the fleet |
|
||||
| Storage | Single-disk ZFS (zstd, `/var` its own dataset for snapshots) |
|
||||
|
||||
## Addressing plan
|
||||
|
||||
|
||||
+5
-4
@@ -40,10 +40,11 @@ in
|
||||
wan.interface = "enp1s0";
|
||||
wan.vlanId = null; # this ISP runs PPPoE untagged on the port
|
||||
wan.macAddress = "a8:b8:e0:01:06:87";
|
||||
trunkPorts = [
|
||||
"enp2s0"
|
||||
# "enp3s0" # STAGING: serves as the uplink until cutover (see the machine config)
|
||||
];
|
||||
trunkPorts = [ "enp2s0" ];
|
||||
# STAGING (remove at cutover, move enp3s0 back into trunkPorts):
|
||||
# DHCP-client uplink into the existing OPNsense LAN so the box has
|
||||
# internet + mesh while it runs alongside the old router.
|
||||
stagingPort = "enp3s0";
|
||||
# Dedicated on-site recovery port: untagged mgmt, always available even
|
||||
# if the switch config is broken.
|
||||
accessPorts.enp4s0 = "mgmt";
|
||||
|
||||
@@ -16,18 +16,6 @@
|
||||
builtins.hashString "sha256" config.networking.hostName
|
||||
);
|
||||
|
||||
# STAGING (remove at cutover, and restore enp3s0 to trunkPorts in clan.nix):
|
||||
# DHCP-client uplink into the existing OPNsense LAN so the box has internet +
|
||||
# mesh while it runs alongside the old router. Default-deny firewall on this
|
||||
# interface (it's in no VLAN zone); PPPoE simply retries until the WAN port
|
||||
# is cabled. Do NOT connect the trunk ports to the production switch while
|
||||
# staging — Kea on tag 10 would fight the OPNsense LAN DHCP in one broadcast
|
||||
# domain.
|
||||
systemd.network.networks."05-staging" = {
|
||||
matchConfig.Name = "enp3s0";
|
||||
networkConfig.DHCP = "ipv4";
|
||||
};
|
||||
|
||||
time.timeZone = "Etc/GMT-7"; # UTC+7 (Thailand, fixed offset, no DST)
|
||||
services.chrony.enable = true;
|
||||
}
|
||||
|
||||
@@ -32,6 +32,7 @@ inventory.instances.router = {
|
||||
wan.vlanId = 10; # or null for untagged PPPoE
|
||||
trunkPorts = [ "enp2s0" ];
|
||||
accessPorts.enp4s0 = "mgmt"; # untagged on-site recovery port
|
||||
# stagingPort = "enp3s0"; # DHCP uplink into the old LAN until cutover
|
||||
vlans = {
|
||||
mgmt.id = 10;
|
||||
lan.id = 20;
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
# other VLANs -> DNS/DHCP on the router + WAN (if allowWan); no inter-VLAN
|
||||
# WAN (ppp0) -> nothing inbound beyond established/related
|
||||
# mesh -> admin SSH + metrics scrapes (same trust boundary as the fleet)
|
||||
# staging -> admin SSH only (pre-cutover uplink into the old LAN)
|
||||
{ settings }:
|
||||
{ lib, ... }:
|
||||
let
|
||||
@@ -26,14 +27,20 @@ in
|
||||
filterForward = true;
|
||||
trustedInterfaces = [ "vlan-mgmt" ];
|
||||
|
||||
# Non-mgmt VLANs may only talk to the router's DNS and DHCP.
|
||||
interfaces = lib.genAttrs nonMgmtIfs (_: {
|
||||
allowedTCPPorts = [ 53 ];
|
||||
allowedUDPPorts = [
|
||||
53
|
||||
67
|
||||
];
|
||||
});
|
||||
# Non-mgmt VLANs may only talk to the router's DNS and DHCP; the staging
|
||||
# uplink (old LAN, pre-cutover) gets admin SSH so the box can be reached
|
||||
# before the mgmt VLAN or the mesh are up.
|
||||
interfaces =
|
||||
lib.genAttrs nonMgmtIfs (_: {
|
||||
allowedTCPPorts = [ 53 ];
|
||||
allowedUDPPorts = [
|
||||
53
|
||||
67
|
||||
];
|
||||
})
|
||||
// lib.optionalAttrs (cfg.stagingPort != null) {
|
||||
${cfg.stagingPort}.allowedTCPPorts = [ 22 ];
|
||||
};
|
||||
|
||||
extraInputRules = ''
|
||||
ip6 saddr ${cfg.mesh.subnet} tcp dport 22 accept comment "admin ssh over the mesh"
|
||||
|
||||
@@ -250,6 +250,22 @@ in
|
||||
'';
|
||||
};
|
||||
|
||||
stagingPort = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
example = "enp3s0";
|
||||
description = ''
|
||||
Temporary DHCPv4-client uplink into the existing LAN while the box
|
||||
runs alongside the router it replaces: gives it internet + mesh
|
||||
before the WAN port is cabled (PPPoE simply retries until then). The
|
||||
port is in no VLAN zone; the firewall admits only SSH on it. Do NOT
|
||||
connect the trunk ports to the production switch while staging —
|
||||
Kea on the mgmt tag would fight the old router's DHCP in one
|
||||
broadcast domain. Set to null at cutover (and usually hand the port
|
||||
back to `trunkPorts`).
|
||||
'';
|
||||
};
|
||||
|
||||
vlans = lib.mkOption {
|
||||
type = lib.types.attrsOf (lib.types.submodule vlanModule);
|
||||
description = "VLANs served at this site; `mgmt` and `lan` are mandatory.";
|
||||
|
||||
@@ -21,6 +21,16 @@ in
|
||||
assertion = lib.all (p: !(cfg.accessPorts ? ${p})) cfg.trunkPorts;
|
||||
message = "router: a port cannot be both a trunk and an access port.";
|
||||
}
|
||||
{
|
||||
assertion =
|
||||
cfg.stagingPort == null
|
||||
|| !(
|
||||
cfg.stagingPort == cfg.wan.interface
|
||||
|| lib.elem cfg.stagingPort cfg.trunkPorts
|
||||
|| cfg.accessPorts ? ${cfg.stagingPort}
|
||||
);
|
||||
message = "router: stagingPort ${toString cfg.stagingPort} is also the WAN, a trunk or an access port.";
|
||||
}
|
||||
];
|
||||
|
||||
# Router diagnostics toolkit: packets (tcpdump), path (mtr), link
|
||||
@@ -73,7 +83,15 @@ in
|
||||
let
|
||||
taggedAll = lib.mapAttrsToList (_: vlan: { VLAN = vlan.id; }) cfg.vlans;
|
||||
in
|
||||
{
|
||||
lib.optionalAttrs (cfg.stagingPort != null) {
|
||||
# Staging uplink (see interface.nix): plain DHCPv4 client on a spare
|
||||
# port, no bridge/VLAN membership, so the firewall treats it as untrusted.
|
||||
"05-staging" = {
|
||||
matchConfig.Name = cfg.stagingPort;
|
||||
networkConfig.DHCP = "ipv4";
|
||||
};
|
||||
}
|
||||
// {
|
||||
# WAN port carries only the PPPoE session; no IP config of its own.
|
||||
"10-wan" = {
|
||||
matchConfig.Name = cfg.wan.interface;
|
||||
|
||||
Reference in New Issue
Block a user