B4L/cnx-network-clan
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add site gateway role (modules/router) and gw-cnx-1

Browse Source 
Reusable cnx.router.* module for the Topton 1U boxes replacing OPNsense:
PPPoE WAN (optionally VLAN-tagged, AIS: 10, secret credentials incl.
username), VLAN-filtering bridge, nftables NAT/firewall with MSS clamp,
Kea DHCP with per-VLAN lease time, Blocky DNS, DHCPv6-PD, CrowdSec with
the ZeroTier mesh whitelisted, optional Omada controller, ZFS disk.

Fleet baseline rides along: admins sops group is now derived for every
machine in clan.nix (secrets encrypt to it from the first vars generate)
and time sync is chrony everywhere instead of systemd-timesyncd.
This commit is contained in:
Berwn
2026-07-28 17:06:07 +07:00
parent a87b579eb7
commit 158252323f
20 changed files with 857 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/mesh-hosts.nix
+1
View File
@@ -23,6 +23,7 @@ let
"ns2"
"mx1"
"web01"
"gw-cnx-1"
] readIp;
# RFC 4193 prefix of this ZeroTier network: fd + the 8-byte network id + the
modules/monitoring/server.nix
+7
View File
@@ -48,6 +48,13 @@ in
(target "ns2" (v6 mesh.hosts.ns2) 9100)
(target "mx1" (v6 mesh.hosts.mx1) 9100)
(target "web01" (v6 mesh.hosts.web01) 9100)
(target "gw-cnx-1" (v6 mesh.hosts.gw-cnx-1) 9100)
];
}
{
job_name = "blocky";
static_configs = [
(target "gw-cnx-1" (v6 mesh.hosts.gw-cnx-1) 4000)
];
}
{
modules/router/crowdsec.nix
+52
View File
@@ -0,0 +1,52 @@
# CrowdSec security engine + nftables bouncer: parses sshd auth attempts from
# the journal and bans offending source IPs at the firewall. Log-based (no
# inline DPI) so it costs the N300 next to nothing.
{
config,
lib,
...
}:
let
cfg = config.cnx.router;
mesh = import ../mesh-hosts.nix { inherit config lib; };
in
{
config = lib.mkIf cfg.enable {
services.crowdsec = {
enable = true;
autoUpdateService = true;
hub.collections = [
"crowdsecurity/linux"
"crowdsecurity/sshd"
];
localConfig = {
acquisitions = [
{
source = "journalctl";
journalctl_filter = [ "_SYSTEMD_UNIT=sshd.service" ];
labels.type = "syslog";
}
];
# Never ban the ZeroTier mesh — it is the only admin path to these
# boxes (no public SSH), so a false positive would lock us out.
# Parser-stage whitelist: mesh events are dropped before any scenario.
parsers.s02Enrich = [
{
name = "cnx/mesh-whitelist";
description = "Whitelist the ZeroTier management mesh";
whitelist = {
reason = "ZeroTier mesh is the admin path";
cidr = [ mesh.subnet ];
};
}
];
};
};
services.crowdsec-firewall-bouncer = {
enable = true;
registerBouncer.enable = true;
settings.mode = "nftables";
};
};
}
modules/router/default.nix
+253
View File
@@ -0,0 +1,253 @@
# Site gateway (OPNsense replacement) for the Topton 1U boxes: PPPoE WAN,
# VLAN-filtering bridge over the LAN ports, and per-VLAN L3 interfaces.
# Imported by machines/gw-<city>-<n>; everything is driven by cnx.router.*.
#
# Fleet addressing convention: each site owns 10.<siteId>.0.0/16. A VLAN's
# subnet defaults to 10.<siteId>.<vlanId>.0/24 with the router at .1 and the
# DHCP pool at .100-.199. VLANs that need more space (e.g. public-wifi guest)
# override `subnet`/`address`/`dhcp.pool` and take a wider block from the
# upper half (10.<siteId>.128.0/17), e.g. guest -> 10.<siteId>.128.0/22.
# VLAN ids: 10 = mgmt, 20 = lan (mandatory); 30 = guest, 40 = iot (reserved).
{
config,
lib,
...
}:
let
cfg = config.cnx.router;
vlanIf = name: "vlan-${name}";
site = toString cfg.siteId;
vlanModule =
{ name, config, ... }:
let
octet = toString config.id;
in
{
options = {
id = lib.mkOption {
type = lib.types.ints.between 1 4094;
description = "802.1Q VLAN id (fleet convention: 10 mgmt, 20 lan, 30 guest, 40 iot).";
};
address = lib.mkOption {
type = lib.types.str;
default = "10.${site}.${octet}.1";
description = "Router address on this VLAN.";
};
prefixLength = lib.mkOption {
type = lib.types.ints.between 8 30;
default = 24;
};
subnet = lib.mkOption {
type = lib.types.str;
default = "10.${site}.${octet}.0/24";
description = "The VLAN's network in CIDR form (must contain `address`).";
};
dhcp = {
enable = lib.mkOption {
type = lib.types.bool;
default = true;
};
pool = {
from = lib.mkOption {
type = lib.types.str;
default = "10.${site}.${octet}.100";
};
to = lib.mkOption {
type = lib.types.str;
default = "10.${site}.${octet}.199";
};
};
leaseTime = lib.mkOption {
type = lib.types.ints.positive;
default = 86400;
description = ''
Lease validity in seconds. Lower it for high-churn networks,
e.g. public-WiFi guest VLANs (3600-7200), so the pool recycles.
'';
};
};
allowWan = lib.mkOption {
type = lib.types.bool;
default = true;
description = "Whether clients on this VLAN may reach the internet.";
};
};
};
in
{
imports = [
./pppoe.nix
./ipv6.nix
./firewall.nix
./dns-dhcp.nix
./crowdsec.nix
./omada.nix
];
options.cnx.router = {
enable = lib.mkEnableOption "site gateway (router) role";
site = lib.mkOption {
type = lib.types.str;
description = "City code of the site, e.g. \"cnx\".";
};
siteId = lib.mkOption {
type = lib.types.ints.between 1 254;
description = "Site number; drives the 10.<siteId>.<vlan>.0/24 addressing.";
};
wan.interface = lib.mkOption {
type = lib.types.str;
description = "Physical WAN port the PPPoE session runs on.";
};
wan.vlanId = lib.mkOption {
type = lib.types.nullOr (lib.types.ints.between 1 4094);
default = null;
description = ''
802.1Q tag the ISP requires for the PPPoE session (AIS Thailand: 10);
null for untagged PPPoE directly on the port. Unrelated to the LAN
VLANs — this tag exists only on the WAN port.
'';
};
wan.macAddress = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "aa:bb:cc:dd:ee:ff";
description = ''
Spoofed MAC for the WAN port, e.g. to keep the MAC the ISP has
pinned (cloned from the old router). null keeps the hardware MAC.
'';
};
wan.pppInterface = lib.mkOption {
type = lib.types.str;
internal = true;
readOnly = true;
default = if cfg.wan.vlanId == null then cfg.wan.interface else "wan-vlan";
description = "Interface pppd dials on (the WAN port or its ISP VLAN).";
};
trunkPorts = lib.mkOption {
type = lib.types.listOf lib.types.str;
description = "LAN ports carrying all VLANs tagged (incl. any 10G SFP+ ports).";
};
vlans = lib.mkOption {
type = lib.types.attrsOf (lib.types.submodule vlanModule);
description = "VLANs served at this site; `mgmt` and `lan` are mandatory.";
};
};
config = lib.mkIf cfg.enable {
assertions = [
{
assertion = cfg.vlans ? mgmt && cfg.vlans ? lan;
message = "cnx.router: every site must define the `mgmt` and `lan` VLANs.";
}
];
networking.useNetworkd = true;
networking.useDHCP = false;
systemd.network.enable = true;
systemd.network.netdevs = {
"20-br0" = {
netdevConfig = {
Name = "br0";
Kind = "bridge";
};
bridgeConfig.VLANFiltering = true;
};
}
// lib.optionalAttrs (cfg.wan.vlanId != null) {
"15-wan-vlan" = {
netdevConfig = {
Name = "wan-vlan";
Kind = "vlan";
};
vlanConfig.Id = cfg.wan.vlanId;
};
}
// lib.mapAttrs' (
name: vlan:
lib.nameValuePair "30-${vlanIf name}" {
netdevConfig = {
Name = vlanIf name;
Kind = "vlan";
};
vlanConfig.Id = vlan.id;
}
) cfg.vlans;
systemd.network.networks =
let
taggedAll = lib.mapAttrsToList (_: vlan: { VLAN = vlan.id; }) cfg.vlans;
in
{
# WAN port carries only the PPPoE session; no IP config of its own.
"10-wan" = {
matchConfig.Name = cfg.wan.interface;
networkConfig.LinkLocalAddressing = "no";
vlan = lib.optional (cfg.wan.vlanId != null) "wan-vlan";
linkConfig = {
RequiredForOnline = "carrier";
}
# The wan-vlan subinterface (and thus the PPPoE session) inherits
# the parent port's MAC, so spoofing here covers both cases.
// lib.optionalAttrs (cfg.wan.macAddress != null) {
MACAddress = cfg.wan.macAddress;
};
};
}
// lib.optionalAttrs (cfg.wan.vlanId != null) {
# The ISP-side VLAN subinterface pppd dials on (e.g. AIS tags PPPoE).
"15-wan-vlan" = {
matchConfig.Name = "wan-vlan";
networkConfig.LinkLocalAddressing = "no";
linkConfig.RequiredForOnline = "no";
};
}
// {
# The bridge itself is L2-only; L3 lives on the vlan-* interfaces,
# which hang off the bridge (tagged on the bridge "self" port).
"20-br0" = {
matchConfig.Name = "br0";
networkConfig.LinkLocalAddressing = "no";
vlan = lib.mapAttrsToList (name: _: vlanIf name) cfg.vlans;
bridgeVLANs = taggedAll;
linkConfig.RequiredForOnline = "no";
};
}
// lib.listToAttrs (
map (port: {
name = "25-trunk-${port}";
value = {
matchConfig.Name = port;
networkConfig.Bridge = "br0";
bridgeVLANs = taggedAll;
linkConfig.RequiredForOnline = "no";
};
}) cfg.trunkPorts
)
// lib.mapAttrs' (
name: vlan:
lib.nameValuePair "40-${vlanIf name}" {
matchConfig.Name = vlanIf name;
address = [ "${vlan.address}/${toString vlan.prefixLength}" ];
networkConfig = {
IPv6AcceptRA = false;
# Announce a /64 carved from the DHCPv6-PD prefix on ppp0 (SLAAC).
IPv6SendRA = true;
DHCPPrefixDelegation = true;
};
dhcpPrefixDelegationConfig.SubnetId = "auto";
linkConfig.RequiredForOnline = "no";
}
) cfg.vlans;
};
}
modules/router/dns-dhcp.nix
+79
View File
@@ -0,0 +1,79 @@
# LAN DHCP (Kea) and DNS (Blocky). Fully declarative: one Kea subnet per VLAN
# with dhcp.enable, Blocky as the blocklist resolver every DHCP lease points
# at. Blocky's HTTP listener (:4000) serves Prometheus metrics, scraped by
# control over the mesh (firewall.nix scopes it to the mesh subnet).
{
config,
lib,
...
}:
let
cfg = config.cnx.router;
dhcpVlans = lib.filterAttrs (_: vlan: vlan.dhcp.enable) cfg.vlans;
in
{
config = lib.mkIf cfg.enable {
services.kea.dhcp4 = {
enable = true;
settings = {
interfaces-config.interfaces = lib.mapAttrsToList (name: _: "vlan-${name}") dhcpVlans;
lease-database = {
type = "memfile";
persist = true;
name = "/var/lib/kea/dhcp4.leases";
};
valid-lifetime = 86400;
subnet4 = lib.mapAttrsToList (name: vlan: {
id = vlan.id;
subnet = vlan.subnet;
interface = "vlan-${name}";
valid-lifetime = vlan.dhcp.leaseTime;
pools = [ { pool = "${vlan.dhcp.pool.from} - ${vlan.dhcp.pool.to}"; } ];
option-data = [
{
name = "routers";
data = vlan.address;
}
{
name = "domain-name-servers";
data = vlan.address;
}
];
}) dhcpVlans;
};
};
services.blocky = {
enable = true;
settings = {
ports = {
dns = 53;
http = 4000;
};
upstreams.groups.default = [
"9.9.9.9"
"149.112.112.112"
"2620:fe::fe"
];
blocking = {
denylists.ads = [
"https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts"
];
clientGroupsBlock.default = [ "ads" ];
};
caching = {
minTime = "5m";
prefetching = true;
};
prometheus.enable = true;
};
};
# The router itself resolves via public resolvers, not via Blocky, so DNS
# for deploys/updates survives a broken local resolver.
networking.nameservers = [
"9.9.9.9"
"1.1.1.1"
];
};
}
modules/router/firewall.nix
+62
View File
@@ -0,0 +1,62 @@
# Router firewall/NAT policy (nftables). Trust model:
# mgmt VLAN -> trusted: router services, all VLANs, WAN
# other VLANs -> DNS/DHCP on the router + WAN (if allowWan); no inter-VLAN
# WAN (ppp0) -> nothing inbound beyond established/related
# mesh -> admin SSH + metrics scrapes (same trust boundary as the fleet)
{
config,
lib,
...
}:
let
cfg = config.cnx.router;
mesh = import ../mesh-hosts.nix { inherit config lib; };
vlanIfs = lib.mapAttrsToList (name: _: "vlan-${name}") cfg.vlans;
wanVlanIfs = lib.mapAttrsToList (name: _: "vlan-${name}") (
lib.filterAttrs (_: vlan: vlan.allowWan) cfg.vlans
);
nonMgmtIfs = lib.filter (i: i != "vlan-mgmt") vlanIfs;
ifSet = ifs: "{ ${lib.concatStringsSep ", " (map (i: "\"${i}\"") ifs)} }";
in
{
config = lib.mkIf cfg.enable {
networking.nftables.enable = true;
# SSH reachable only from the mgmt VLAN (trusted) and the mesh — never
# from the WAN or the other VLANs.
services.openssh.openFirewall = false;
networking.firewall = {
enable = true;
filterForward = true;
trustedInterfaces = [ "vlan-mgmt" ];
# Non-mgmt VLANs may only talk to the router's DNS and DHCP.
interfaces = lib.genAttrs nonMgmtIfs (_: {
allowedTCPPorts = [ 53 ];
allowedUDPPorts = [
53
67
];
});
extraInputRules = ''
ip6 saddr ${mesh.subnet} tcp dport 22 accept comment "admin ssh over the mesh"
ip6 saddr ${mesh.subnet} tcp dport 4000 accept comment "blocky metrics scrape from control"
'';
extraForwardRules = ''
tcp flags syn tcp option maxseg size set rt mtu comment "MSS clamp for PPPoE mtu 1492"
iifname "vlan-mgmt" accept comment "mgmt reaches all VLANs and the WAN"
iifname ${ifSet wanVlanIfs} oifname "ppp0" accept comment "LAN to internet"
'';
};
networking.nat = {
enable = true;
externalInterface = "ppp0";
internalInterfaces = vlanIfs;
};
};
}
modules/router/ipv6.nix
+31
View File
@@ -0,0 +1,31 @@
# IPv6 on the PPPoE uplink: run networkd's DHCPv6 client on ppp0 to obtain a
# delegated prefix; each vlan-* interface (default.nix) carves a /64 out of it
# via DHCPPrefixDelegation and announces it to clients with SLAAC.
{
config,
lib,
...
}:
let
cfg = config.cnx.router;
in
{
config = lib.mkIf cfg.enable {
systemd.network.networks."45-ppp0" = {
matchConfig.Name = "ppp0";
networkConfig = {
DHCP = "ipv6";
# pppd owns the v4 address/route on this link; don't let networkd
# tear them down.
KeepConfiguration = "static";
# Default v6 route comes from the ISP's RA when they send one.
IPv6AcceptRA = true;
};
# Many PPPoE ISPs never send an RA with the M flag; solicit regardless.
dhcpV6Config.WithoutRA = "solicit";
linkConfig.RequiredForOnline = "no";
};
boot.kernel.sysctl."net.ipv6.conf.all.forwarding" = lib.mkDefault 1;
};
}
modules/router/omada.nix
+43
View File
@@ -0,0 +1,43 @@
# TP-Link Omada SDN controller for sites with Omada APs/switches. There is no
# nixpkgs package, so it runs as a podman container (mbentley/omada-controller,
# the de-facto standard image). Host networking because device adoption relies
# on L2 broadcast discovery (UDP 29810) on the mgmt VLAN; the default-deny
# input firewall keeps its ports unreachable from WAN and non-mgmt VLANs.
{
config,
lib,
...
}:
let
cfg = config.cnx.router;
mesh = import ../mesh-hosts.nix { inherit config lib; };
in
{
options.cnx.router.omada.enable =
lib.mkEnableOption "TP-Link Omada SDN controller (podman container)";
config = lib.mkIf (cfg.enable && cfg.omada.enable) {
virtualisation.podman.enable = true;
virtualisation.oci-containers = {
backend = "podman";
containers.omada = {
image = "docker.io/mbentley/omada-controller:5.15";
extraOptions = [ "--network=host" ];
environment.TZ = config.time.timeZone;
volumes = [
"/var/lib/omada/data:/opt/tplink/EAPController/data"
"/var/lib/omada/logs:/opt/tplink/EAPController/logs"
];
};
};
# Admin UI (8043) also reachable over the mesh, like Grafana on control.
networking.firewall.extraInputRules = ''
ip6 saddr ${mesh.subnet} tcp dport 8043 accept comment "omada ui over the mesh"
'';
# Controller state (adopted devices, site config, cert) — declared as clan
# state so a borgbackup client can pick it up; backup wiring is a later step.
clan.core.state.omada.folders = [ "/var/lib/omada" ];
};
}
modules/router/pppoe.nix
+64
View File
@@ -0,0 +1,64 @@
# PPPoE WAN session. ISP credentials are entered once at `clan vars generate`
# (prompts). Both are secret — AIS often uses the same string for username and
# password — so neither may land in the Nix store: pppd reads the username from
# an included secret options file and the password from chap/pap-secrets.
{
config,
lib,
...
}:
let
cfg = config.cnx.router;
creds = config.clan.core.vars.generators.pppoe-credentials;
in
{
config = lib.mkIf cfg.enable {
clan.core.vars.generators.pppoe-credentials = {
prompts.username = {
description = "PPPoE username (from the ISP)";
type = "hidden";
};
prompts.password = {
description = "PPPoE password (from the ISP)";
type = "hidden";
};
files."user-opts".secret = true;
files."chap-secrets".secret = true;
script = ''
user="$(cat "$prompts"/username)"
pass="$(cat "$prompts"/password)"
printf 'user "%s"\n' "$user" > "$out"/user-opts
printf '"%s" * "%s"\n' "$user" "$pass" > "$out"/chap-secrets
'';
};
services.pppd = {
enable = true;
peers.wan = {
autostart = true;
config = ''
plugin pppoe.so ${cfg.wan.pppInterface}
ifname ppp0
file ${creds.files."user-opts".path}
noipdefault
defaultroute
noauth
hide-password
persist
maxfail 0
holdoff 5
lcp-echo-interval 15
lcp-echo-failure 3
+ipv6
mtu 1492
mru 1492
'';
};
};
# pppd looks up the password for `user` in these files at dial time; both
# point at the same generated `"<user>" * "<pass>"` line (PAP and CHAP).
environment.etc."ppp/chap-secrets".source = creds.files."chap-secrets".path;
environment.etc."ppp/pap-secrets".source = creds.files."chap-secrets".path;
};
}