mob next [ci-skip] [ci skip] [skip ci]
lastFile:docs/src/gateways.md
This commit is contained in:
@@ -1,5 +1,10 @@
|
||||
{ config, lib, ... }:
|
||||
let
|
||||
hosts = import ./modules/hosts.nix;
|
||||
mesh = import ./modules/mesh-hosts.nix {
|
||||
dir = config.directory;
|
||||
inherit lib;
|
||||
};
|
||||
|
||||
# Single source of the machine list: inventory AND the per-machine baseline
|
||||
# below are both derived from it, so no machine can be added without the
|
||||
@@ -23,6 +28,9 @@ in
|
||||
|
||||
inventory.machines = fleet;
|
||||
|
||||
# Services defined in this repo (module.input = "self"). See modules/clan/.
|
||||
modules.router = ./modules/clan/router;
|
||||
|
||||
inventory.instances = {
|
||||
|
||||
# Admin SSH keys + root password, split per the clan-core migration off
|
||||
@@ -87,6 +95,85 @@ in
|
||||
roles.server.machines.control = { };
|
||||
roles.client.machines.ns1 = { };
|
||||
};
|
||||
|
||||
# Site gateways: one router instance per site, settings = the site's
|
||||
# port/VLAN layout. Everything else (PPPoE, Kea/Blocky, nftables, IPv6-PD,
|
||||
# CrowdSec, iperf3, speedtest) follows from modules/clan/router.
|
||||
router = {
|
||||
module = {
|
||||
name = "router";
|
||||
input = "self";
|
||||
};
|
||||
# Fleet-wide settings: admin paths (SSH, scrapes, iperf3, Omada UI) are
|
||||
# opened to the mesh; internal proxy names live under
|
||||
# <site><n>.cnx.network with wildcard certs issued via DNS-01 against
|
||||
# ns1 (which must declare the gateway's acme-secret.nix generator).
|
||||
roles.default.settings = {
|
||||
mesh.subnet = mesh.subnet;
|
||||
proxy = {
|
||||
domain = "cnx.network";
|
||||
acme.nameserver = hosts.ns1.ipv4;
|
||||
acme.email = "postmaster@cnx.email";
|
||||
};
|
||||
};
|
||||
# Chiang Mai (site 1): Topton 1U, Intel N300, 4x i226-V 2.5G. Port roles
|
||||
# use the expected igc names — verify against facter.json after the
|
||||
# first install and adjust if the box enumerates differently.
|
||||
roles.default.machines.gw-cnx-1.settings = {
|
||||
site = "cnx";
|
||||
siteId = 1;
|
||||
wan.interface = "enp1s0";
|
||||
wan.vlanId = null; # this ISP runs PPPoE untagged on the port
|
||||
wan.macAddress = "a8:b8:e0:01:06:87";
|
||||
trunkPorts = [
|
||||
"enp2s0"
|
||||
# "enp3s0" # STAGING: serves as the uplink until cutover (see the machine config)
|
||||
];
|
||||
# Dedicated on-site recovery port: untagged mgmt, always available even
|
||||
# if the switch config is broken.
|
||||
accessPorts.enp4s0 = "mgmt";
|
||||
# Replaces the newedge.house OPNsense box; renumbered to the fleet
|
||||
# convention (10.1.<vlanId>.0/24, router .1, pool .100-.199). The old
|
||||
# untagged LAN becomes tagged mgmt — infra switch ports get PVID 10.
|
||||
vlans = {
|
||||
mgmt = {
|
||||
id = 10; # 10.1.10.0/24 — servers, APs, switches, Omada, admin
|
||||
dhcp.reservations.storinator01 = {
|
||||
hwAddress = "7c:c2:55:e0:d6:40";
|
||||
ipAddress = "10.1.10.53";
|
||||
};
|
||||
};
|
||||
lan.id = 20; # 10.1.20.0/24 — trusted clients
|
||||
iot.id = 40; # 10.1.40.0/24
|
||||
voip.id = 50; # 10.1.50.0/24
|
||||
dmz.id = 60; # 10.1.60.0/24
|
||||
unit1.id = 110; # 10.1.110.0/24
|
||||
unit2.id = 120; # 10.1.120.0/24
|
||||
unit3.id = 130; # 10.1.130.0/24
|
||||
unit4.id = 140; # 10.1.140.0/24
|
||||
unit5 = {
|
||||
id = 150; # 10.1.150.0/24
|
||||
dhcp.reservations.newt = {
|
||||
hwAddress = "7c:d3:0a:21:58:0b";
|
||||
ipAddress = "10.1.150.22";
|
||||
};
|
||||
};
|
||||
};
|
||||
# This site runs the Omada controller for its APs/switches.
|
||||
omada.enable = true;
|
||||
|
||||
# Internal reverse proxy: real wildcard cert *.cnx1.cnx.network; Blocky
|
||||
# resolves the names to the router's LAN address for mgmt+lan clients.
|
||||
proxy = {
|
||||
enable = true;
|
||||
services.omada = {
|
||||
# Omada's UI is HTTPS with a self-signed cert on the host network.
|
||||
backend = "https://127.0.0.1:8043";
|
||||
insecureSkipVerify = true;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
# Fleet-wide baseline applied to every machine. Secrets minted by
|
||||
|
||||
+45
-24
@@ -3,11 +3,20 @@
|
||||
NixOS routers (OPNsense replacements) on Topton 1U fanless boxes — Intel N300,
|
||||
4x i226-V 2.5G ports (some units add 2x 10G SFP+), 16GB DDR5, 256GB mSATA.
|
||||
Four sites are planned; the first is `gw-cnx-1` (Chiang Mai). Everything lives
|
||||
in the reusable `modules/router/` module (`cnx.router.*` options); a site's
|
||||
machine config only sets port names, VLANs, and feature flags.
|
||||
in the `router` clan service (`modules/clan/router/`, registered as
|
||||
`modules.router` in `clan.nix`); a site joins by adding its machine to the
|
||||
`router` inventory instance with settings for port names, VLANs, and feature
|
||||
flags (`inventory.instances.router.roles.default.machines.<gw>.settings`).
|
||||
|
||||
Naming: `gw-<city>-<n>`, e.g. `gw-cnx-1`.
|
||||
|
||||
The service has a NixOS VM test (`modules/clan/router/tests/vm/`): an ISP
|
||||
node runs a PPPoE access concentrator, a `client` sits on the tagged lan/iot
|
||||
VLANs behind the trunk, an `admin` on the untagged mgmt access port. It checks
|
||||
the PPPoE dial-in, leases/reservations, Blocky, NAT, `allowWan`, the mgmt-only
|
||||
trust model and SSH exposure. Run it with `nix build .#checks.x86_64-linux.router`
|
||||
(also part of `nix flake check`).
|
||||
|
||||
## What each gateway runs
|
||||
|
||||
| Function | Implementation |
|
||||
@@ -55,22 +64,24 @@ Trust model: mgmt → everything; other VLANs → router DNS/DHCP + internet onl
|
||||
|
||||
## Adding a site
|
||||
|
||||
1. Copy `machines/gw-cnx-1/` to `machines/gw-<city>-<n>/`; set `site`,
|
||||
`siteId` (next free number), port names, VLANs, and `omada.enable`.
|
||||
Fill the real disk id in `disko.nix` (from the installer:
|
||||
`ls -l /dev/disk/by-id/`). Keep the `mgmt`/`lan` VLANs.
|
||||
2. Add the machine to `inventory.machines` in `clan.nix` and to the machine
|
||||
list in `modules/mesh-hosts.nix`. Do **not** add it to `modules/hosts.nix`
|
||||
(dynamic PPPoE IP; clan connects over the mesh).
|
||||
1. Copy `machines/gw-cnx-1/` to `machines/gw-<city>-<n>/` (disko, facter,
|
||||
hostId, timezone). Fill the real disk id in `disko.nix` (from the
|
||||
installer: `ls -l /dev/disk/by-id/`).
|
||||
2. Add the machine to `inventory.machines` in `clan.nix`, to the `router`
|
||||
instance (`roles.default.machines.gw-<city>-<n>.settings`: `site`,
|
||||
`siteId` (next free number), port names, VLANs, `omada.enable`; keep the
|
||||
`mgmt`/`lan` VLANs), and to the machine list in `modules/mesh-hosts.nix`.
|
||||
Do **not** add it to `modules/hosts.nix` (dynamic PPPoE IP; clan connects
|
||||
over the mesh).
|
||||
3. Add its node_exporter and Blocky scrape targets in
|
||||
`modules/monitoring/server.nix`.
|
||||
4. If the site uses the internal proxy (`cnx.router.proxy.enable`), wire ns1:
|
||||
import `(import ../../modules/dns/acme-gw-secret.nix "gw-<city>-<n>")` in
|
||||
**both** the gateway's and ns1's configuration, and on ns1 add the
|
||||
`dns-acme-gw-<city>-<n>-knot` generator, its `keyFiles` entry, and an
|
||||
`acl_acme_gw_<city>_<n>` scoped to `_acme-challenge.<city><n>` on the
|
||||
`cnx.network` zone (copy the `gw-cnx-1` blocks). Then
|
||||
`clan vars generate ns1` and redeploy ns1.
|
||||
4. If the site uses the internal proxy (`proxy.enable` in its settings), wire
|
||||
ns1: import `(import ../../modules/clan/router/acme-secret.nix
|
||||
"gw-<city>-<n>")` in ns1's configuration (the gateway declares the same
|
||||
shared generator itself), and on ns1 add the `dns-acme-gw-<city>-<n>-knot`
|
||||
generator, its `keyFiles` entry, and an `acl_acme_gw_<city>_<n>` scoped to
|
||||
`_acme-challenge.<city><n>` on the `cnx.network` zone (copy the `gw-cnx-1`
|
||||
blocks). Then `clan vars generate ns1` and redeploy ns1.
|
||||
5. `git add` the new machine directory — flake evaluation only sees
|
||||
git-tracked files, so an untracked `machines/gw-…/` is silently ignored.
|
||||
6. `clan vars generate gw-<city>-<n>` — prompts for the site's PPPoE
|
||||
@@ -86,9 +97,10 @@ Trust model: mgmt → everything; other VLANs → router DNS/DHCP + internet onl
|
||||
|
||||
## Omada controller
|
||||
|
||||
Sites with TP-Link Omada APs/switches set `cnx.router.omada.enable = true`.
|
||||
The controller runs as a podman container (`mbentley/omada-controller` — there
|
||||
is no nixpkgs package) with host networking on the mgmt VLAN, where adoption
|
||||
Sites with TP-Link Omada APs/switches set `omada.enable = true` in their
|
||||
router settings. The controller runs as a podman container
|
||||
(`mbentley/omada-controller` — there is no nixpkgs package) with host
|
||||
networking on the mgmt VLAN, where adoption
|
||||
broadcasts (UDP 29810) live. UI: `https://<mgmt-addr>:8043` from mgmt, or the
|
||||
gateway's mesh address from an admin machine. Controller state is under
|
||||
`/var/lib/omada`, declared as clan state (`clan.core.state.omada`); wiring it
|
||||
@@ -96,12 +108,12 @@ into the borgbackup instance is still a follow-up.
|
||||
|
||||
## Internal reverse proxy
|
||||
|
||||
`cnx.router.proxy.enable` puts Caddy on the gateway, terminating TLS for
|
||||
`<service>.<site><n>.cnx.network` (e.g. `https://omada.cnx1.cnx.network`) and
|
||||
forwarding to internal backends:
|
||||
`proxy.enable` in a site's router settings puts Caddy on the gateway,
|
||||
terminating TLS for `<service>.<site><n>.cnx.network` (e.g.
|
||||
`https://omada.cnx1.cnx.network`) and forwarding to internal backends:
|
||||
|
||||
```nix
|
||||
cnx.router.proxy = {
|
||||
proxy = {
|
||||
enable = true;
|
||||
services.omada = {
|
||||
backend = "https://127.0.0.1:8043";
|
||||
@@ -114,13 +126,22 @@ cnx.router.proxy = {
|
||||
gateway, issued via ACME DNS-01 (lego/RFC2136) against ns1 — works behind
|
||||
PPPoE with zero inbound reachability, and browsers trust it without a CA
|
||||
install. Each gateway has its own TSIG key (`acme_gw_<city>_<n>`) that ns1
|
||||
scopes to `_acme-challenge.<site><n>` TXT records only.
|
||||
scopes to `_acme-challenge.<site><n>` TXT records only. The zone, ns1's
|
||||
address and the ACME contact are fleet-wide role settings in `clan.nix`
|
||||
(`proxy.domain`, `proxy.acme.nameserver`, `proxy.acme.email`).
|
||||
- **Resolution**: the names exist only internally — Blocky answers
|
||||
`*.<site><n>.cnx.network` with the router's `lan` address; the public
|
||||
`cnx.network` zone never carries them.
|
||||
- **Access**: `proxy.allowVlans` (default `mgmt` + `lan`) get 443 (and 80 for
|
||||
the HTTP→HTTPS redirect). Not exposed to WAN, guest VLANs, or the mesh.
|
||||
|
||||
## Using the service from another clan
|
||||
|
||||
The service is exported as `clan.modules.router` of this flake; add this
|
||||
repo as a flake input and reference it with `module.input = "<input name>"`.
|
||||
`modules/clan/router/README.md` is the consumer-facing description (it is
|
||||
also the module's manifest readme, shown by `clan modules`).
|
||||
|
||||
## Runbook
|
||||
|
||||
- **PPPoE down**: `systemctl status pppd-wan`, `journalctl -u pppd-wan` on the
|
||||
|
||||
@@ -48,6 +48,9 @@
|
||||
"x86_64-darwin"
|
||||
];
|
||||
|
||||
# perSystem.clan.nixosTests: NixOS VM tests of our own clan services.
|
||||
imports = [ clan-core.flakeModules.testModule ];
|
||||
|
||||
flake = {
|
||||
inherit (clan.config) nixosConfigurations nixosModules clanInternals;
|
||||
clan = clan.config;
|
||||
@@ -84,6 +87,12 @@
|
||||
formatter = treefmtEval.config.build.wrapper;
|
||||
checks.formatting = treefmtEval.config.build.check self;
|
||||
|
||||
# `nix build .#checks.x86_64-linux.router` (VM test, Linux only).
|
||||
clan.nixosTests.router = {
|
||||
imports = [ ./modules/clan/router/tests/vm ];
|
||||
clan.modules.router = ./modules/clan/router;
|
||||
};
|
||||
|
||||
# Add the Clan cli tool to the dev shell.
|
||||
# Use "nix develop" to enter the dev shell.
|
||||
devShells.default = pkgs.mkShell {
|
||||
|
||||
@@ -1,13 +1,9 @@
|
||||
# Site gateway Chiang Mai (site 1): Topton 1U, Intel N300, 4x i226-V 2.5G.
|
||||
# Port roles below use the expected igc names — verify against facter.json
|
||||
# after the first install and adjust if the box enumerates differently.
|
||||
# The router itself (ports, VLANs, features) is the `router` service instance
|
||||
# in clan.nix; only machine-local bits live here.
|
||||
{ config, lib, ... }:
|
||||
{
|
||||
imports = [
|
||||
../../modules/router
|
||||
../../modules/monitoring/exporters.nix
|
||||
(import ../../modules/dns/acme-gw-secret.nix "gw-cnx-1")
|
||||
];
|
||||
imports = [ ../../modules/monitoring/exporters.nix ];
|
||||
|
||||
clan.core.sops.defaultGroups = [ "admins" ];
|
||||
|
||||
@@ -20,68 +16,13 @@
|
||||
builtins.hashString "sha256" config.networking.hostName
|
||||
);
|
||||
|
||||
cnx.router = {
|
||||
enable = true;
|
||||
site = "cnx";
|
||||
siteId = 1;
|
||||
wan.interface = "enp1s0";
|
||||
wan.vlanId = null; # this ISP runs PPPoE untagged on the port
|
||||
wan.macAddress = "a8:b8:e0:01:06:87";
|
||||
trunkPorts = [
|
||||
"enp2s0"
|
||||
# "enp3s0" # STAGING: serves as the uplink below until cutover
|
||||
];
|
||||
# Dedicated on-site recovery port: untagged mgmt, always available even
|
||||
# if the switch config is broken.
|
||||
accessPorts.enp4s0 = "mgmt";
|
||||
# Replaces the newedge.house OPNsense box; renumbered to the fleet
|
||||
# convention (10.1.<vlanId>.0/24, router .1, pool .100-.199). The old
|
||||
# untagged LAN becomes tagged mgmt — infra switch ports get PVID 10.
|
||||
vlans = {
|
||||
mgmt = {
|
||||
id = 10; # 10.1.10.0/24 — servers, APs, switches, Omada, admin
|
||||
dhcp.reservations.storinator01 = {
|
||||
hwAddress = "7c:c2:55:e0:d6:40";
|
||||
ipAddress = "10.1.10.53";
|
||||
};
|
||||
};
|
||||
lan.id = 20; # 10.1.20.0/24 — trusted clients
|
||||
iot.id = 40; # 10.1.40.0/24
|
||||
voip.id = 50; # 10.1.50.0/24
|
||||
dmz.id = 60; # 10.1.60.0/24
|
||||
unit1.id = 110; # 10.1.110.0/24
|
||||
unit2.id = 120; # 10.1.120.0/24
|
||||
unit3.id = 130; # 10.1.130.0/24
|
||||
unit4.id = 140; # 10.1.140.0/24
|
||||
unit5 = {
|
||||
id = 150; # 10.1.150.0/24
|
||||
dhcp.reservations.newt = {
|
||||
hwAddress = "7c:d3:0a:21:58:0b";
|
||||
ipAddress = "10.1.150.22";
|
||||
};
|
||||
};
|
||||
};
|
||||
# This site runs the Omada controller for its APs/switches.
|
||||
omada.enable = true;
|
||||
|
||||
# Internal reverse proxy: real wildcard cert *.cnx1.cnx.network; Blocky
|
||||
# resolves the names to the router's LAN address for mgmt+lan clients.
|
||||
proxy = {
|
||||
enable = true;
|
||||
services.omada = {
|
||||
# Omada's UI is HTTPS with a self-signed cert on the host network.
|
||||
backend = "https://127.0.0.1:8043";
|
||||
insecureSkipVerify = true;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
# STAGING (remove at cutover, and restore enp3s0 to trunkPorts): DHCP-client
|
||||
# uplink into the existing OPNsense LAN so the box has internet + mesh while
|
||||
# it runs alongside the old router. Default-deny firewall on this interface
|
||||
# (it's in no VLAN zone); PPPoE simply retries until the WAN port is cabled.
|
||||
# Do NOT connect the trunk ports to the production switch while staging —
|
||||
# Kea on tag 10 would fight the OPNsense LAN DHCP in one broadcast domain.
|
||||
# STAGING (remove at cutover, and restore enp3s0 to trunkPorts in clan.nix):
|
||||
# DHCP-client uplink into the existing OPNsense LAN so the box has internet +
|
||||
# mesh while it runs alongside the old router. Default-deny firewall on this
|
||||
# interface (it's in no VLAN zone); PPPoE simply retries until the WAN port
|
||||
# is cabled. Do NOT connect the trunk ports to the production switch while
|
||||
# staging — Kea on tag 10 would fight the OPNsense LAN DHCP in one broadcast
|
||||
# domain.
|
||||
systemd.network.networks."05-staging" = {
|
||||
matchConfig.Name = "enp3s0";
|
||||
networkConfig.DHCP = "ipv4";
|
||||
|
||||
@@ -6,7 +6,10 @@
|
||||
}:
|
||||
let
|
||||
domains = import ../../modules/dns/domains.nix;
|
||||
mesh = import ../../modules/mesh-hosts.nix { inherit config lib; };
|
||||
mesh = import ../../modules/mesh-hosts.nix {
|
||||
dir = config.clan.core.settings.directory;
|
||||
inherit lib;
|
||||
};
|
||||
hosts = import ../../modules/hosts.nix;
|
||||
in
|
||||
{
|
||||
@@ -14,7 +17,8 @@ in
|
||||
../../modules/dns/authoritative.nix
|
||||
../../modules/dns/acme-mx1-secret.nix
|
||||
../../modules/dns/acme-web01-secret.nix
|
||||
(import ../../modules/dns/acme-gw-secret.nix "gw-cnx-1")
|
||||
# Same shared TSIG secret the gateway's router service declares (proxy).
|
||||
(import ../../modules/clan/router/acme-secret.nix "gw-cnx-1")
|
||||
../../modules/static-ipv6.nix
|
||||
../../modules/monitoring/exporters.nix
|
||||
];
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
# router
|
||||
|
||||
Turns a machine with several NICs into a site gateway: PPPoE WAN (ISP
|
||||
credentials via vars prompts), a VLAN-filtering bridge over the LAN ports with
|
||||
one L3 interface per VLAN, Kea DHCP and Blocky DNS per VLAN, nftables
|
||||
firewall/NAT, DHCPv6-PD, CrowdSec, an iperf3 server and a WAN speed-test
|
||||
timer. Optional: the TP-Link Omada controller (podman) and an internal Caddy
|
||||
reverse proxy with a real wildcard certificate (ACME DNS-01).
|
||||
|
||||
Addressing convention: a site owns `10.<siteId>.0.0/16`; VLAN `<id>` defaults
|
||||
to `10.<siteId>.<id>.0/24`, router at `.1`, DHCP pool `.100-.199`. The `mgmt`
|
||||
and `lan` VLANs are mandatory. Trust model: mgmt reaches everything; other
|
||||
VLANs get router DNS/DHCP and (with `allowWan`) the internet, no inter-VLAN;
|
||||
WAN nothing inbound; the admin mesh (`mesh.subnet`) gets SSH, metrics, iperf3
|
||||
and the Omada UI.
|
||||
|
||||
## Usage from another clan
|
||||
|
||||
```nix
|
||||
# flake.nix
|
||||
inputs.cnx-network.url = "git+https://<host>/B4L/cnx-network-clan";
|
||||
|
||||
# clan.nix
|
||||
inventory.instances.router = {
|
||||
module = { name = "router"; input = "cnx-network"; };
|
||||
roles.default.settings.mesh.subnet = "fd..::/88"; # your admin overlay
|
||||
roles.default.machines.gw-1.settings = {
|
||||
site = "ams";
|
||||
siteId = 1;
|
||||
wan.interface = "enp1s0";
|
||||
wan.vlanId = 10; # or null for untagged PPPoE
|
||||
trunkPorts = [ "enp2s0" ];
|
||||
accessPorts.enp4s0 = "mgmt"; # untagged on-site recovery port
|
||||
vlans = {
|
||||
mgmt.id = 10;
|
||||
lan.id = 20;
|
||||
iot = { id = 40; allowWan = false; };
|
||||
};
|
||||
};
|
||||
};
|
||||
```
|
||||
|
||||
Then `clan vars generate gw-1` prompts for the PPPoE username/password.
|
||||
|
||||
### Internal proxy
|
||||
|
||||
`proxy.enable` serves `<name>.<site><siteId>.<proxy.domain>` under a wildcard
|
||||
certificate obtained via RFC 2136 DNS-01 against `proxy.acme.nameserver`. The
|
||||
gateway signs updates with TSIG key `acme_<hostname with _>`, whose secret is
|
||||
the shared `dns-acme-<hostname>-secret` generator declared by this service.
|
||||
The nameserver machine must declare the same generator so both sides hold one
|
||||
secret — import `acme-secret.nix` from this directory with the gateway's name:
|
||||
|
||||
```nix
|
||||
imports = [ (import "${inputs.cnx-network}/modules/clan/router/acme-secret.nix" "gw-1") ];
|
||||
```
|
||||
|
||||
and load the key with an acl scoped to `_acme-challenge.<site><siteId>`.
|
||||
|
||||
The service does not open the WAN to anything; reach gateways over your mesh.
|
||||
One instance per machine.
|
||||
@@ -0,0 +1,22 @@
|
||||
# Shared TSIG secret for a gateway's dedicated ACME key (function: machine
|
||||
# name -> NixOS module). The acme_<machine> key lets that gateway — and only
|
||||
# it — write _acme-challenge.<site><siteId> TXT records on the authoritative
|
||||
# nameserver to obtain its internal wildcard cert via DNS-01 (proxy.nix).
|
||||
#
|
||||
# The router service declares it on the gateway automatically when
|
||||
# proxy.enable is set. The nameserver machine must declare the very same
|
||||
# generator so both sides share one secret:
|
||||
# imports = [ (import <router-service>/acme-secret.nix "gw-cnx-1") ];
|
||||
# and then load it into its DNS server as key acme_gw_cnx_1 (hmac-sha256)
|
||||
# with an acl scoped to that gateway's _acme-challenge label.
|
||||
machine:
|
||||
{ pkgs, ... }:
|
||||
{
|
||||
clan.core.vars.generators."dns-acme-${machine}-secret" = {
|
||||
share = true;
|
||||
files."secret".secret = true;
|
||||
runtimeInputs = [ pkgs.openssl ];
|
||||
# 32 random bytes, base64 — a valid hmac-sha256 TSIG secret.
|
||||
script = ''openssl rand -base64 32 | tr -d '\n' > "$out"/secret'';
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
# CrowdSec security engine + nftables bouncer: parses sshd auth attempts from
|
||||
# the journal and bans offending source IPs at the firewall. Log-based (no
|
||||
# inline DPI) so it costs the N300 next to nothing.
|
||||
{ settings }:
|
||||
{ ... }:
|
||||
let
|
||||
cfg = settings;
|
||||
in
|
||||
{
|
||||
services.crowdsec = {
|
||||
enable = true;
|
||||
autoUpdateService = true;
|
||||
hub.collections = [
|
||||
"crowdsecurity/linux"
|
||||
"crowdsecurity/sshd"
|
||||
];
|
||||
localConfig = {
|
||||
acquisitions = [
|
||||
{
|
||||
source = "journalctl";
|
||||
journalctl_filter = [ "_SYSTEMD_UNIT=sshd.service" ];
|
||||
labels.type = "syslog";
|
||||
}
|
||||
];
|
||||
# Never ban the ZeroTier mesh — it is the only admin path to these
|
||||
# boxes (no public SSH), so a false positive would lock us out.
|
||||
# Parser-stage whitelist: mesh events are dropped before any scenario.
|
||||
parsers.s02Enrich = [
|
||||
{
|
||||
name = "cnx/mesh-whitelist";
|
||||
description = "Whitelist the ZeroTier management mesh";
|
||||
whitelist = {
|
||||
reason = "ZeroTier mesh is the admin path";
|
||||
cidr = [ cfg.mesh.subnet ];
|
||||
};
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
services.crowdsec-firewall-bouncer = {
|
||||
enable = true;
|
||||
registerBouncer.enable = true;
|
||||
settings.mode = "nftables";
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
# Site gateway (OPNsense replacement) for the Topton 1U boxes, as a clan
|
||||
# service: PPPoE WAN, VLAN-filtering bridge over the LAN ports, per-VLAN
|
||||
# DHCP/DNS, firewall/NAT, and the optional Omada controller / internal proxy.
|
||||
#
|
||||
# Exported as `clan.modules.router` of this flake; used here with
|
||||
# `module.input = "self"` and from other clans with `module.input =
|
||||
# "<this flake's input name>"`. README.md has the consumer view. A site joins
|
||||
# through the inventory:
|
||||
#
|
||||
# inventory.instances.router = {
|
||||
# module = { name = "router"; input = "self"; };
|
||||
# roles.default.settings.mesh.subnet = ...; # fleet-wide
|
||||
# roles.default.machines.gw-<city>-<n>.settings = { site = ...; ... };
|
||||
# };
|
||||
#
|
||||
# The settings schema lives in interface.nix. Implementation files that need
|
||||
# the settings are functions `{ settings }: <NixOS module>`; the evaluated
|
||||
# settings are handed in with importApply so nothing goes through
|
||||
# machine-level options. The rest are plain NixOS modules.
|
||||
{ lib, ... }:
|
||||
{
|
||||
_class = "clan.service";
|
||||
manifest.name = "router";
|
||||
manifest.description = "Site gateway: PPPoE WAN, VLAN bridge, DHCP/DNS, firewall/NAT";
|
||||
manifest.categories = [ "Network" ];
|
||||
manifest.readme = builtins.readFile ./README.md;
|
||||
|
||||
roles.default = {
|
||||
description = "Turns the machine into the site's router (one instance per machine).";
|
||||
interface = ./interface.nix;
|
||||
|
||||
perInstance =
|
||||
{ settings, machine, ... }:
|
||||
{
|
||||
nixosModule.imports = [
|
||||
./ipv6.nix
|
||||
]
|
||||
# The proxy's TSIG secret is shared with the nameserver (acme-secret.nix).
|
||||
++ lib.optional settings.proxy.enable (import ./acme-secret.nix machine.name)
|
||||
++ map (file: lib.modules.importApply file { inherit settings; }) [
|
||||
./network.nix
|
||||
./pppoe.nix
|
||||
./firewall.nix
|
||||
./dns-dhcp.nix
|
||||
./crowdsec.nix
|
||||
./omada.nix
|
||||
./proxy.nix
|
||||
./iperf.nix
|
||||
./speedtest.nix
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
# A machine has exactly one WAN port and one VLAN layout; two instances would
|
||||
# both claim br0/ppp0 and fight over Kea/Blocky/nftables.
|
||||
perMachine =
|
||||
{ instances, machine, ... }:
|
||||
{
|
||||
nixosModule.assertions = [
|
||||
{
|
||||
assertion = lib.length (lib.attrNames instances) == 1;
|
||||
message = "router: ${machine.name} is a gateway in several instances (${lib.concatStringsSep ", " (lib.attrNames instances)}); a machine can only be one router.";
|
||||
}
|
||||
];
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
# LAN DHCP (Kea) and DNS (Blocky). Fully declarative: one Kea subnet per VLAN
|
||||
# with dhcp.enable, Blocky as the blocklist resolver every DHCP lease points
|
||||
# at. Blocky's HTTP listener (:4000) serves Prometheus metrics, scraped by
|
||||
# control over the mesh (firewall.nix scopes it to the mesh subnet).
|
||||
{ settings }:
|
||||
{ lib, ... }:
|
||||
let
|
||||
cfg = settings;
|
||||
dhcpVlans = lib.filterAttrs (_: vlan: vlan.dhcp.enable) cfg.vlans;
|
||||
in
|
||||
{
|
||||
services.kea.dhcp4 = {
|
||||
enable = true;
|
||||
settings = {
|
||||
interfaces-config.interfaces = lib.mapAttrsToList (name: _: "vlan-${name}") dhcpVlans;
|
||||
lease-database = {
|
||||
type = "memfile";
|
||||
persist = true;
|
||||
name = "/var/lib/kea/dhcp4.leases";
|
||||
};
|
||||
valid-lifetime = 86400;
|
||||
subnet4 = lib.mapAttrsToList (name: vlan: {
|
||||
id = vlan.id;
|
||||
subnet = vlan.subnet;
|
||||
interface = "vlan-${name}";
|
||||
valid-lifetime = vlan.dhcp.leaseTime;
|
||||
pools = [ { pool = "${vlan.dhcp.pool.from} - ${vlan.dhcp.pool.to}"; } ];
|
||||
reservations = lib.mapAttrsToList (host: res: {
|
||||
hostname = host;
|
||||
hw-address = res.hwAddress;
|
||||
ip-address = res.ipAddress;
|
||||
}) vlan.dhcp.reservations;
|
||||
option-data = [
|
||||
{
|
||||
name = "routers";
|
||||
data = vlan.address;
|
||||
}
|
||||
{
|
||||
name = "domain-name-servers";
|
||||
data = vlan.address;
|
||||
}
|
||||
];
|
||||
}) dhcpVlans;
|
||||
};
|
||||
};
|
||||
|
||||
services.blocky = {
|
||||
enable = true;
|
||||
settings = {
|
||||
ports = {
|
||||
dns = 53;
|
||||
http = 4000;
|
||||
};
|
||||
upstreams.groups.default = [
|
||||
"9.9.9.9"
|
||||
"149.112.112.112"
|
||||
"2620:fe::fe"
|
||||
];
|
||||
blocking = {
|
||||
denylists.ads = [
|
||||
"https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts"
|
||||
];
|
||||
clientGroupsBlock.default = [ "ads" ];
|
||||
};
|
||||
caching = {
|
||||
minTime = "5m";
|
||||
prefetching = true;
|
||||
};
|
||||
prometheus.enable = true;
|
||||
};
|
||||
};
|
||||
|
||||
# The router itself resolves via public resolvers, not via Blocky, so DNS
|
||||
# for deploys/updates survives a broken local resolver. networkd would
|
||||
# enable systemd-resolved by default, whose stub listener on 127.0.0.53:53
|
||||
# makes Blocky's wildcard :53 bind fail — plain resolv.conf instead.
|
||||
services.resolved.enable = false;
|
||||
networking.nameservers = [
|
||||
"9.9.9.9"
|
||||
"1.1.1.1"
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
# Router firewall/NAT policy (nftables). Trust model:
|
||||
# mgmt VLAN -> trusted: router services, all VLANs, WAN
|
||||
# other VLANs -> DNS/DHCP on the router + WAN (if allowWan); no inter-VLAN
|
||||
# WAN (ppp0) -> nothing inbound beyond established/related
|
||||
# mesh -> admin SSH + metrics scrapes (same trust boundary as the fleet)
|
||||
{ settings }:
|
||||
{ lib, ... }:
|
||||
let
|
||||
cfg = settings;
|
||||
|
||||
vlanIfs = lib.mapAttrsToList (name: _: "vlan-${name}") cfg.vlans;
|
||||
wanVlanIfs = lib.mapAttrsToList (name: _: "vlan-${name}") (
|
||||
lib.filterAttrs (_: vlan: vlan.allowWan) cfg.vlans
|
||||
);
|
||||
nonMgmtIfs = lib.filter (i: i != "vlan-mgmt") vlanIfs;
|
||||
ifSet = ifs: "{ ${lib.concatStringsSep ", " (map (i: "\"${i}\"") ifs)} }";
|
||||
in
|
||||
{
|
||||
networking.nftables.enable = true;
|
||||
|
||||
# SSH reachable only from the mgmt VLAN (trusted) and the mesh — never
|
||||
# from the WAN or the other VLANs.
|
||||
services.openssh.openFirewall = false;
|
||||
|
||||
networking.firewall = {
|
||||
enable = true;
|
||||
filterForward = true;
|
||||
trustedInterfaces = [ "vlan-mgmt" ];
|
||||
|
||||
# Non-mgmt VLANs may only talk to the router's DNS and DHCP.
|
||||
interfaces = lib.genAttrs nonMgmtIfs (_: {
|
||||
allowedTCPPorts = [ 53 ];
|
||||
allowedUDPPorts = [
|
||||
53
|
||||
67
|
||||
];
|
||||
});
|
||||
|
||||
extraInputRules = ''
|
||||
ip6 saddr ${cfg.mesh.subnet} tcp dport 22 accept comment "admin ssh over the mesh"
|
||||
ip6 saddr ${cfg.mesh.subnet} tcp dport 4000 accept comment "blocky metrics scrape from control"
|
||||
'';
|
||||
|
||||
extraForwardRules = ''
|
||||
tcp flags syn tcp option maxseg size set rt mtu comment "MSS clamp for PPPoE mtu 1492"
|
||||
iifname "vlan-mgmt" accept comment "mgmt reaches all VLANs and the WAN"
|
||||
iifname ${ifSet wanVlanIfs} oifname "ppp0" accept comment "LAN to internet"
|
||||
'';
|
||||
};
|
||||
|
||||
networking.nat = {
|
||||
enable = true;
|
||||
externalInterface = "ppp0";
|
||||
internalInterfaces = vlanIfs;
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,234 @@
|
||||
# Settings of the `router` service (inventory `roles.default.settings`).
|
||||
# Pure schema: no machine config is reachable here; the implementation files
|
||||
# get the evaluated result as `settings`.
|
||||
#
|
||||
# Fleet addressing convention: each site owns 10.<siteId>.0.0/16. A VLAN's
|
||||
# subnet defaults to 10.<siteId>.<vlanId>.0/24 with the router at .1 and the
|
||||
# DHCP pool at .100-.199. VLANs that need more space (e.g. public-wifi guest)
|
||||
# override `subnet`/`address`/`dhcp.pool` and take a wider block from the
|
||||
# upper half (10.<siteId>.128.0/17), e.g. guest -> 10.<siteId>.128.0/22.
|
||||
# VLAN ids: 10 = mgmt, 20 = lan (mandatory); 30 = guest, 40 = iot (reserved).
|
||||
{ config, lib, ... }:
|
||||
let
|
||||
site = toString config.siteId;
|
||||
|
||||
vlanModule =
|
||||
{ config, ... }:
|
||||
let
|
||||
octet = toString config.id;
|
||||
in
|
||||
{
|
||||
options = {
|
||||
id = lib.mkOption {
|
||||
type = lib.types.ints.between 1 4094;
|
||||
description = "802.1Q VLAN id (fleet convention: 10 mgmt, 20 lan, 30 guest, 40 iot).";
|
||||
};
|
||||
address = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "10.${site}.${octet}.1";
|
||||
defaultText = lib.literalExpression ''"10.<siteId>.<id>.1"'';
|
||||
description = "Router address on this VLAN.";
|
||||
};
|
||||
prefixLength = lib.mkOption {
|
||||
type = lib.types.ints.between 8 30;
|
||||
default = 24;
|
||||
};
|
||||
subnet = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "10.${site}.${octet}.0/24";
|
||||
defaultText = lib.literalExpression ''"10.<siteId>.<id>.0/24"'';
|
||||
description = "The VLAN's network in CIDR form (must contain `address`).";
|
||||
};
|
||||
dhcp = {
|
||||
enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
};
|
||||
pool = {
|
||||
from = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "10.${site}.${octet}.100";
|
||||
defaultText = lib.literalExpression ''"10.<siteId>.<id>.100"'';
|
||||
};
|
||||
to = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "10.${site}.${octet}.199";
|
||||
defaultText = lib.literalExpression ''"10.<siteId>.<id>.199"'';
|
||||
};
|
||||
};
|
||||
leaseTime = lib.mkOption {
|
||||
type = lib.types.ints.positive;
|
||||
default = 86400;
|
||||
description = ''
|
||||
Lease validity in seconds. Lower it for high-churn networks,
|
||||
e.g. public-WiFi guest VLANs (3600-7200), so the pool recycles.
|
||||
'';
|
||||
};
|
||||
reservations = lib.mkOption {
|
||||
type = lib.types.attrsOf (
|
||||
lib.types.submodule {
|
||||
options = {
|
||||
hwAddress = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
example = "aa:bb:cc:dd:ee:ff";
|
||||
description = "Client MAC address.";
|
||||
};
|
||||
ipAddress = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "Fixed address handed to this client (inside the VLAN's subnet, outside the pool).";
|
||||
};
|
||||
};
|
||||
}
|
||||
);
|
||||
default = { };
|
||||
description = "Static DHCP leases; the attribute name becomes the client's hostname.";
|
||||
};
|
||||
};
|
||||
allowWan = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
description = "Whether clients on this VLAN may reach the internet.";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
proxyServiceModule = {
|
||||
options = {
|
||||
backend = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
example = "https://127.0.0.1:8043";
|
||||
description = "URL Caddy forwards to (internal/mesh address).";
|
||||
};
|
||||
insecureSkipVerify = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
description = "Skip TLS verification towards the backend (self-signed upstreams like Omada).";
|
||||
};
|
||||
};
|
||||
};
|
||||
in
|
||||
{
|
||||
options = {
|
||||
site = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "City code of the site, e.g. \"cnx\".";
|
||||
};
|
||||
|
||||
siteId = lib.mkOption {
|
||||
type = lib.types.ints.between 1 254;
|
||||
description = "Site number; drives the 10.<siteId>.<vlan>.0/24 addressing.";
|
||||
};
|
||||
|
||||
wan.interface = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "Physical WAN port the PPPoE session runs on.";
|
||||
};
|
||||
|
||||
wan.vlanId = lib.mkOption {
|
||||
type = lib.types.nullOr (lib.types.ints.between 1 4094);
|
||||
default = null;
|
||||
description = ''
|
||||
802.1Q tag the ISP requires for the PPPoE session (AIS Thailand: 10);
|
||||
null for untagged PPPoE directly on the port. Unrelated to the LAN
|
||||
VLANs — this tag exists only on the WAN port.
|
||||
'';
|
||||
};
|
||||
|
||||
wan.macAddress = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
example = "aa:bb:cc:dd:ee:ff";
|
||||
description = ''
|
||||
Spoofed MAC for the WAN port, e.g. to keep the MAC the ISP has
|
||||
pinned (cloned from the old router). null keeps the hardware MAC.
|
||||
'';
|
||||
};
|
||||
|
||||
trunkPorts = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
description = "LAN ports carrying all VLANs tagged (incl. any 10G SFP+ ports).";
|
||||
};
|
||||
|
||||
accessPorts = lib.mkOption {
|
||||
type = lib.types.attrsOf lib.types.str;
|
||||
default = { };
|
||||
example = {
|
||||
enp4s0 = "mgmt";
|
||||
};
|
||||
description = ''
|
||||
Ports acting as untagged access ports on a single VLAN (port name ->
|
||||
VLAN name). Frames are untagged on the wire; the bridge tags them with
|
||||
the VLAN's PVID. Use for an always-available on-site mgmt port.
|
||||
'';
|
||||
};
|
||||
|
||||
vlans = lib.mkOption {
|
||||
type = lib.types.attrsOf (lib.types.submodule vlanModule);
|
||||
description = "VLANs served at this site; `mgmt` and `lan` are mandatory.";
|
||||
};
|
||||
|
||||
mesh.subnet = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
example = "fd12:3456:789a:bcde:f000::/88";
|
||||
description = ''
|
||||
IPv6 prefix of the admin mesh (the ZeroTier overlay; see
|
||||
modules/mesh-hosts.nix). Admin SSH, metrics scrapes, iperf3 and the
|
||||
Omada UI accept connections from it, and CrowdSec never bans it.
|
||||
'';
|
||||
};
|
||||
|
||||
omada.enable = lib.mkEnableOption "TP-Link Omada SDN controller (podman container)";
|
||||
|
||||
proxy = {
|
||||
enable = lib.mkEnableOption "internal reverse proxy (Caddy, wildcard cert via DNS-01)";
|
||||
|
||||
domain = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
example = "example.net";
|
||||
description = ''
|
||||
Parent zone of the proxy names: services are served as
|
||||
<name>.<site><siteId>.<domain> under a wildcard certificate.
|
||||
'';
|
||||
};
|
||||
|
||||
acme = {
|
||||
nameserver = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
example = "203.0.113.53";
|
||||
description = ''
|
||||
Authoritative nameserver of `domain` that accepts RFC 2136
|
||||
updates for _acme-challenge.<site><siteId> with this gateway's
|
||||
TSIG key (acme_<hostname with _>, secret from the shared
|
||||
dns-acme-<hostname>-secret generator, see acme-secret.nix).
|
||||
'';
|
||||
};
|
||||
email = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
example = "postmaster@example.net";
|
||||
description = "ACME account contact.";
|
||||
};
|
||||
};
|
||||
|
||||
services = lib.mkOption {
|
||||
type = lib.types.attrsOf (lib.types.submodule proxyServiceModule);
|
||||
default = { };
|
||||
description = "Proxied services; attr name becomes <name>.<site><siteId>.<domain>.";
|
||||
};
|
||||
|
||||
allowVlans = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [
|
||||
"mgmt"
|
||||
"lan"
|
||||
];
|
||||
description = "VLANs whose clients may reach the proxy (443, plus 80 for the redirect).";
|
||||
};
|
||||
};
|
||||
|
||||
speedtest.interval = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "hourly";
|
||||
description = "systemd OnCalendar spec for the WAN speed test.";
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
# iperf3 server on every gateway, for throughput testing from any LAN segment
|
||||
# (e.g. validating AP/switch links: `iperf3 -c 10.<siteId>.<vlan>.1`) and from
|
||||
# admin machines over the mesh. Never reachable from the WAN (default-deny).
|
||||
{ settings }:
|
||||
{ lib, ... }:
|
||||
let
|
||||
cfg = settings;
|
||||
vlanIfs = lib.mapAttrsToList (name: _: "vlan-${name}") cfg.vlans;
|
||||
in
|
||||
{
|
||||
services.iperf3.enable = true;
|
||||
|
||||
networking.firewall.interfaces = lib.genAttrs vlanIfs (_: {
|
||||
allowedTCPPorts = [ 5201 ];
|
||||
allowedUDPPorts = [ 5201 ];
|
||||
});
|
||||
|
||||
networking.firewall.extraInputRules = ''
|
||||
ip6 saddr ${cfg.mesh.subnet} tcp dport 5201 accept comment "iperf3 over the mesh"
|
||||
ip6 saddr ${cfg.mesh.subnet} udp dport 5201 accept comment "iperf3 over the mesh"
|
||||
'';
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
# IPv6 on the PPPoE uplink: run networkd's DHCPv6 client on ppp0 to obtain a
|
||||
# delegated prefix; each vlan-* interface (network.nix) carves a /64 out of it
|
||||
# via DHCPPrefixDelegation and announces it to clients with SLAAC.
|
||||
{ lib, ... }:
|
||||
{
|
||||
systemd.network.networks."45-ppp0" = {
|
||||
matchConfig.Name = "ppp0";
|
||||
networkConfig = {
|
||||
DHCP = "ipv6";
|
||||
# pppd owns the v4 address/route on this link; don't let networkd
|
||||
# tear them down.
|
||||
KeepConfiguration = "static";
|
||||
# Default v6 route comes from the ISP's RA when they send one.
|
||||
IPv6AcceptRA = true;
|
||||
};
|
||||
# Many PPPoE ISPs never send an RA with the M flag; solicit regardless.
|
||||
dhcpV6Config.WithoutRA = "solicit";
|
||||
linkConfig.RequiredForOnline = "no";
|
||||
};
|
||||
|
||||
boot.kernel.sysctl."net.ipv6.conf.all.forwarding" = lib.mkDefault 1;
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
# L2/L3 of the gateway: PPPoE WAN port (optionally on an ISP VLAN), a
|
||||
# VLAN-filtering bridge over the LAN ports, and one L3 interface per VLAN.
|
||||
{ settings }:
|
||||
{ lib, pkgs, ... }:
|
||||
let
|
||||
cfg = settings;
|
||||
|
||||
vlanIf = name: "vlan-${name}";
|
||||
in
|
||||
{
|
||||
assertions = [
|
||||
{
|
||||
assertion = cfg.vlans ? mgmt && cfg.vlans ? lan;
|
||||
message = "router: every site must define the `mgmt` and `lan` VLANs.";
|
||||
}
|
||||
{
|
||||
assertion = lib.all (v: cfg.vlans ? ${v}) (lib.attrValues cfg.accessPorts);
|
||||
message = "router: every accessPorts value must name a defined VLAN.";
|
||||
}
|
||||
{
|
||||
assertion = lib.all (p: !(cfg.accessPorts ? ${p})) cfg.trunkPorts;
|
||||
message = "router: a port cannot be both a trunk and an access port.";
|
||||
}
|
||||
];
|
||||
|
||||
# Router diagnostics toolkit: packets (tcpdump), path (mtr), link
|
||||
# negotiation (ethtool), NAT state (conntrack), DNS (kdig), per-flow
|
||||
# bandwidth (iftop), WAN throughput (librespeed-cli; iperf3 covers LAN).
|
||||
environment.systemPackages = with pkgs; [
|
||||
tcpdump
|
||||
mtr
|
||||
ethtool
|
||||
conntrack-tools
|
||||
knot-dns
|
||||
iftop
|
||||
librespeed-cli
|
||||
];
|
||||
|
||||
networking.useNetworkd = true;
|
||||
networking.useDHCP = false;
|
||||
systemd.network.enable = true;
|
||||
|
||||
systemd.network.netdevs = {
|
||||
"20-br0" = {
|
||||
netdevConfig = {
|
||||
Name = "br0";
|
||||
Kind = "bridge";
|
||||
};
|
||||
bridgeConfig.VLANFiltering = true;
|
||||
};
|
||||
}
|
||||
// lib.optionalAttrs (cfg.wan.vlanId != null) {
|
||||
"15-wan-vlan" = {
|
||||
netdevConfig = {
|
||||
Name = "wan-vlan";
|
||||
Kind = "vlan";
|
||||
};
|
||||
vlanConfig.Id = cfg.wan.vlanId;
|
||||
};
|
||||
}
|
||||
// lib.mapAttrs' (
|
||||
name: vlan:
|
||||
lib.nameValuePair "30-${vlanIf name}" {
|
||||
netdevConfig = {
|
||||
Name = vlanIf name;
|
||||
Kind = "vlan";
|
||||
};
|
||||
vlanConfig.Id = vlan.id;
|
||||
}
|
||||
) cfg.vlans;
|
||||
|
||||
systemd.network.networks =
|
||||
let
|
||||
taggedAll = lib.mapAttrsToList (_: vlan: { VLAN = vlan.id; }) cfg.vlans;
|
||||
in
|
||||
{
|
||||
# WAN port carries only the PPPoE session; no IP config of its own.
|
||||
"10-wan" = {
|
||||
matchConfig.Name = cfg.wan.interface;
|
||||
networkConfig.LinkLocalAddressing = "no";
|
||||
vlan = lib.optional (cfg.wan.vlanId != null) "wan-vlan";
|
||||
linkConfig = {
|
||||
RequiredForOnline = "carrier";
|
||||
}
|
||||
# The wan-vlan subinterface (and thus the PPPoE session) inherits
|
||||
# the parent port's MAC, so spoofing here covers both cases.
|
||||
// lib.optionalAttrs (cfg.wan.macAddress != null) {
|
||||
MACAddress = cfg.wan.macAddress;
|
||||
};
|
||||
};
|
||||
}
|
||||
// lib.optionalAttrs (cfg.wan.vlanId != null) {
|
||||
# The ISP-side VLAN subinterface pppd dials on (e.g. AIS tags PPPoE).
|
||||
"15-wan-vlan" = {
|
||||
matchConfig.Name = "wan-vlan";
|
||||
networkConfig.LinkLocalAddressing = "no";
|
||||
linkConfig.RequiredForOnline = "no";
|
||||
};
|
||||
}
|
||||
// {
|
||||
# The bridge itself is L2-only; L3 lives on the vlan-* interfaces,
|
||||
# which hang off the bridge (tagged on the bridge "self" port).
|
||||
"20-br0" = {
|
||||
matchConfig.Name = "br0";
|
||||
networkConfig.LinkLocalAddressing = "no";
|
||||
vlan = lib.mapAttrsToList (name: _: vlanIf name) cfg.vlans;
|
||||
bridgeVLANs = taggedAll;
|
||||
linkConfig.RequiredForOnline = "no";
|
||||
};
|
||||
}
|
||||
// lib.listToAttrs (
|
||||
map (port: {
|
||||
name = "25-trunk-${port}";
|
||||
value = {
|
||||
matchConfig.Name = port;
|
||||
networkConfig.Bridge = "br0";
|
||||
bridgeVLANs = taggedAll;
|
||||
linkConfig.RequiredForOnline = "no";
|
||||
};
|
||||
}) cfg.trunkPorts
|
||||
)
|
||||
// lib.mapAttrs' (
|
||||
port: vlanName:
|
||||
lib.nameValuePair "25-access-${port}" {
|
||||
matchConfig.Name = port;
|
||||
networkConfig.Bridge = "br0";
|
||||
bridgeVLANs = [
|
||||
{
|
||||
VLAN = cfg.vlans.${vlanName}.id;
|
||||
PVID = cfg.vlans.${vlanName}.id;
|
||||
EgressUntagged = cfg.vlans.${vlanName}.id;
|
||||
}
|
||||
];
|
||||
linkConfig.RequiredForOnline = "no";
|
||||
}
|
||||
) cfg.accessPorts
|
||||
// lib.mapAttrs' (
|
||||
name: vlan:
|
||||
lib.nameValuePair "40-${vlanIf name}" {
|
||||
matchConfig.Name = vlanIf name;
|
||||
address = [ "${vlan.address}/${toString vlan.prefixLength}" ];
|
||||
networkConfig = {
|
||||
IPv6AcceptRA = false;
|
||||
# Announce a /64 carved from the DHCPv6-PD prefix on ppp0 (SLAAC).
|
||||
IPv6SendRA = true;
|
||||
DHCPPrefixDelegation = true;
|
||||
};
|
||||
dhcpPrefixDelegationConfig.SubnetId = "auto";
|
||||
linkConfig.RequiredForOnline = "no";
|
||||
}
|
||||
) cfg.vlans;
|
||||
}
|
||||
@@ -3,20 +3,13 @@
|
||||
# the de-facto standard image). Host networking because device adoption relies
|
||||
# on L2 broadcast discovery (UDP 29810) on the mgmt VLAN; the default-deny
|
||||
# input firewall keeps its ports unreachable from WAN and non-mgmt VLANs.
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
...
|
||||
}:
|
||||
{ settings }:
|
||||
{ config, lib, ... }:
|
||||
let
|
||||
cfg = config.cnx.router;
|
||||
mesh = import ../mesh-hosts.nix { inherit config lib; };
|
||||
cfg = settings;
|
||||
in
|
||||
{
|
||||
options.cnx.router.omada.enable =
|
||||
lib.mkEnableOption "TP-Link Omada SDN controller (podman container)";
|
||||
|
||||
config = lib.mkIf (cfg.enable && cfg.omada.enable) {
|
||||
config = lib.mkIf cfg.omada.enable {
|
||||
virtualisation.podman.enable = true;
|
||||
virtualisation.oci-containers = {
|
||||
backend = "podman";
|
||||
@@ -33,7 +26,7 @@ in
|
||||
|
||||
# Admin UI (8043) also reachable over the mesh, like Grafana on control.
|
||||
networking.firewall.extraInputRules = ''
|
||||
ip6 saddr ${mesh.subnet} tcp dport 8043 accept comment "omada ui over the mesh"
|
||||
ip6 saddr ${cfg.mesh.subnet} tcp dport 8043 accept comment "omada ui over the mesh"
|
||||
'';
|
||||
|
||||
# Controller state (adopted devices, site config, cert) — declared as clan
|
||||
@@ -0,0 +1,61 @@
|
||||
# PPPoE WAN session. ISP credentials are entered once at `clan vars generate`
|
||||
# (prompts). Both are secret — AIS often uses the same string for username and
|
||||
# password — so neither may land in the Nix store: pppd reads the username from
|
||||
# an included secret options file and the password from chap/pap-secrets.
|
||||
{ settings }:
|
||||
{ config, ... }:
|
||||
let
|
||||
cfg = settings;
|
||||
creds = config.clan.core.vars.generators.pppoe-credentials;
|
||||
# Interface pppd dials on: the WAN port itself, or its ISP VLAN (network.nix).
|
||||
pppInterface = if cfg.wan.vlanId == null then cfg.wan.interface else "wan-vlan";
|
||||
in
|
||||
{
|
||||
clan.core.vars.generators.pppoe-credentials = {
|
||||
prompts.username = {
|
||||
description = "PPPoE username (from the ISP)";
|
||||
type = "hidden";
|
||||
};
|
||||
prompts.password = {
|
||||
description = "PPPoE password (from the ISP)";
|
||||
type = "hidden";
|
||||
};
|
||||
files."user-opts".secret = true;
|
||||
files."chap-secrets".secret = true;
|
||||
script = ''
|
||||
user="$(cat "$prompts"/username)"
|
||||
pass="$(cat "$prompts"/password)"
|
||||
printf 'user "%s"\n' "$user" > "$out"/user-opts
|
||||
printf '"%s" * "%s"\n' "$user" "$pass" > "$out"/chap-secrets
|
||||
'';
|
||||
};
|
||||
|
||||
services.pppd = {
|
||||
enable = true;
|
||||
peers.wan = {
|
||||
autostart = true;
|
||||
config = ''
|
||||
plugin pppoe.so ${pppInterface}
|
||||
ifname ppp0
|
||||
file ${creds.files."user-opts".path}
|
||||
noipdefault
|
||||
defaultroute
|
||||
noauth
|
||||
hide-password
|
||||
persist
|
||||
maxfail 0
|
||||
holdoff 5
|
||||
lcp-echo-interval 15
|
||||
lcp-echo-failure 3
|
||||
+ipv6
|
||||
mtu 1492
|
||||
mru 1492
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
# pppd looks up the password for `user` in these files at dial time; both
|
||||
# point at the same generated `"<user>" * "<pass>"` line (PAP and CHAP).
|
||||
environment.etc."ppp/chap-secrets".source = creds.files."chap-secrets".path;
|
||||
environment.etc."ppp/pap-secrets".source = creds.files."chap-secrets".path;
|
||||
}
|
||||
@@ -1,94 +1,57 @@
|
||||
# Internal reverse proxy for the gateway: Caddy terminates TLS for
|
||||
# <service>.<site><siteId>.cnx.network (e.g. omada.cnx1.cnx.network) and
|
||||
# <service>.<site><siteId>.<proxy.domain> (e.g. omada.cnx1.cnx.network) and
|
||||
# forwards to backends by their internal address. The cert is a real Let's
|
||||
# Encrypt wildcard (*.<site><siteId>.cnx.network) obtained via ACME DNS-01
|
||||
# against ns1 with a gateway-scoped TSIG key, so browsers trust it without
|
||||
# any CA install; the names only *resolve* internally — Blocky answers them
|
||||
# with the router's LAN address, the public zone never carries them.
|
||||
# Encrypt wildcard (*.<site><siteId>.<domain>) obtained via ACME DNS-01
|
||||
# against proxy.acme.nameserver with a gateway-scoped TSIG key, so browsers
|
||||
# trust it without any CA install; the names only *resolve* internally —
|
||||
# Blocky answers them with the router's LAN address, the public zone never
|
||||
# carries them.
|
||||
#
|
||||
# Requires the machine to also import the shared secret generator:
|
||||
# (import ../../modules/dns/acme-gw-secret.nix "<hostname>")
|
||||
# and ns1 to hold the matching key + acl (see machines/ns1/configuration.nix).
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
...
|
||||
}:
|
||||
# The TSIG secret is the shared dns-acme-<hostname>-secret generator
|
||||
# (acme-secret.nix, declared here via default.nix); the nameserver machine
|
||||
# must declare the same generator and load the key (this fleet: ns1).
|
||||
{ settings }:
|
||||
{ config, lib, ... }:
|
||||
let
|
||||
cfg = config.cnx.router;
|
||||
hosts = import ../hosts.nix;
|
||||
cfg = settings;
|
||||
hostname = config.networking.hostName;
|
||||
tsigKey = "acme_${lib.replaceStrings [ "-" ] [ "_" ] hostname}";
|
||||
certName = "${cfg.site}${toString cfg.siteId}.cnx.network";
|
||||
|
||||
serviceModule = {
|
||||
options = {
|
||||
backend = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
example = "https://127.0.0.1:8043";
|
||||
description = "URL Caddy forwards to (internal/mesh address).";
|
||||
};
|
||||
insecureSkipVerify = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
description = "Skip TLS verification towards the backend (self-signed upstreams like Omada).";
|
||||
};
|
||||
};
|
||||
};
|
||||
certName = "${cfg.site}${toString cfg.siteId}.${cfg.proxy.domain}";
|
||||
nameserver = "${cfg.proxy.acme.nameserver}:53";
|
||||
in
|
||||
{
|
||||
options.cnx.router.proxy = {
|
||||
enable = lib.mkEnableOption "internal reverse proxy (Caddy, wildcard cert via DNS-01)";
|
||||
|
||||
services = lib.mkOption {
|
||||
type = lib.types.attrsOf (lib.types.submodule serviceModule);
|
||||
default = { };
|
||||
description = "Proxied services; attr name becomes <name>.${certName}.";
|
||||
};
|
||||
|
||||
allowVlans = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [
|
||||
"mgmt"
|
||||
"lan"
|
||||
];
|
||||
description = "VLANs whose clients may reach the proxy (443, plus 80 for the redirect).";
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf (cfg.enable && cfg.proxy.enable) {
|
||||
config = lib.mkIf cfg.proxy.enable {
|
||||
assertions = [
|
||||
{
|
||||
assertion = lib.all (v: cfg.vlans ? ${v}) cfg.proxy.allowVlans;
|
||||
message = "cnx.router.proxy.allowVlans must name VLANs defined in cnx.router.vlans.";
|
||||
message = "router: proxy.allowVlans must name VLANs defined in vlans.";
|
||||
}
|
||||
];
|
||||
|
||||
# Render the shared per-gateway TSIG secret into a lego rfc2136 env file;
|
||||
# same pattern as web01 (modules/web-proxy.nix), scoped on ns1 to
|
||||
# _acme-challenge.<site><siteId> TXT records only.
|
||||
# Render the shared per-gateway TSIG secret into a lego rfc2136 env file,
|
||||
# scoped on the nameserver to _acme-challenge.<site><siteId> TXT records.
|
||||
clan.core.vars.generators."dns-acme-${hostname}-rfc2136" = {
|
||||
files."rfc2136.env".secret = true; # root-owned; systemd reads it as root
|
||||
dependencies = [ "dns-acme-${hostname}-secret" ];
|
||||
script = ''
|
||||
printf 'RFC2136_NAMESERVER=${hosts.ns1.ipv4}:53\nRFC2136_TSIG_ALGORITHM=hmac-sha256.\nRFC2136_TSIG_KEY=${tsigKey}\nRFC2136_TSIG_SECRET=%s\n' \
|
||||
printf 'RFC2136_NAMESERVER=${nameserver}\nRFC2136_TSIG_ALGORITHM=hmac-sha256.\nRFC2136_TSIG_KEY=${tsigKey}\nRFC2136_TSIG_SECRET=%s\n' \
|
||||
"$(cat "$in"/dns-acme-${hostname}-secret/secret)" > "$out"/rfc2136.env
|
||||
'';
|
||||
};
|
||||
|
||||
security.acme = {
|
||||
acceptTerms = true;
|
||||
defaults.email = "postmaster@cnx.email";
|
||||
# One wildcard for every proxied service; DNS-01 against ns1, so issuance
|
||||
# works behind PPPoE with no inbound reachability at all.
|
||||
defaults.email = cfg.proxy.acme.email;
|
||||
# One wildcard for every proxied service; DNS-01, so issuance works
|
||||
# behind PPPoE with no inbound reachability at all.
|
||||
certs.${certName} = {
|
||||
domain = "*.${certName}";
|
||||
dnsProvider = "rfc2136";
|
||||
environmentFile =
|
||||
config.clan.core.vars.generators."dns-acme-${hostname}-rfc2136".files."rfc2136.env".path;
|
||||
# ns1 is the only nameserver that accepts this key's UPDATE; check
|
||||
# propagation against it directly rather than a public resolver.
|
||||
dnsResolver = "${hosts.ns1.ipv4}:53";
|
||||
# Only that nameserver accepts this key's UPDATE; check propagation
|
||||
# against it directly rather than a public resolver.
|
||||
dnsResolver = nameserver;
|
||||
# Caddy reads the cert from explicit file paths (tls directive below),
|
||||
# so it won't notice a renewal on its own.
|
||||
reloadServices = [ "caddy.service" ];
|
||||
@@ -0,0 +1,55 @@
|
||||
# Periodic WAN speed test so ISP degradation shows up as a trend instead of a
|
||||
# complaint. A timer runs librespeed-cli and writes the results as Prometheus
|
||||
# metrics into node_exporter's textfile collector — they ride the existing
|
||||
# 9100 scrape to VictoriaMetrics, where alerts.nix compares each run against
|
||||
# the link's own 7-day median (no per-site threshold to maintain).
|
||||
{ settings }:
|
||||
{ pkgs, ... }:
|
||||
let
|
||||
cfg = settings;
|
||||
textfileDir = "/var/lib/speedtest";
|
||||
in
|
||||
{
|
||||
services.prometheus.exporters.node.extraFlags = [
|
||||
"--collector.textfile.directory=${textfileDir}"
|
||||
];
|
||||
|
||||
systemd.services.speedtest = {
|
||||
description = "WAN speed test to Prometheus textfile metrics";
|
||||
after = [ "network-online.target" ];
|
||||
wants = [ "network-online.target" ];
|
||||
path = [
|
||||
pkgs.librespeed-cli
|
||||
pkgs.jq
|
||||
];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
StateDirectory = "speedtest";
|
||||
# One test at boot would race PPPoE and log a spurious failure.
|
||||
ExecCondition = "${pkgs.iproute2}/bin/ip link show ppp0";
|
||||
};
|
||||
script = ''
|
||||
tmp="${textfileDir}/.speedtest.prom.tmp"
|
||||
if result=$(librespeed-cli --json); then
|
||||
jq -r '.[0]
|
||||
| "speedtest_download_mbps \(.download)",
|
||||
"speedtest_upload_mbps \(.upload)",
|
||||
"speedtest_ping_ms \(.ping)",
|
||||
"speedtest_jitter_ms \(.jitter)",
|
||||
"speedtest_success 1"' <<<"$result" > "$tmp"
|
||||
else
|
||||
echo "speedtest_success 0" > "$tmp"
|
||||
fi
|
||||
mv "$tmp" "${textfileDir}/speedtest.prom"
|
||||
'';
|
||||
};
|
||||
|
||||
systemd.timers.speedtest = {
|
||||
wantedBy = [ "timers.target" ];
|
||||
timerConfig = {
|
||||
OnCalendar = cfg.speedtest.interval;
|
||||
RandomizedDelaySec = "10m";
|
||||
Persistent = true;
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
# End-to-end VM test of the router service: a PPPoE access concentrator plays
|
||||
# the ISP on the WAN port, a trunk carries tagged lan/iot VLANs to `client`,
|
||||
# and an untagged access port carries mgmt to `admin`.
|
||||
#
|
||||
# isp ---(vlan 1: PPPoE)--- wan [gw] trunk ---(vlan 2: tagged 20/40)--- client
|
||||
# access --(vlan 3: untagged mgmt)--- admin
|
||||
#
|
||||
# What is proven: PPPoE dial-in with the vars-provided credentials, bridge
|
||||
# VLAN tagging/untagging, Kea leases and reservations per VLAN, Blocky
|
||||
# answering on the VLAN with the blocklist active, NAT to the WAN, and the
|
||||
# firewall trust model (allowWan, mgmt-only SSH, no inter-VLAN forwarding).
|
||||
{ pkgs, lib, ... }:
|
||||
let
|
||||
# The vars mock answers every prompt with "mock-prompt-value-<name>"; the
|
||||
# ISP side must accept exactly those.
|
||||
chapSecrets = ''"mock-prompt-value-username" * "mock-prompt-value-password" *'';
|
||||
ispAddress = "192.0.2.1";
|
||||
|
||||
clientMac = "02:00:00:00:00:20";
|
||||
clientAddress = "10.9.20.50";
|
||||
adminMac = "02:00:00:00:00:10";
|
||||
adminAddress = "10.9.10.50";
|
||||
in
|
||||
{
|
||||
name = "router";
|
||||
|
||||
clan = {
|
||||
directory = ./.;
|
||||
# Bridges, VLAN netdevs, PPPoE and nftables need a real kernel.
|
||||
test.useContainers = false;
|
||||
inventory = {
|
||||
# Every node is a clan machine (the test framework's defaults require
|
||||
# it); only gw gets the router role.
|
||||
machines = {
|
||||
gw = { };
|
||||
isp = { };
|
||||
client = { };
|
||||
admin = { };
|
||||
};
|
||||
|
||||
instances.router = {
|
||||
module.name = "router";
|
||||
module.input = "self";
|
||||
roles.default.machines.gw.settings = {
|
||||
site = "tst";
|
||||
siteId = 9;
|
||||
mesh.subnet = "fd00:7e57:c1a1:c0de::/64";
|
||||
wan.interface = "wan";
|
||||
trunkPorts = [ "trunk" ];
|
||||
accessPorts.access = "mgmt";
|
||||
vlans = {
|
||||
mgmt = {
|
||||
id = 10;
|
||||
dhcp.reservations.admin = {
|
||||
hwAddress = adminMac;
|
||||
ipAddress = adminAddress;
|
||||
};
|
||||
};
|
||||
lan = {
|
||||
id = 20;
|
||||
dhcp.reservations.client = {
|
||||
hwAddress = clientMac;
|
||||
ipAddress = clientAddress;
|
||||
};
|
||||
};
|
||||
iot = {
|
||||
id = 40;
|
||||
allowWan = false;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
nodes = {
|
||||
gw = {
|
||||
virtualisation.interfaces = {
|
||||
wan = {
|
||||
vlan = 1;
|
||||
assignIP = false;
|
||||
};
|
||||
trunk = {
|
||||
vlan = 2;
|
||||
assignIP = false;
|
||||
};
|
||||
access = {
|
||||
vlan = 3;
|
||||
assignIP = false;
|
||||
};
|
||||
};
|
||||
|
||||
# Something must listen on 22 for the mgmt-only SSH rule to be observable
|
||||
# (a refused and a dropped connection look the same to the client).
|
||||
services.openssh.enable = true;
|
||||
|
||||
# The sandbox has no internet: serve the blocklist from a local file
|
||||
# instead of GitHub, and skip CrowdSec, whose hub sync needs the network
|
||||
# (it is not what this test exercises).
|
||||
services.blocky.settings.blocking.denylists.ads = lib.mkForce [
|
||||
(toString (pkgs.writeText "ads.hosts" "0.0.0.0 ads.example.com\n"))
|
||||
];
|
||||
services.crowdsec.enable = lib.mkForce false;
|
||||
services.crowdsec-firewall-bouncer.enable = lib.mkForce false;
|
||||
};
|
||||
|
||||
isp = {
|
||||
virtualisation.interfaces.wan = {
|
||||
vlan = 1;
|
||||
assignIP = false;
|
||||
};
|
||||
# PPPoE access concentrator: one session, peer gets 192.0.2.10.
|
||||
systemd.services.pppoe-server = {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "network.target" ];
|
||||
serviceConfig.ExecStart =
|
||||
"${pkgs.rp-pppoe}/sbin/pppoe-server -F -O /etc/ppp/pppoe-server-options"
|
||||
+ " -q ${pkgs.ppp}/sbin/pppd -I wan -L ${ispAddress} -R 192.0.2.10";
|
||||
};
|
||||
environment.etc = {
|
||||
"ppp/pppoe-server-options".text = ''
|
||||
plugin pppoe.so
|
||||
require-chap
|
||||
lcp-echo-interval 10
|
||||
lcp-echo-failure 2
|
||||
nobsdcomp
|
||||
noccp
|
||||
novj
|
||||
'';
|
||||
"ppp/chap-secrets" = {
|
||||
text = chapSecrets;
|
||||
mode = "0640";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
client = {
|
||||
virtualisation.interfaces.trunk = {
|
||||
vlan = 2;
|
||||
assignIP = false;
|
||||
};
|
||||
networking.useDHCP = false;
|
||||
networking.vlans = {
|
||||
lan0 = {
|
||||
id = 20;
|
||||
interface = "trunk";
|
||||
};
|
||||
iot0 = {
|
||||
id = 40;
|
||||
interface = "trunk";
|
||||
};
|
||||
};
|
||||
networking.interfaces.lan0 = {
|
||||
useDHCP = true;
|
||||
macAddress = clientMac;
|
||||
};
|
||||
networking.interfaces.iot0.useDHCP = true;
|
||||
environment.systemPackages = [
|
||||
pkgs.dnsutils
|
||||
pkgs.netcat
|
||||
];
|
||||
};
|
||||
|
||||
admin = {
|
||||
virtualisation.interfaces.access = {
|
||||
vlan = 3;
|
||||
assignIP = false;
|
||||
};
|
||||
networking.useDHCP = false;
|
||||
networking.interfaces.access = {
|
||||
useDHCP = true;
|
||||
macAddress = adminMac;
|
||||
};
|
||||
environment.systemPackages = [ pkgs.netcat ];
|
||||
};
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
start_all()
|
||||
|
||||
with subtest("PPPoE session comes up with the vars credentials"):
|
||||
gw.wait_for_unit("pppd-wan.service")
|
||||
gw.wait_until_succeeds("ping -c1 -W1 ${ispAddress}")
|
||||
|
||||
with subtest("DHCP hands out reserved leases per VLAN"):
|
||||
gw.wait_for_unit("kea-dhcp4-server.service")
|
||||
client.wait_until_succeeds("ip -4 addr show lan0 | grep -q 'inet ${clientAddress}/24'")
|
||||
client.wait_until_succeeds("ip -4 addr show iot0 | grep -q 'inet 10.9.40.1[0-9][0-9]/24'")
|
||||
admin.wait_until_succeeds("ip -4 addr show access | grep -q 'inet ${adminAddress}/24'")
|
||||
|
||||
with subtest("Blocky serves the VLAN and blocks the denylist"):
|
||||
gw.wait_for_unit("blocky.service")
|
||||
answer = client.wait_until_succeeds("dig +short +time=2 @10.9.20.1 ads.example.com")
|
||||
assert answer.strip() == "0.0.0.0", f"expected blocked answer, got: {answer!r}"
|
||||
|
||||
with subtest("NAT to the WAN only for VLANs with allowWan"):
|
||||
client.wait_until_succeeds("ping -c1 -W1 -I lan0 ${ispAddress}")
|
||||
client.fail("ping -c1 -W2 -I iot0 ${ispAddress}")
|
||||
|
||||
with subtest("mgmt reaches other VLANs, other VLANs do not"):
|
||||
admin.succeed("ping -c1 -W2 ${clientAddress}")
|
||||
client.fail("ping -c1 -W2 -I lan0 ${adminAddress}")
|
||||
|
||||
with subtest("SSH on the router only from mgmt"):
|
||||
gw.wait_for_open_port(22)
|
||||
admin.succeed("nc -z -w2 10.9.10.1 22")
|
||||
client.fail("nc -z -w2 10.9.20.1 22")
|
||||
'';
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
# Shared TSIG secret for a gateway's dedicated ACME key (function: machine
|
||||
# name -> module). The acme_gw_<x> key lets that gateway — and only it — write
|
||||
# _acme-challenge.<label> TXT records on ns1 to obtain its internal wildcard
|
||||
# cert via DNS-01. ns1 scopes it with a matching acl on the cnx.network zone.
|
||||
# Import on BOTH ns1 and the gateway machine, applied with the machine name:
|
||||
# (import ../../modules/dns/acme-gw-secret.nix "gw-cnx-1")
|
||||
machine:
|
||||
{ pkgs, ... }:
|
||||
{
|
||||
clan.core.vars.generators."dns-acme-${machine}-secret" = {
|
||||
share = true;
|
||||
files."secret".secret = true;
|
||||
runtimeInputs = [ pkgs.openssl ];
|
||||
# 32 random bytes, base64 — a valid hmac-sha256 TSIG secret.
|
||||
script = ''openssl rand -base64 32 | tr -d '\n' > "$out"/secret'';
|
||||
};
|
||||
}
|
||||
@@ -6,7 +6,10 @@
|
||||
}:
|
||||
let
|
||||
# ZeroTier addresses — zone transfers run over the mesh, not the public net.
|
||||
mesh = import ../mesh-hosts.nix { inherit config lib; };
|
||||
mesh = import ../mesh-hosts.nix {
|
||||
dir = config.clan.core.settings.directory;
|
||||
inherit lib;
|
||||
};
|
||||
ns1zt = mesh.hosts.ns1;
|
||||
ns2zt = mesh.hosts.ns2;
|
||||
in
|
||||
|
||||
+4
-1
@@ -8,7 +8,10 @@
|
||||
...
|
||||
}:
|
||||
let
|
||||
mesh = import ./mesh-hosts.nix { inherit config lib; };
|
||||
mesh = import ./mesh-hosts.nix {
|
||||
dir = config.clan.core.settings.directory;
|
||||
inherit lib;
|
||||
};
|
||||
port = 8080;
|
||||
|
||||
site = pkgs.stdenvNoCC.mkDerivation {
|
||||
|
||||
@@ -8,10 +8,10 @@
|
||||
# vars/shared/zerotier-network-<instance>/network-id (instance = "zerotier",
|
||||
# the inventory.instances.zerotier name in clan.nix). This keeps the mesh map in
|
||||
# lockstep with the actual identities: regenerate or re-key a node and its
|
||||
# address here follows automatically. Call as: import ../mesh-hosts.nix { inherit config lib; }.
|
||||
{ config, lib }:
|
||||
# address here follows automatically. `dir` is the clan directory: from NixOS
|
||||
# `config.clan.core.settings.directory`, from clan.nix `config.directory`.
|
||||
{ dir, lib }:
|
||||
let
|
||||
dir = config.clan.core.settings.directory;
|
||||
instance = "zerotier";
|
||||
|
||||
readIp =
|
||||
|
||||
@@ -61,7 +61,7 @@ in
|
||||
}
|
||||
{
|
||||
# WAN quality on the site gateways, fed by the hourly speedtest timer
|
||||
# (modules/router/speedtest.nix) via the node_exporter textfile
|
||||
# (modules/clan/router/speedtest.nix) via the node_exporter textfile
|
||||
# collector. Degradation is judged against the link's own 7-day median,
|
||||
# so no per-site/per-ISP-plan threshold needs maintaining.
|
||||
name = "wan";
|
||||
|
||||
@@ -10,7 +10,10 @@
|
||||
...
|
||||
}:
|
||||
let
|
||||
mesh = import ../mesh-hosts.nix { inherit config lib; };
|
||||
mesh = import ../mesh-hosts.nix {
|
||||
dir = config.clan.core.settings.directory;
|
||||
inherit lib;
|
||||
};
|
||||
knotEnabled = config.services.knot.enable;
|
||||
# node_exporter on every host; knot-exporter only where Knot runs.
|
||||
ports = [ 9100 ] ++ lib.optional knotEnabled 9433;
|
||||
|
||||
@@ -16,7 +16,10 @@
|
||||
...
|
||||
}:
|
||||
let
|
||||
mesh = import ../mesh-hosts.nix { inherit config lib; };
|
||||
mesh = import ../mesh-hosts.nix {
|
||||
dir = config.clan.core.settings.directory;
|
||||
inherit lib;
|
||||
};
|
||||
|
||||
# Grafana 13 (since the externalisation of core data sources) no longer bundles
|
||||
# the Elasticsearch data source, and nixpkgs has no grafanaPlugins entry for it,
|
||||
|
||||
@@ -9,7 +9,10 @@
|
||||
...
|
||||
}:
|
||||
let
|
||||
mesh = import ../mesh-hosts.nix { inherit config lib; };
|
||||
mesh = import ../mesh-hosts.nix {
|
||||
dir = config.clan.core.settings.directory;
|
||||
inherit lib;
|
||||
};
|
||||
probes = import ./blackbox-probes.nix { inherit lib; };
|
||||
vmPort = 8428;
|
||||
logsPort = 9428;
|
||||
|
||||
@@ -1,52 +0,0 @@
|
||||
# CrowdSec security engine + nftables bouncer: parses sshd auth attempts from
|
||||
# the journal and bans offending source IPs at the firewall. Log-based (no
|
||||
# inline DPI) so it costs the N300 next to nothing.
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
...
|
||||
}:
|
||||
let
|
||||
cfg = config.cnx.router;
|
||||
mesh = import ../mesh-hosts.nix { inherit config lib; };
|
||||
in
|
||||
{
|
||||
config = lib.mkIf cfg.enable {
|
||||
services.crowdsec = {
|
||||
enable = true;
|
||||
autoUpdateService = true;
|
||||
hub.collections = [
|
||||
"crowdsecurity/linux"
|
||||
"crowdsecurity/sshd"
|
||||
];
|
||||
localConfig = {
|
||||
acquisitions = [
|
||||
{
|
||||
source = "journalctl";
|
||||
journalctl_filter = [ "_SYSTEMD_UNIT=sshd.service" ];
|
||||
labels.type = "syslog";
|
||||
}
|
||||
];
|
||||
# Never ban the ZeroTier mesh — it is the only admin path to these
|
||||
# boxes (no public SSH), so a false positive would lock us out.
|
||||
# Parser-stage whitelist: mesh events are dropped before any scenario.
|
||||
parsers.s02Enrich = [
|
||||
{
|
||||
name = "cnx/mesh-whitelist";
|
||||
description = "Whitelist the ZeroTier management mesh";
|
||||
whitelist = {
|
||||
reason = "ZeroTier mesh is the admin path";
|
||||
cidr = [ mesh.subnet ];
|
||||
};
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
services.crowdsec-firewall-bouncer = {
|
||||
enable = true;
|
||||
registerBouncer.enable = true;
|
||||
settings.mode = "nftables";
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -1,325 +0,0 @@
|
||||
# Site gateway (OPNsense replacement) for the Topton 1U boxes: PPPoE WAN,
|
||||
# VLAN-filtering bridge over the LAN ports, and per-VLAN L3 interfaces.
|
||||
# Imported by machines/gw-<city>-<n>; everything is driven by cnx.router.*.
|
||||
#
|
||||
# Fleet addressing convention: each site owns 10.<siteId>.0.0/16. A VLAN's
|
||||
# subnet defaults to 10.<siteId>.<vlanId>.0/24 with the router at .1 and the
|
||||
# DHCP pool at .100-.199. VLANs that need more space (e.g. public-wifi guest)
|
||||
# override `subnet`/`address`/`dhcp.pool` and take a wider block from the
|
||||
# upper half (10.<siteId>.128.0/17), e.g. guest -> 10.<siteId>.128.0/22.
|
||||
# VLAN ids: 10 = mgmt, 20 = lan (mandatory); 30 = guest, 40 = iot (reserved).
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
let
|
||||
cfg = config.cnx.router;
|
||||
|
||||
vlanIf = name: "vlan-${name}";
|
||||
site = toString cfg.siteId;
|
||||
|
||||
vlanModule =
|
||||
{ name, config, ... }:
|
||||
let
|
||||
octet = toString config.id;
|
||||
in
|
||||
{
|
||||
options = {
|
||||
id = lib.mkOption {
|
||||
type = lib.types.ints.between 1 4094;
|
||||
description = "802.1Q VLAN id (fleet convention: 10 mgmt, 20 lan, 30 guest, 40 iot).";
|
||||
};
|
||||
address = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "10.${site}.${octet}.1";
|
||||
description = "Router address on this VLAN.";
|
||||
};
|
||||
prefixLength = lib.mkOption {
|
||||
type = lib.types.ints.between 8 30;
|
||||
default = 24;
|
||||
};
|
||||
subnet = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "10.${site}.${octet}.0/24";
|
||||
description = "The VLAN's network in CIDR form (must contain `address`).";
|
||||
};
|
||||
dhcp = {
|
||||
enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
};
|
||||
pool = {
|
||||
from = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "10.${site}.${octet}.100";
|
||||
};
|
||||
to = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "10.${site}.${octet}.199";
|
||||
};
|
||||
};
|
||||
leaseTime = lib.mkOption {
|
||||
type = lib.types.ints.positive;
|
||||
default = 86400;
|
||||
description = ''
|
||||
Lease validity in seconds. Lower it for high-churn networks,
|
||||
e.g. public-WiFi guest VLANs (3600-7200), so the pool recycles.
|
||||
'';
|
||||
};
|
||||
reservations = lib.mkOption {
|
||||
type = lib.types.attrsOf (
|
||||
lib.types.submodule {
|
||||
options = {
|
||||
hwAddress = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
example = "aa:bb:cc:dd:ee:ff";
|
||||
description = "Client MAC address.";
|
||||
};
|
||||
ipAddress = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "Fixed address handed to this client (inside the VLAN's subnet, outside the pool).";
|
||||
};
|
||||
};
|
||||
}
|
||||
);
|
||||
default = { };
|
||||
description = "Static DHCP leases; the attribute name becomes the client's hostname.";
|
||||
};
|
||||
};
|
||||
allowWan = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
description = "Whether clients on this VLAN may reach the internet.";
|
||||
};
|
||||
};
|
||||
};
|
||||
in
|
||||
{
|
||||
imports = [
|
||||
./pppoe.nix
|
||||
./ipv6.nix
|
||||
./firewall.nix
|
||||
./dns-dhcp.nix
|
||||
./crowdsec.nix
|
||||
./omada.nix
|
||||
./proxy.nix
|
||||
./iperf.nix
|
||||
./speedtest.nix
|
||||
];
|
||||
|
||||
options.cnx.router = {
|
||||
enable = lib.mkEnableOption "site gateway (router) role";
|
||||
|
||||
site = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "City code of the site, e.g. \"cnx\".";
|
||||
};
|
||||
|
||||
siteId = lib.mkOption {
|
||||
type = lib.types.ints.between 1 254;
|
||||
description = "Site number; drives the 10.<siteId>.<vlan>.0/24 addressing.";
|
||||
};
|
||||
|
||||
wan.interface = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "Physical WAN port the PPPoE session runs on.";
|
||||
};
|
||||
|
||||
wan.vlanId = lib.mkOption {
|
||||
type = lib.types.nullOr (lib.types.ints.between 1 4094);
|
||||
default = null;
|
||||
description = ''
|
||||
802.1Q tag the ISP requires for the PPPoE session (AIS Thailand: 10);
|
||||
null for untagged PPPoE directly on the port. Unrelated to the LAN
|
||||
VLANs — this tag exists only on the WAN port.
|
||||
'';
|
||||
};
|
||||
|
||||
wan.macAddress = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
example = "aa:bb:cc:dd:ee:ff";
|
||||
description = ''
|
||||
Spoofed MAC for the WAN port, e.g. to keep the MAC the ISP has
|
||||
pinned (cloned from the old router). null keeps the hardware MAC.
|
||||
'';
|
||||
};
|
||||
|
||||
wan.pppInterface = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
internal = true;
|
||||
readOnly = true;
|
||||
default = if cfg.wan.vlanId == null then cfg.wan.interface else "wan-vlan";
|
||||
description = "Interface pppd dials on (the WAN port or its ISP VLAN).";
|
||||
};
|
||||
|
||||
trunkPorts = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
description = "LAN ports carrying all VLANs tagged (incl. any 10G SFP+ ports).";
|
||||
};
|
||||
|
||||
accessPorts = lib.mkOption {
|
||||
type = lib.types.attrsOf lib.types.str;
|
||||
default = { };
|
||||
example = {
|
||||
enp4s0 = "mgmt";
|
||||
};
|
||||
description = ''
|
||||
Ports acting as untagged access ports on a single VLAN (port name ->
|
||||
VLAN name). Frames are untagged on the wire; the bridge tags them with
|
||||
the VLAN's PVID. Use for an always-available on-site mgmt port.
|
||||
'';
|
||||
};
|
||||
|
||||
vlans = lib.mkOption {
|
||||
type = lib.types.attrsOf (lib.types.submodule vlanModule);
|
||||
description = "VLANs served at this site; `mgmt` and `lan` are mandatory.";
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
assertions = [
|
||||
{
|
||||
assertion = cfg.vlans ? mgmt && cfg.vlans ? lan;
|
||||
message = "cnx.router: every site must define the `mgmt` and `lan` VLANs.";
|
||||
}
|
||||
{
|
||||
assertion = lib.all (v: cfg.vlans ? ${v}) (lib.attrValues cfg.accessPorts);
|
||||
message = "cnx.router: every accessPorts value must name a defined VLAN.";
|
||||
}
|
||||
{
|
||||
assertion = lib.all (p: !(cfg.accessPorts ? ${p})) cfg.trunkPorts;
|
||||
message = "cnx.router: a port cannot be both a trunk and an access port.";
|
||||
}
|
||||
];
|
||||
|
||||
# Router diagnostics toolkit: packets (tcpdump), path (mtr), link
|
||||
# negotiation (ethtool), NAT state (conntrack), DNS (kdig), per-flow
|
||||
# bandwidth (iftop), WAN throughput (librespeed-cli; iperf3 covers LAN).
|
||||
environment.systemPackages = with pkgs; [
|
||||
tcpdump
|
||||
mtr
|
||||
ethtool
|
||||
conntrack-tools
|
||||
knot-dns
|
||||
iftop
|
||||
librespeed-cli
|
||||
];
|
||||
|
||||
networking.useNetworkd = true;
|
||||
networking.useDHCP = false;
|
||||
systemd.network.enable = true;
|
||||
|
||||
systemd.network.netdevs = {
|
||||
"20-br0" = {
|
||||
netdevConfig = {
|
||||
Name = "br0";
|
||||
Kind = "bridge";
|
||||
};
|
||||
bridgeConfig.VLANFiltering = true;
|
||||
};
|
||||
}
|
||||
// lib.optionalAttrs (cfg.wan.vlanId != null) {
|
||||
"15-wan-vlan" = {
|
||||
netdevConfig = {
|
||||
Name = "wan-vlan";
|
||||
Kind = "vlan";
|
||||
};
|
||||
vlanConfig.Id = cfg.wan.vlanId;
|
||||
};
|
||||
}
|
||||
// lib.mapAttrs' (
|
||||
name: vlan:
|
||||
lib.nameValuePair "30-${vlanIf name}" {
|
||||
netdevConfig = {
|
||||
Name = vlanIf name;
|
||||
Kind = "vlan";
|
||||
};
|
||||
vlanConfig.Id = vlan.id;
|
||||
}
|
||||
) cfg.vlans;
|
||||
|
||||
systemd.network.networks =
|
||||
let
|
||||
taggedAll = lib.mapAttrsToList (_: vlan: { VLAN = vlan.id; }) cfg.vlans;
|
||||
in
|
||||
{
|
||||
# WAN port carries only the PPPoE session; no IP config of its own.
|
||||
"10-wan" = {
|
||||
matchConfig.Name = cfg.wan.interface;
|
||||
networkConfig.LinkLocalAddressing = "no";
|
||||
vlan = lib.optional (cfg.wan.vlanId != null) "wan-vlan";
|
||||
linkConfig = {
|
||||
RequiredForOnline = "carrier";
|
||||
}
|
||||
# The wan-vlan subinterface (and thus the PPPoE session) inherits
|
||||
# the parent port's MAC, so spoofing here covers both cases.
|
||||
// lib.optionalAttrs (cfg.wan.macAddress != null) {
|
||||
MACAddress = cfg.wan.macAddress;
|
||||
};
|
||||
};
|
||||
}
|
||||
// lib.optionalAttrs (cfg.wan.vlanId != null) {
|
||||
# The ISP-side VLAN subinterface pppd dials on (e.g. AIS tags PPPoE).
|
||||
"15-wan-vlan" = {
|
||||
matchConfig.Name = "wan-vlan";
|
||||
networkConfig.LinkLocalAddressing = "no";
|
||||
linkConfig.RequiredForOnline = "no";
|
||||
};
|
||||
}
|
||||
// {
|
||||
# The bridge itself is L2-only; L3 lives on the vlan-* interfaces,
|
||||
# which hang off the bridge (tagged on the bridge "self" port).
|
||||
"20-br0" = {
|
||||
matchConfig.Name = "br0";
|
||||
networkConfig.LinkLocalAddressing = "no";
|
||||
vlan = lib.mapAttrsToList (name: _: vlanIf name) cfg.vlans;
|
||||
bridgeVLANs = taggedAll;
|
||||
linkConfig.RequiredForOnline = "no";
|
||||
};
|
||||
}
|
||||
// lib.listToAttrs (
|
||||
map (port: {
|
||||
name = "25-trunk-${port}";
|
||||
value = {
|
||||
matchConfig.Name = port;
|
||||
networkConfig.Bridge = "br0";
|
||||
bridgeVLANs = taggedAll;
|
||||
linkConfig.RequiredForOnline = "no";
|
||||
};
|
||||
}) cfg.trunkPorts
|
||||
)
|
||||
// lib.mapAttrs' (
|
||||
port: vlanName:
|
||||
lib.nameValuePair "25-access-${port}" {
|
||||
matchConfig.Name = port;
|
||||
networkConfig.Bridge = "br0";
|
||||
bridgeVLANs = [
|
||||
{
|
||||
VLAN = cfg.vlans.${vlanName}.id;
|
||||
PVID = cfg.vlans.${vlanName}.id;
|
||||
EgressUntagged = cfg.vlans.${vlanName}.id;
|
||||
}
|
||||
];
|
||||
linkConfig.RequiredForOnline = "no";
|
||||
}
|
||||
) cfg.accessPorts
|
||||
// lib.mapAttrs' (
|
||||
name: vlan:
|
||||
lib.nameValuePair "40-${vlanIf name}" {
|
||||
matchConfig.Name = vlanIf name;
|
||||
address = [ "${vlan.address}/${toString vlan.prefixLength}" ];
|
||||
networkConfig = {
|
||||
IPv6AcceptRA = false;
|
||||
# Announce a /64 carved from the DHCPv6-PD prefix on ppp0 (SLAAC).
|
||||
IPv6SendRA = true;
|
||||
DHCPPrefixDelegation = true;
|
||||
};
|
||||
dhcpPrefixDelegationConfig.SubnetId = "auto";
|
||||
linkConfig.RequiredForOnline = "no";
|
||||
}
|
||||
) cfg.vlans;
|
||||
};
|
||||
}
|
||||
@@ -1,84 +0,0 @@
|
||||
# LAN DHCP (Kea) and DNS (Blocky). Fully declarative: one Kea subnet per VLAN
|
||||
# with dhcp.enable, Blocky as the blocklist resolver every DHCP lease points
|
||||
# at. Blocky's HTTP listener (:4000) serves Prometheus metrics, scraped by
|
||||
# control over the mesh (firewall.nix scopes it to the mesh subnet).
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
...
|
||||
}:
|
||||
let
|
||||
cfg = config.cnx.router;
|
||||
dhcpVlans = lib.filterAttrs (_: vlan: vlan.dhcp.enable) cfg.vlans;
|
||||
in
|
||||
{
|
||||
config = lib.mkIf cfg.enable {
|
||||
services.kea.dhcp4 = {
|
||||
enable = true;
|
||||
settings = {
|
||||
interfaces-config.interfaces = lib.mapAttrsToList (name: _: "vlan-${name}") dhcpVlans;
|
||||
lease-database = {
|
||||
type = "memfile";
|
||||
persist = true;
|
||||
name = "/var/lib/kea/dhcp4.leases";
|
||||
};
|
||||
valid-lifetime = 86400;
|
||||
subnet4 = lib.mapAttrsToList (name: vlan: {
|
||||
id = vlan.id;
|
||||
subnet = vlan.subnet;
|
||||
interface = "vlan-${name}";
|
||||
valid-lifetime = vlan.dhcp.leaseTime;
|
||||
pools = [ { pool = "${vlan.dhcp.pool.from} - ${vlan.dhcp.pool.to}"; } ];
|
||||
reservations = lib.mapAttrsToList (host: res: {
|
||||
hostname = host;
|
||||
hw-address = res.hwAddress;
|
||||
ip-address = res.ipAddress;
|
||||
}) vlan.dhcp.reservations;
|
||||
option-data = [
|
||||
{
|
||||
name = "routers";
|
||||
data = vlan.address;
|
||||
}
|
||||
{
|
||||
name = "domain-name-servers";
|
||||
data = vlan.address;
|
||||
}
|
||||
];
|
||||
}) dhcpVlans;
|
||||
};
|
||||
};
|
||||
|
||||
services.blocky = {
|
||||
enable = true;
|
||||
settings = {
|
||||
ports = {
|
||||
dns = 53;
|
||||
http = 4000;
|
||||
};
|
||||
upstreams.groups.default = [
|
||||
"9.9.9.9"
|
||||
"149.112.112.112"
|
||||
"2620:fe::fe"
|
||||
];
|
||||
blocking = {
|
||||
denylists.ads = [
|
||||
"https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts"
|
||||
];
|
||||
clientGroupsBlock.default = [ "ads" ];
|
||||
};
|
||||
caching = {
|
||||
minTime = "5m";
|
||||
prefetching = true;
|
||||
};
|
||||
prometheus.enable = true;
|
||||
};
|
||||
};
|
||||
|
||||
# The router itself resolves via public resolvers, not via Blocky, so DNS
|
||||
# for deploys/updates survives a broken local resolver.
|
||||
networking.nameservers = [
|
||||
"9.9.9.9"
|
||||
"1.1.1.1"
|
||||
];
|
||||
};
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
# Router firewall/NAT policy (nftables). Trust model:
|
||||
# mgmt VLAN -> trusted: router services, all VLANs, WAN
|
||||
# other VLANs -> DNS/DHCP on the router + WAN (if allowWan); no inter-VLAN
|
||||
# WAN (ppp0) -> nothing inbound beyond established/related
|
||||
# mesh -> admin SSH + metrics scrapes (same trust boundary as the fleet)
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
...
|
||||
}:
|
||||
let
|
||||
cfg = config.cnx.router;
|
||||
mesh = import ../mesh-hosts.nix { inherit config lib; };
|
||||
|
||||
vlanIfs = lib.mapAttrsToList (name: _: "vlan-${name}") cfg.vlans;
|
||||
wanVlanIfs = lib.mapAttrsToList (name: _: "vlan-${name}") (
|
||||
lib.filterAttrs (_: vlan: vlan.allowWan) cfg.vlans
|
||||
);
|
||||
nonMgmtIfs = lib.filter (i: i != "vlan-mgmt") vlanIfs;
|
||||
ifSet = ifs: "{ ${lib.concatStringsSep ", " (map (i: "\"${i}\"") ifs)} }";
|
||||
in
|
||||
{
|
||||
config = lib.mkIf cfg.enable {
|
||||
networking.nftables.enable = true;
|
||||
|
||||
# SSH reachable only from the mgmt VLAN (trusted) and the mesh — never
|
||||
# from the WAN or the other VLANs.
|
||||
services.openssh.openFirewall = false;
|
||||
|
||||
networking.firewall = {
|
||||
enable = true;
|
||||
filterForward = true;
|
||||
trustedInterfaces = [ "vlan-mgmt" ];
|
||||
|
||||
# Non-mgmt VLANs may only talk to the router's DNS and DHCP.
|
||||
interfaces = lib.genAttrs nonMgmtIfs (_: {
|
||||
allowedTCPPorts = [ 53 ];
|
||||
allowedUDPPorts = [
|
||||
53
|
||||
67
|
||||
];
|
||||
});
|
||||
|
||||
extraInputRules = ''
|
||||
ip6 saddr ${mesh.subnet} tcp dport 22 accept comment "admin ssh over the mesh"
|
||||
ip6 saddr ${mesh.subnet} tcp dport 4000 accept comment "blocky metrics scrape from control"
|
||||
'';
|
||||
|
||||
extraForwardRules = ''
|
||||
tcp flags syn tcp option maxseg size set rt mtu comment "MSS clamp for PPPoE mtu 1492"
|
||||
iifname "vlan-mgmt" accept comment "mgmt reaches all VLANs and the WAN"
|
||||
iifname ${ifSet wanVlanIfs} oifname "ppp0" accept comment "LAN to internet"
|
||||
'';
|
||||
};
|
||||
|
||||
networking.nat = {
|
||||
enable = true;
|
||||
externalInterface = "ppp0";
|
||||
internalInterfaces = vlanIfs;
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
# iperf3 server on every gateway, for throughput testing from any LAN segment
|
||||
# (e.g. validating AP/switch links: `iperf3 -c 10.<siteId>.<vlan>.1`) and from
|
||||
# admin machines over the mesh. Never reachable from the WAN (default-deny).
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
...
|
||||
}:
|
||||
let
|
||||
cfg = config.cnx.router;
|
||||
mesh = import ../mesh-hosts.nix { inherit config lib; };
|
||||
vlanIfs = lib.mapAttrsToList (name: _: "vlan-${name}") cfg.vlans;
|
||||
in
|
||||
{
|
||||
config = lib.mkIf cfg.enable {
|
||||
services.iperf3.enable = true;
|
||||
|
||||
networking.firewall.interfaces = lib.genAttrs vlanIfs (_: {
|
||||
allowedTCPPorts = [ 5201 ];
|
||||
allowedUDPPorts = [ 5201 ];
|
||||
});
|
||||
|
||||
networking.firewall.extraInputRules = ''
|
||||
ip6 saddr ${mesh.subnet} tcp dport 5201 accept comment "iperf3 over the mesh"
|
||||
ip6 saddr ${mesh.subnet} udp dport 5201 accept comment "iperf3 over the mesh"
|
||||
'';
|
||||
};
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
# IPv6 on the PPPoE uplink: run networkd's DHCPv6 client on ppp0 to obtain a
|
||||
# delegated prefix; each vlan-* interface (default.nix) carves a /64 out of it
|
||||
# via DHCPPrefixDelegation and announces it to clients with SLAAC.
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
...
|
||||
}:
|
||||
let
|
||||
cfg = config.cnx.router;
|
||||
in
|
||||
{
|
||||
config = lib.mkIf cfg.enable {
|
||||
systemd.network.networks."45-ppp0" = {
|
||||
matchConfig.Name = "ppp0";
|
||||
networkConfig = {
|
||||
DHCP = "ipv6";
|
||||
# pppd owns the v4 address/route on this link; don't let networkd
|
||||
# tear them down.
|
||||
KeepConfiguration = "static";
|
||||
# Default v6 route comes from the ISP's RA when they send one.
|
||||
IPv6AcceptRA = true;
|
||||
};
|
||||
# Many PPPoE ISPs never send an RA with the M flag; solicit regardless.
|
||||
dhcpV6Config.WithoutRA = "solicit";
|
||||
linkConfig.RequiredForOnline = "no";
|
||||
};
|
||||
|
||||
boot.kernel.sysctl."net.ipv6.conf.all.forwarding" = lib.mkDefault 1;
|
||||
};
|
||||
}
|
||||
@@ -1,64 +0,0 @@
|
||||
# PPPoE WAN session. ISP credentials are entered once at `clan vars generate`
|
||||
# (prompts). Both are secret — AIS often uses the same string for username and
|
||||
# password — so neither may land in the Nix store: pppd reads the username from
|
||||
# an included secret options file and the password from chap/pap-secrets.
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
...
|
||||
}:
|
||||
let
|
||||
cfg = config.cnx.router;
|
||||
creds = config.clan.core.vars.generators.pppoe-credentials;
|
||||
in
|
||||
{
|
||||
config = lib.mkIf cfg.enable {
|
||||
clan.core.vars.generators.pppoe-credentials = {
|
||||
prompts.username = {
|
||||
description = "PPPoE username (from the ISP)";
|
||||
type = "hidden";
|
||||
};
|
||||
prompts.password = {
|
||||
description = "PPPoE password (from the ISP)";
|
||||
type = "hidden";
|
||||
};
|
||||
files."user-opts".secret = true;
|
||||
files."chap-secrets".secret = true;
|
||||
script = ''
|
||||
user="$(cat "$prompts"/username)"
|
||||
pass="$(cat "$prompts"/password)"
|
||||
printf 'user "%s"\n' "$user" > "$out"/user-opts
|
||||
printf '"%s" * "%s"\n' "$user" "$pass" > "$out"/chap-secrets
|
||||
'';
|
||||
};
|
||||
|
||||
services.pppd = {
|
||||
enable = true;
|
||||
peers.wan = {
|
||||
autostart = true;
|
||||
config = ''
|
||||
plugin pppoe.so ${cfg.wan.pppInterface}
|
||||
ifname ppp0
|
||||
file ${creds.files."user-opts".path}
|
||||
noipdefault
|
||||
defaultroute
|
||||
noauth
|
||||
hide-password
|
||||
persist
|
||||
maxfail 0
|
||||
holdoff 5
|
||||
lcp-echo-interval 15
|
||||
lcp-echo-failure 3
|
||||
+ipv6
|
||||
mtu 1492
|
||||
mru 1492
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
# pppd looks up the password for `user` in these files at dial time; both
|
||||
# point at the same generated `"<user>" * "<pass>"` line (PAP and CHAP).
|
||||
environment.etc."ppp/chap-secrets".source = creds.files."chap-secrets".path;
|
||||
environment.etc."ppp/pap-secrets".source = creds.files."chap-secrets".path;
|
||||
};
|
||||
}
|
||||
@@ -1,67 +0,0 @@
|
||||
# Periodic WAN speed test so ISP degradation shows up as a trend instead of a
|
||||
# complaint. A timer runs librespeed-cli and writes the results as Prometheus
|
||||
# metrics into node_exporter's textfile collector — they ride the existing
|
||||
# 9100 scrape to VictoriaMetrics, where alerts.nix compares each run against
|
||||
# the link's own 7-day median (no per-site threshold to maintain).
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
let
|
||||
cfg = config.cnx.router;
|
||||
textfileDir = "/var/lib/speedtest";
|
||||
in
|
||||
{
|
||||
options.cnx.router.speedtest.interval = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "hourly";
|
||||
description = "systemd OnCalendar spec for the WAN speed test.";
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
services.prometheus.exporters.node.extraFlags = [
|
||||
"--collector.textfile.directory=${textfileDir}"
|
||||
];
|
||||
|
||||
systemd.services.speedtest = {
|
||||
description = "WAN speed test to Prometheus textfile metrics";
|
||||
after = [ "network-online.target" ];
|
||||
wants = [ "network-online.target" ];
|
||||
path = [
|
||||
pkgs.librespeed-cli
|
||||
pkgs.jq
|
||||
];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
StateDirectory = "speedtest";
|
||||
# One test at boot would race PPPoE and log a spurious failure.
|
||||
ExecCondition = "${pkgs.iproute2}/bin/ip link show ppp0";
|
||||
};
|
||||
script = ''
|
||||
tmp="${textfileDir}/.speedtest.prom.tmp"
|
||||
if result=$(librespeed-cli --json); then
|
||||
jq -r '.[0]
|
||||
| "speedtest_download_mbps \(.download)",
|
||||
"speedtest_upload_mbps \(.upload)",
|
||||
"speedtest_ping_ms \(.ping)",
|
||||
"speedtest_jitter_ms \(.jitter)",
|
||||
"speedtest_success 1"' <<<"$result" > "$tmp"
|
||||
else
|
||||
echo "speedtest_success 0" > "$tmp"
|
||||
fi
|
||||
mv "$tmp" "${textfileDir}/speedtest.prom"
|
||||
'';
|
||||
};
|
||||
|
||||
systemd.timers.speedtest = {
|
||||
wantedBy = [ "timers.target" ];
|
||||
timerConfig = {
|
||||
OnCalendar = cfg.speedtest.interval;
|
||||
RandomizedDelaySec = "10m";
|
||||
Persistent = true;
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -10,7 +10,10 @@
|
||||
...
|
||||
}:
|
||||
let
|
||||
mesh = import ./mesh-hosts.nix { inherit config lib; };
|
||||
mesh = import ./mesh-hosts.nix {
|
||||
dir = config.clan.core.settings.directory;
|
||||
inherit lib;
|
||||
};
|
||||
hosts = import ./hosts.nix;
|
||||
certName = "cnx.network";
|
||||
in
|
||||
|
||||
Reference in New Issue
Block a user