Files
infra/machines/stellio/configuration.nix
T
kurogeek 31ebf2d081 machines/stellio: deprecate withdrawn DHCPv6-PD prefixes (interim copy of cnx-network fa11327)
After the reboot vlan-lan got its /64 at boot, but IPv6 still failed for
clients: the first PPPoE session lands on another AIS BNG, is torn down
minutes later, and the reconnect brings a different /64 (07:53
2405:9800:b911:91e3 -> 07:57 b910:1f5d). networkd announces the new
prefix at once but never deprecates the old one (systemd#29651), so
clients keep it preferred for up to 30 minutes and source selection
sends every connection from a dead address.

router-radv-deprecate.service watches vlan-* for networkd removing a
dynamic address and sends RAs with that /64 at preferred lifetime 0 via
a short-lived radvd (SIGKILLed: no shutdown RA). Identical script to the
upstream commit; drop once the cnx-network input is bumped past it.

Verified on stellio: the stale prefix flipped to deprecated on the
client at the first RA and pings resumed; a synthetic dynamic address
removed from vlan-lan produced the same RA end to end.
2026-10-08 08:13:23 +00:00

190 lines
7.8 KiB
Nix

{
config,
inputs,
lib,
pkgs,
...
}:
{
imports = [
inputs.matthew-hardware.nixosModules.mt7986a-glinet-gl-mt6000
];
# matthew-hardware's common/generic-uefi-image.nix still sets
# `image.repart.enable = true`; nixpkgs removed that option (importing
# image/repart.nix now always defines system.build.image). Declare it so the
# definition has somewhere to land. Drop once upstream stops setting it.
options.image.repart.enable = lib.mkOption {
type = lib.types.bool;
default = true;
internal = true;
};
config = {
hardware.mt7986a-glinet-gl-mt6000.enable = true;
hardware.mt7986a-glinet-gl-mt6000.zealous = true;
hardware.mt7986a-glinet-gl-mt6000.image.repart.enable = true;
nixpkgs.hostPlatform.system = "aarch64-linux";
nixpkgs.buildPlatform.system = "x86_64-linux";
system.stateVersion = "26.11";
clan.core.sops.defaultGroups = [ "admins" ];
clan.core.settings.name = "stellio";
clan.core.settings.machine.description = "Flint-2 router";
# Root is ext4 on the Flint 2's eMMC: keep steady-state writes off it.
# The journal is persistent but bounded (256M) and synced once a minute
# (the router needs logs for pppd/kea debugging and, after the Sep 2026
# unlogged reboots, for post-mortem; volatile storage left nothing).
# /tmp is tmpfs, no core dumps, and ext4 skips atime updates and
# flushes its journal every 60 s instead of 5 s so the small periodic
# writes (Kea leases, timesyncd clock, speedtest textfile, journal)
# coalesce. A power cut loses at most a minute of that state.
fileSystems."/".options = [
"noatime"
"commit=60"
];
boot.tmp.useTmpfs = true;
services.journald.settings.Journal = {
Storage = "persistent";
SystemMaxUse = "256M";
SyncIntervalSec = "1m";
};
systemd.coredump.enable = false;
# Upstream 07d9958739 (mainline 2026-05, not in linux-6.18.y). The MT7531
# switch on the Flint 2 hangs off MDIO, and mt7530_get_stats64 took the
# MDIO mutex under the RCU read lock that /proc/net/dev holds, so any
# stats reader (zerotier-one, node exporter, `ip -s link`) that raced a
# PHY access oopsed the box: "BUG: scheduling while atomic" ->
# "Unable to handle kernel paging request" on 2026-09-20 09:02 UTC.
# Drop once boot.kernelPackages moves to a kernel that carries it.
boot.kernelPatches = [
{
name = "mt7530-get-stats64-atomic";
patch = ./mt7530-get-stats64-atomic.patch;
}
];
# nixpkgs' services.blocky check runs the aarch64 blocky in a
# system.checks derivation, which an x86_64 builder without binfmt cannot
# execute (`--max-jobs 0` -> Exec format error). Validate the same YAML
# with the build host's blocky instead. Drop once cnx-network's router
# service carries this itself (router/dns: validate the Blocky config
# with the build host's binary).
services.blocky.enableConfigCheck = false;
system.checks = [
(pkgs.runCommand "check-blocky-config" { } ''
${lib.getExe pkgs.buildPackages.blocky} --config ${
(pkgs.formats.yaml { }).generate "blocky-config.yaml" config.services.blocky.settings
} validate && touch $out
'')
];
# The router service opens SSH only to the mgmt VLAN, the staging port
# and the ZeroTier mesh (inventories/router.nix). Also admit it over
# Yggdrasil (fleet-wide instance in inventories/default.nix) so the box
# stays reachable when ZeroTier's controller/relays are down. Scoped to
# the tunnel interface (clan-core's yggdrasil service names it `ygg`):
# only decrypted 200::/7 traffic arrives there, so a spoofed source on
# the WAN never matches.
networking.firewall.interfaces.${config.services.yggdrasil.settings.IfName}.allowedTCPPorts = [
22
];
# AIS delegates a single /64 over PPPoE (seen 2026-10-02: "ppp0: DHCP:
# received delegated prefix 2405:9800:b910:1f5d::/64"). The router
# service hands out sub-prefixes with SubnetId=auto, so whichever VLAN
# interface comes up first takes the whole /64 -- on this box vlan-mgmt
# did, and vlan-lan logged "Couldn't find a suitable prefix. Ran out of
# address space": no RA prefix on lan, so clients had no global IPv6.
# Pin the ids: lan (where the clients are) always gets subnet 0, i.e.
# the entire prefix when it is a /64; mgmt asks for subnet 1, which
# fails harmlessly on a /64 (mgmt is reached over v4 and the meshes) and
# starts working unchanged if AIS ever delegates wider. Drop once
# cnx-network's router service exposes a per-VLAN subnet id.
systemd.network.networks."40-vlan-lan".dhcpPrefixDelegationConfig.SubnetId = lib.mkForce "0x0";
systemd.network.networks."40-vlan-mgmt".dhcpPrefixDelegationConfig.SubnetId = lib.mkForce "0x1";
# Interim copy of cnx-network-clan fa11327 (router/ipv6: deprecate
# withdrawn DHCPv6-PD prefixes); drop when the input is bumped past it.
# networkd never deprecates a prefix it stops announcing (systemd#29651):
# on every reboot the first PPPoE session lands on another AIS BNG, is
# torn down minutes later, and the reconnect brings a different /64.
# networkd announces the new prefix at once but clients keep the old
# one preferred for the rest of its 1800 s lifetime, so RFC 6724 source
# selection picks the dead prefix and every v6 connection fails for up
# to 30 minutes (2026-10-08 07:57). Watch vlan-* for networkd removing a
# dynamic (PD) address and send a few RAs with that /64 at preferred
# lifetime 0; radvd is only the packet generator, killed hard after ~7 s
# so it sends no shutdown RA (router lifetime 0) and does not keep
# deprecating a prefix the ISP re-delegates on reconnect.
systemd.services.router-radv-deprecate = {
description = "Deprecate withdrawn DHCPv6-PD prefixes on the VLANs";
wantedBy = [ "multi-user.target" ];
after = [ "systemd-networkd.service" ];
path = [
pkgs.iproute2
pkgs.ipcalc
pkgs.radvd
];
serviceConfig = {
Restart = "always";
RestartSec = 2;
RuntimeDirectory = "router-radv-deprecate";
};
script = ''
conf=$RUNTIME_DIRECTORY/radvd.conf
pid=$RUNTIME_DIRECTORY/radvd.pid
# "Deleted 4: vlan-lan inet6 2001:db8::1/64 scope global dynamic mngtmpaddr \ valid_lft ..."
ip -o -6 monitor address | while read -r event _ dev _ addr _ scope rest; do
[[ $event == Deleted && $dev == vlan-* && $scope == global && $rest == *dynamic* ]] || continue
prefix="$(ipcalc --no-decorate -n "$addr")/''${addr##*/}"
echo "$dev: deprecating withdrawn prefix $prefix"
cat > "$conf" <<EOF
interface $dev {
AdvSendAdvert on;
MinRtrAdvInterval 3;
MaxRtrAdvInterval 4;
AdvDefaultLifetime 1800;
AdvCurHopLimit 0;
prefix $prefix {
AdvOnLink on;
AdvAutonomous on;
AdvPreferredLifetime 0;
AdvValidLifetime 7200;
};
};
EOF
rm -f "$pid"
radvd -C "$conf" -p "$pid" -n -m stderr &
sleep 7
kill -KILL $! 2>/dev/null || true
wait $! || true
done
'';
};
systemd.services.heartbeat-push = {
description = "Heartbeat push to uptime monitor";
serviceConfig = {
Type = "oneshot";
ExecStart = "${pkgs.curl}/bin/curl -s -o /dev/null https://uptime.b4l.co.th/api/push/X0WCHAcY5gPf1U8If7BT1FLjpacZqGZu?status=up&msg=OK&ping=";
};
};
systemd.timers.heartbeat-push = {
description = "Heartbeat push timer";
wantedBy = [ "timers.target" ];
timerConfig = {
OnBootSec = "60s";
OnUnitActiveSec = "60s";
Unit = "heartbeat-push.service";
};
};
};
}