From 5066d4ee46515d38ce4955553f3eeef97b2ea0b8 Mon Sep 17 00:00:00 2001 From: kurogeek Date: Tue, 29 Sep 2026 07:46:41 +0000 Subject: [PATCH] phonebox: arm hardware watchdog and auto-reboot on panic mirach and almach have each hung at least twice since July with nothing in the journal or pstore, and stayed down until someone power-cycled the board. RuntimeWatchdogSec was 0 and kernel.panic was 0, so a hung kernel/PID1 or a panic could never self-recover. Arm the systemd hardware watchdog (30s) and reboot 10s after a panic on all phonebox nodes. Watchdog resets will show up as extra entries in journalctl --list-boots. --- modules/clan/phonebox/default.nix | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/modules/clan/phonebox/default.nix b/modules/clan/phonebox/default.nix index 462910fa..6cde7801 100644 --- a/modules/clan/phonebox/default.nix +++ b/modules/clan/phonebox/default.nix @@ -256,6 +256,13 @@ ''; }; + # Phonebox nodes are headless and unattended. Observed on mirach/almach + # (2026-07..09): hard hangs leave no journal/pstore trace and stay down + # until someone power-cycles the board. Arm the hardware watchdog so a + # hung kernel/PID1 resets itself, and reboot instead of halting on panic. + systemd.settings.Manager.RuntimeWatchdogSec = "30s"; + boot.kernel.sysctl."kernel.panic" = 10; + networking.interfaces = { ${ata-interface} = { useDHCP = false;