feat(collector): forecast and ntfy in ping-collector; ThaiWater keeps alerts current at night

Phase 2b/2c of docs/RUST_REWRITE.md. With FORECAST_IN_COLLECTOR=true (shared
.env) ping-collector, after each cycle that brought newer RID or HII readings
and at least hourly: loads the last 336 h (RID + HII mirror fill), fetches the
rain forecast, runs every exported model (rust/forecast), stores
forecast_history and forecast_current, and sends the ntfy notifications.
The Python API then serves forecast_current as /forecast (computing in-process
only when it is older than FORECAST_MAX_AGE_SECONDS) and neither forecasts nor
notifies.

ntfy (collector/src/notify.rs): a port of src/notify.py with the same topics,
texts, hysteresis, capacity guard and notification_state keys, checked word
for word on 751 messages generated by Python (testdata/gen_notify_cases.py).
The change: for stations whose ThaiWater twin is the same sensor, a reading
newer than RID's drives the level alerts (marked as ThaiWater telemetry), and
the feed counts as stale only when neither source has a reading for 3 h. RID
alone lagging is reported once after RID_STALE_ALERT_HOURS (18). This ends the
nightly "gauge feed stale" / "recovered" pairs caused by RID's late reports.

Commands: `forecast [--json] [--save] [--now] [--rain-file]`,
`notify --dry-run`. scripts/forecast_parity.py compares Python and the
collector on the live database (same moment, same rain); locally 48/48 rows
identical. install_collector.sh: models export + that check on every install,
--cutover-forecast (refuses on a failed check) and --rollback-forecast.

Trees are packed into 24-byte nodes: 70 MB peak with all 15 models (was 140).
Forecast values are written to NUMERIC as shortest round-trip text, so they
round as psycopg2's do. DB test schemas get a counter (parallel tests collided
on Windows' coarse clock).

scripts/install_collector.sh

@@ -1,12 +1,18 @@
 #!/usr/bin/env bash
-# Install the Rust collector (rust/collector, `ping-collector`) and switch
-# data collection over to it, or back.
+# Install the Rust collector (rust/collector, `ping-collector`) and move work
+# from the Python app to it, or back.
 #
-#   sudo bash scripts/install_collector.sh             # install binary + unit, dry-run check
-#   sudo bash scripts/install_collector.sh --cutover   # install, then Rust collects:
-#                                                      #   COLLECTION_ENABLED=false, start
-#                                                      #   ping-collector, restart water-monitor
-#   sudo bash scripts/install_collector.sh --rollback  # Python collects again
+#   sudo bash scripts/install_collector.sh                     # install binary + unit; checks only
+#   sudo bash scripts/install_collector.sh --cutover           # Rust collects (COLLECTION_ENABLED=false)
+#   sudo bash scripts/install_collector.sh --cutover-forecast  # Rust also forecasts and sends the
+#                                                              # ntfy notifications (FORECAST_IN_COLLECTOR=true)
+#   sudo bash scripts/install_collector.sh --rollback-forecast # forecasts + notifications back to Python
+#   sudo bash scripts/install_collector.sh --rollback          # Python does everything again
+#
+# Every install also exports the trained models for the Rust forecaster and
+# compares its forecast with Python's on the live data
+# (scripts/forecast_parity.py); --cutover-forecast refuses to switch when they
+# differ (FORCE=1 overrides).
 #
 # The binary comes from BIN=/path/to/ping-collector (e.g. the "ping-collector-
 # linux-x86_64" CI artifact) or from rust/target/release/ping-collector, built
@@ -20,10 +26,12 @@ APP_DIR="${APP_DIR:-/opt/thailand-water-monitor}"
 SERVICE_USER="${SERVICE_USER:-water-monitor}" # as in scripts/ping-collector.service
 ENV_FILE="${APP_DIR}/.env"
 STATUS_FILE="${APP_DIR}/collector-status.json"
+PYTHON="${APP_DIR}/.venv/bin/python"
 REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 MODE="${1:-install}"
 
 log() { printf '\033[1;32m==>\033[0m %s\n' "$*"; }
+warn() { printf '\033[1;33m[warn]\033[0m %s\n' "$*" >&2; }
 die() { printf '\033[1;31m[error]\033[0m %s\n' "$*" >&2; exit 1; }
 
 [ "$(id -u)" -eq 0 ] || die "run as root (sudo)"
@@ -38,15 +46,44 @@ set_env() {
     fi
 }
 
-if [ "$MODE" = "--rollback" ]; then
-    log "Stopping ping-collector; the Python app collects again"
-    systemctl disable --now ping-collector.service 2>/dev/null || true
-    set_env COLLECTION_ENABLED true
-    systemctl restart water-monitor.service
-    log "Done. Check: journalctl -u water-monitor -n 50"
-    exit 0
-fi
-case "$MODE" in install | --cutover) ;; *) die "unknown option: $MODE" ;; esac
+# Run a command the way the services run: same user, working directory and
+# EnvironmentFile parsing (systemd's, not a .env library's).
+as_service() {
+    if systemd-run --version >/dev/null 2>&1; then
+        systemd-run --quiet --pipe --wait --collect \
+            -p User="$SERVICE_USER" -p Group="$(id -gn "$SERVICE_USER")" \
+            -p WorkingDirectory="$APP_DIR" -p EnvironmentFile="$ENV_FILE" "$@"
+    else
+        (cd "$APP_DIR" && runuser -u "$SERVICE_USER" -- "$@")
+    fi
+}
+
+show() {
+    sleep "$1"
+    systemctl --no-pager --lines=0 status ping-collector.service || true
+    journalctl -u ping-collector -n 25 --no-pager || true
+}
+
+case "$MODE" in
+    --rollback)
+        log "Stopping ping-collector; the Python app collects, forecasts and notifies again"
+        systemctl disable --now ping-collector.service 2>/dev/null || true
+        set_env FORECAST_IN_COLLECTOR false
+        set_env COLLECTION_ENABLED true
+        systemctl restart water-monitor.service
+        log "Done. Check: journalctl -u water-monitor -n 50"
+        exit 0
+        ;;
+    --rollback-forecast)
+        log "Forecasts and notifications back to the Python app; ping-collector keeps collecting"
+        set_env FORECAST_IN_COLLECTOR false
+        systemctl restart ping-collector.service water-monitor.service
+        log "Done. Check: journalctl -u water-monitor -n 50"
+        exit 0
+        ;;
+    install | --cutover | --cutover-forecast) ;;
+    *) die "unknown option: $MODE" ;;
+esac
 
 BIN="${BIN:-${REPO_DIR}/rust/target/release/ping-collector}"
 [ -x "$BIN" ] || die "no binary at $BIN: build it (see header) or pass BIN=/path/to/ping-collector"
@@ -63,26 +100,47 @@ touch "$STATUS_FILE"
 chown "${SERVICE_USER}:$(id -gn "$SERVICE_USER")" "$STATUS_FILE"
 
 log "Dry run against the configured database (writes nothing)"
-# Run it the way the service will: same user, working directory and
-# EnvironmentFile parsing (systemd's, not a .env library's).
-if systemd-run --version >/dev/null 2>&1; then
-    systemd-run --quiet --pipe --wait --collect \
-        -p User="$SERVICE_USER" -p Group="$(id -gn "$SERVICE_USER")" \
-        -p WorkingDirectory="$APP_DIR" -p EnvironmentFile="$ENV_FILE" \
-        "${APP_DIR}/bin/ping-collector" once --dry-run
+as_service "${APP_DIR}/bin/ping-collector" once --dry-run
+
+forecast_ok=1
+if compgen -G "${APP_DIR}/models/flood_*.joblib" >/dev/null; then
+    log "Exporting the trained models for the Rust forecaster (models/flood_*.json.gz)"
+    as_service "$PYTHON" -m src.ml.export
+    log "Forecast check: Python and ping-collector on the same data, moment and rain"
+    as_service "$PYTHON" scripts/forecast_parity.py || forecast_ok=0
 else
-    (cd "$APP_DIR" && "${APP_DIR}/bin/ping-collector" --env-file "$ENV_FILE" once --dry-run)
+    warn "no trained models in ${APP_DIR}/models: forecast check skipped"
+    forecast_ok=0
 fi
 
-if [ "$MODE" = "--cutover" ]; then
-    log "Cutover: ping-collector collects, the Python app forecasts and notifies"
-    set_env COLLECTION_ENABLED false
-    systemctl enable --now ping-collector.service
-    systemctl restart water-monitor.service
-    sleep 20
-    systemctl --no-pager --lines=0 status ping-collector.service || true
-    journalctl -u ping-collector -n 20 --no-pager || true
-    log "Rollback if needed: sudo bash scripts/install_collector.sh --rollback"
-else
-    log "Installed, not started. Switch over with: sudo bash scripts/install_collector.sh --cutover"
-fi
+case "$MODE" in
+    --cutover)
+        log "Cutover: ping-collector collects; the Python app forecasts and notifies"
+        set_env COLLECTION_ENABLED false
+        systemctl enable ping-collector.service
+        systemctl restart ping-collector.service water-monitor.service
+        show 20
+        log "Next step: --cutover-forecast. Rollback: sudo bash scripts/install_collector.sh --rollback"
+        ;;
+    --cutover-forecast)
+        grep -q '^COLLECTION_ENABLED=false' "$ENV_FILE" ||
+            die "ping-collector must collect first: run --cutover"
+        if [ "$forecast_ok" != 1 ] && [ "${FORCE:-0}" != 1 ]; then
+            die "the forecast check did not pass; not switching (FORCE=1 overrides)"
+        fi
+        log "Cutover: ping-collector also forecasts and notifies; the Python app serves its forecast"
+        set_env FORECAST_IN_COLLECTOR true
+        systemctl enable ping-collector.service
+        systemctl restart ping-collector.service water-monitor.service
+        show 30
+        log "Rollback if needed: sudo bash scripts/install_collector.sh --rollback-forecast"
+        ;;
+    *)
+        if [ "$(systemctl is-active ping-collector.service 2>/dev/null)" = active ]; then
+            systemctl restart ping-collector.service
+            log "Installed; ping-collector restarted on the new binary"
+        else
+            log "Installed, not started. Switch over with: sudo bash scripts/install_collector.sh --cutover"
+        fi
+        ;;
+esac