feat(collector): forecast and ntfy in ping-collector; ThaiWater keeps alerts current at night

Phase 2b/2c of docs/RUST_REWRITE.md. With FORECAST_IN_COLLECTOR=true (shared
.env) ping-collector, after each cycle that brought newer RID or HII readings
and at least hourly: loads the last 336 h (RID + HII mirror fill), fetches the
rain forecast, runs every exported model (rust/forecast), stores
forecast_history and forecast_current, and sends the ntfy notifications.
The Python API then serves forecast_current as /forecast (computing in-process
only when it is older than FORECAST_MAX_AGE_SECONDS) and neither forecasts nor
notifies.

ntfy (collector/src/notify.rs): a port of src/notify.py with the same topics,
texts, hysteresis, capacity guard and notification_state keys, checked word
for word on 751 messages generated by Python (testdata/gen_notify_cases.py).
The change: for stations whose ThaiWater twin is the same sensor, a reading
newer than RID's drives the level alerts (marked as ThaiWater telemetry), and
the feed counts as stale only when neither source has a reading for 3 h. RID
alone lagging is reported once after RID_STALE_ALERT_HOURS (18). This ends the
nightly "gauge feed stale" / "recovered" pairs caused by RID's late reports.

Commands: `forecast [--json] [--save] [--now] [--rain-file]`,
`notify --dry-run`. scripts/forecast_parity.py compares Python and the
collector on the live database (same moment, same rain); locally 48/48 rows
identical. install_collector.sh: models export + that check on every install,
--cutover-forecast (refuses on a failed check) and --rollback-forecast.

Trees are packed into 24-byte nodes: 70 MB peak with all 15 models (was 140).
Forecast values are written to NUMERIC as shortest round-trip text, so they
round as psycopg2's do. DB test schemas get a counter (parallel tests collided
on Windows' coarse clock).

scripts/forecast_parity.py

@@ -0,0 +1,178 @@
+#!/usr/bin/env python3
+"""Compare the Python forecast with ping-collector's on the live database.
+
+Run before setting FORECAST_IN_COLLECTOR=true, and after any change to src/ml
+or rust/forecast. On the server, as the service user from the app dir:
+
+    sudo -u water-monitor .venv/bin/python scripts/forecast_parity.py
+
+Both sides get the same moment and the same Open-Meteo rain series (fetched
+once here and handed to the collector as a file), so a difference is a porting
+bug, not timing. Exit 0 when every row matches: station, horizon, as_of,
+source, model and every level exactly; probabilities within a few units in the
+last place (numpy's and libm's exp may differ in the last bit).
+"""
+
+import argparse
+import datetime
+import json
+import math
+import os
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+REPO = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(REPO))
+
+ULPS = 4
+EXACT = (
+    "station_code",
+    "horizon_hours",
+    "as_of",
+    "source",
+    "model_version",
+    "trained_at",
+    "current_level",
+    "predicted_max_level",
+    "threshold_warning",
+    "threshold_danger",
+)
+
+
+def ulps(a: float, b: float) -> float:
+    if a == b:
+        return 0
+    return abs(a - b) / math.ulp(max(abs(a), abs(b)))
+
+
+def compare(py: list, rs: list) -> list:
+    problems = []
+    if len(py) != len(rs):
+        problems.append(f"row count: python {len(py)}, collector {len(rs)}")
+    for p, r in zip(py, rs):
+        where = f"{p.get('station_code')} {p.get('horizon_hours')}h"
+        for k in EXACT:
+            if p.get(k) != r.get(k):
+                problems.append(
+                    f"{where} {k}: python {p.get(k)!r}, collector {r.get(k)!r}"
+                )
+        for k in ("p_warning", "p_danger"):
+            d = ulps(float(p[k]), float(r[k]))
+            if d > ULPS:
+                problems.append(
+                    f"{where} {k}: python {p[k]!r}, collector {r[k]!r} ({d:.0f} ulp)"
+                )
+        ps, rs_ = p.get("stages"), r.get("stages")
+        if (ps is None) != (rs_ is None) or (ps and len(ps) != len(rs_)):
+            problems.append(f"{where} stages differ")
+        elif ps:
+            for a, b in zip(ps, rs_):
+                if a["stage"] != b["stage"] or a["level"] != b["level"]:
+                    problems.append(f"{where} stage {a['stage']} differs")
+                elif ulps(a["p_exceed"], b["p_exceed"]) > ULPS:
+                    problems.append(
+                        f"{where} stage {a['stage']} p_exceed {a['p_exceed']} vs {b['p_exceed']}"
+                    )
+    return problems
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+    parser.add_argument("--collector", default=str(REPO / "bin" / "ping-collector"))
+    parser.add_argument("--models-dir", default=str(REPO / "models"))
+    parser.add_argument(
+        "--db-url", default=None, help="default: the app's configuration"
+    )
+    parser.add_argument("--now", default=None, help="local time, default the clock")
+    args = parser.parse_args()
+
+    from src.ml import predict
+    from src.ml import rain as rain_mod
+    from src.ml.export import json_path_for
+
+    models = Path(args.models_dir)
+    stale = [
+        p.name
+        for p in models.glob("flood_*.joblib")
+        if not json_path_for(p).exists()
+        or json_path_for(p).stat().st_mtime < p.stat().st_mtime
+    ]
+    if stale:
+        print(
+            f"not exported for the collector: {', '.join(stale)}; run `python -m src.ml.export`"
+        )
+        return 2
+
+    now = (
+        datetime.datetime.fromisoformat(args.now)
+        if args.now
+        else datetime.datetime.now()
+    )
+    rain = rain_mod.serving_series()
+    if rain is None:
+        import pandas as pd
+
+        rain = pd.Series(dtype=float)
+    py = predict.get_latest_forecasts(
+        db_url=args.db_url, models_dir=models, now=now, rain=rain
+    )
+    py = json.loads(json.dumps(py, default=float))
+
+    with tempfile.TemporaryDirectory() as tmp:
+        rain_file = Path(tmp) / "rain.json"
+        rain_file.write_text(
+            json.dumps(
+                {
+                    "timestamps": [t.strftime("%Y-%m-%dT%H:%M:%S") for t in rain.index],
+                    "values": [None if math.isnan(v) else float(v) for v in rain],
+                }
+            )
+        )
+        env = dict(os.environ, MODELS_DIR=str(models), RUST_LOG="error")
+        if args.db_url:
+            env["POSTGRES_CONNECTION_STRING"] = args.db_url
+        out = subprocess.run(
+            [
+                args.collector,
+                "forecast",
+                "--json",
+                "--now",
+                now.isoformat(),
+                "--rain-file",
+                str(rain_file),
+            ],
+            capture_output=True,
+            text=True,
+            env=env,
+            cwd=REPO,
+            check=False,
+        )
+    if out.returncode != 0:
+        print(f"ping-collector forecast failed:\n{out.stderr}")
+        return 2
+    rs = json.loads(out.stdout)
+
+    problems = compare(py, rs)
+    models_used = sorted({r["model_version"] for r in py})
+    n_model = sum(r["source"] == "model" for r in py)
+    print(
+        f"as of {now:%Y-%m-%d %H:%M:%S}: {len(py)} rows ({n_model} from models {models_used}), "
+        f"rain {int(rain.notna().sum())} h"
+    )
+    if problems:
+        print(f"{len(problems)} DIFFERENCES:")
+        for p in problems[:40]:
+            print("  " + p)
+        return 1
+    exact = sum(p[k] == r[k] for p, r in zip(py, rs) for k in ("p_warning", "p_danger"))
+    print(
+        f"identical: every level and field; probabilities {exact}/{2 * len(py)} bit-identical, "
+        f"the rest within {ULPS} ulp"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())