grabowski
7e642555b4
Phase 4 of docs/RUST_REWRITE.md. Since 2026-10-18 ping-collector (rust/) collects, forecasts, notifies (ntfy + Matrix) and serves the website/API; nothing in production ran the Python service code any more. Removed (~17k lines): the FastAPI app (web_api, schemas, health_check, metrics, postgres_history, forecast_history, thaiwater), the Python RID collector and CLI (water_scraper_v3, main, run.py), the notifier and Matrix alerting (notify, alerting), the multi-backend adapters (SQLite, MySQL, InfluxDB, VictoriaMetrics) with demo databases, models/validators/exceptions/ logging/rate limiter, ml/skill.py (served by Rust now), the migration tooling (api_parity, collector_parity, e2e_notify, install_collector.sh, gen_notify_cases.py, spikes/rust-forecast), Docker/compose/Grafana files, sql/, the release workflow and the docs for all of that. Their tests went with them. Kept in Python: src/ml (training, evaluation harness, export; predict.py as the reference the Rust forecaster is tested against), the HII and RID-dam clients/stores for the archive backfills (their live collectors removed), the backfill scripts, retrain.sh + timer, forecast_parity.py. - PostgreSQL only: src/config.py is database_url() from POSTGRES_* / .env; the stores lost their sqlite/mysql branches (and the 2026-08 column migration). The database tests run against a throwaway schema from TEST_DATABASE_URL (tests/conftest.py pg_url, skipped without it), as the Rust tests do; CI gets a PostgreSQL service. 100 tests (66 without a DB). - The notifier settings moved from src/notify.py into src/ml/export.py; flood_config.json regenerates byte-identical. - tests/test_dashboard.py checks every path the dashboard fetches against ping-collector's router. - Dependencies: fastapi, uvicorn, pydantic, influxdb, pymysql, psutil, schedule, sphinx, pyinstaller, mypy, pytest-asyncio dropped; python-dotenv 1.1.1 explicit (it came in through uvicorn[standard]). - Formatting contract extended to scripts/ and tests/ (reformatted once). - scripts/install.sh is the one installer (fresh host or update, in place in a checkout without touching its ownership): binary, units, uv sync, .env defaults, removes the retired water-monitor.service and any old --alert-check cron job, dry run + forecast check, restart, readiness wait. - Docs, CLAUDE.md, CONTRIBUTING.md, .env.example, Makefile, unit comments and Rust doc comments updated to the Rust service.
76 lines
3.2 KiB
Python
76 lines
3.2 KiB
Python
"""Summarise rolling-origin harness output side by side.
|
|
|
|
Usage:
|
|
uv run python scripts/summarize_eval.py models/eval_2026-09-12.json [more.json ...]
|
|
|
|
Aggregates each (station, variant) across folds: mean MAE, mean flood-regime
|
|
MAE, mean Brier, total false-alarm episodes, and every warning event with its
|
|
first-alert lead and the 24 h-ahead peak error -- the operational numbers that
|
|
decide whether a variant ships.
|
|
"""
|
|
|
|
import json
|
|
import statistics
|
|
import sys
|
|
from collections import OrderedDict
|
|
|
|
|
|
def summarize(paths):
|
|
for path in paths:
|
|
results = json.load(open(path, encoding="utf-8"))
|
|
print(f"\n##### {path}")
|
|
for station in results:
|
|
print(f"\n=== {station['station']} (warn {station['warn_thr']:.2f} m) ===")
|
|
agg = OrderedDict()
|
|
for fold in station["folds"]:
|
|
for name, m in fold["variants"].items():
|
|
a = agg.setdefault(
|
|
name,
|
|
{"mae": [], "mae_hi": [], "brier": [], "fa": 0, "events": []},
|
|
)
|
|
a["mae"].append(m["mae"])
|
|
if m.get("mae_above_2p5") is not None:
|
|
a["mae_hi"].append(m["mae_above_2p5"])
|
|
if m.get("brier_warn") is not None:
|
|
a["brier"].append(m["brier_warn"])
|
|
a["fa"] += m["false_alarm_episodes"]
|
|
for e in m["events"]:
|
|
err = (
|
|
None
|
|
if e["peak_pred_24h_before"] is None
|
|
else e["peak_pred_24h_before"] - e["peak_level"]
|
|
)
|
|
a["events"].append(
|
|
(
|
|
fold["year"],
|
|
e["crossing"][:10],
|
|
e["lead_h"],
|
|
e["peak_level"],
|
|
err,
|
|
)
|
|
)
|
|
print(
|
|
f"{'variant':22} {'MAE':>6} {'MAE_hi':>7} {'Brier':>7} {'FA':>3} events: year crossing lead_h peak(err24h)"
|
|
)
|
|
for name, a in agg.items():
|
|
ev = " ".join(
|
|
f"{y} {d} {'—' if l is None else format(l, '+.0f')}h {p:.2f}({'—' if err is None else format(err, '+.2f')})"
|
|
for y, d, l, p, err in a["events"]
|
|
)
|
|
leads = [l for *_, l, _, _ in a["events"] if l is not None]
|
|
print(
|
|
f"{name:22} {statistics.mean(a['mae']):6.3f} "
|
|
f"{statistics.mean(a['mae_hi']) if a['mae_hi'] else float('nan'):7.3f} "
|
|
f"{statistics.mean(a['brier']) if a['brier'] else float('nan'):7.4f} "
|
|
f"{a['fa']:>3} {ev}"
|
|
)
|
|
if leads:
|
|
print(
|
|
f"{'':22} lead: mean {statistics.mean(leads):+.1f} h, min {min(leads):+.0f} h, "
|
|
f"missed {sum(1 for *_, l, _, _ in a['events'] if l is None)}/{len(a['events'])}"
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
summarize(sys.argv[1:] or ["models/eval_variants.json"])
|