feat(collector): forecast and ntfy in ping-collector; ThaiWater keeps alerts current at night
CI / Format & lint (push) Successful in 22s
CI / Test suite (push) Successful in 59s
Docs / Validate documentation (push) Successful in 25s
Rust / fmt, clippy, test, build (push) Successful in 10m2s
Security / Dependency vulnerabilities (push) Successful in 2m11s
Security / Static analysis (push) Successful in 22s
Security / License report (push) Successful in 1m37s
CI / Format & lint (push) Successful in 22s
CI / Test suite (push) Successful in 59s
Docs / Validate documentation (push) Successful in 25s
Rust / fmt, clippy, test, build (push) Successful in 10m2s
Security / Dependency vulnerabilities (push) Successful in 2m11s
Security / Static analysis (push) Successful in 22s
Security / License report (push) Successful in 1m37s
Phase 2b/2c of docs/RUST_REWRITE.md. With FORECAST_IN_COLLECTOR=true (shared .env) ping-collector, after each cycle that brought newer RID or HII readings and at least hourly: loads the last 336 h (RID + HII mirror fill), fetches the rain forecast, runs every exported model (rust/forecast), stores forecast_history and forecast_current, and sends the ntfy notifications. The Python API then serves forecast_current as /forecast (computing in-process only when it is older than FORECAST_MAX_AGE_SECONDS) and neither forecasts nor notifies. ntfy (collector/src/notify.rs): a port of src/notify.py with the same topics, texts, hysteresis, capacity guard and notification_state keys, checked word for word on 751 messages generated by Python (testdata/gen_notify_cases.py). The change: for stations whose ThaiWater twin is the same sensor, a reading newer than RID's drives the level alerts (marked as ThaiWater telemetry), and the feed counts as stale only when neither source has a reading for 3 h. RID alone lagging is reported once after RID_STALE_ALERT_HOURS (18). This ends the nightly "gauge feed stale" / "recovered" pairs caused by RID's late reports. Commands: `forecast [--json] [--save] [--now] [--rain-file]`, `notify --dry-run`. scripts/forecast_parity.py compares Python and the collector on the live database (same moment, same rain); locally 48/48 rows identical. install_collector.sh: models export + that check on every install, --cutover-forecast (refuses on a failed check) and --rollback-forecast. Trees are packed into 24-byte nodes: 70 MB peak with all 15 models (was 140). Forecast values are written to NUMERIC as shortest round-trip text, so they round as psycopg2's do. DB test schemas get a counter (parallel tests collided on Windows' coarse clock).
This commit is contained in:
1 parent
5c4f464383
commit
6ae67dec63
27 files changed
+2656
-118
No files matched your search
@@ -0,0 +1,107 @@
|
||||
"""FORECAST_IN_COLLECTOR=true: ping-collector forecasts and notifies; the API
|
||||
serves its stored forecast and falls back to computing when that is stale."""
|
||||
|
||||
import asyncio
|
||||
import datetime
|
||||
import json
|
||||
import types
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import create_engine, text
|
||||
|
||||
from src import web_api
|
||||
from src.config import Config
|
||||
|
||||
ROWS = [
|
||||
{
|
||||
"station_code": "P.1",
|
||||
"horizon_hours": 24,
|
||||
"p_warning": 0.012,
|
||||
"p_danger": 0.001,
|
||||
"predicted_max_level": 1.31,
|
||||
"current_level": 1.23,
|
||||
"as_of": "2026-10-18T13:00:00",
|
||||
"model_version": "hgb-v3+abc",
|
||||
"trained_at": "2026-10-01T03:42:00",
|
||||
"source": "model",
|
||||
"threshold_warning": 3.7,
|
||||
"threshold_danger": 4.2,
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def collector_db(tmp_path, monkeypatch):
|
||||
engine = create_engine(f"sqlite:///{tmp_path / 'db.sqlite'}")
|
||||
with engine.begin() as conn:
|
||||
conn.execute(
|
||||
text(
|
||||
"CREATE TABLE forecast_current (id VARCHAR(16) PRIMARY KEY, "
|
||||
"computed_at TIMESTAMP NOT NULL, model_versions TEXT, payload TEXT NOT NULL)"
|
||||
)
|
||||
)
|
||||
scraper = types.SimpleNamespace(db_adapter=types.SimpleNamespace(engine=engine))
|
||||
monkeypatch.setitem(web_api.app_state, "scraper", scraper)
|
||||
monkeypatch.setattr(Config, "FORECAST_IN_COLLECTOR", True)
|
||||
monkeypatch.setattr(web_api, "FORECAST_CACHE", {})
|
||||
|
||||
def store(age_minutes):
|
||||
computed = datetime.datetime.now() - datetime.timedelta(minutes=age_minutes)
|
||||
with engine.begin() as conn:
|
||||
conn.execute(text("DELETE FROM forecast_current"))
|
||||
conn.execute(
|
||||
text(
|
||||
"INSERT INTO forecast_current VALUES ('all', :t, 'hgb-v3+abc', :p)"
|
||||
),
|
||||
{"t": computed, "p": json.dumps(ROWS)},
|
||||
)
|
||||
|
||||
return store
|
||||
|
||||
|
||||
def test_serves_the_collectors_forecast(collector_db, monkeypatch):
|
||||
collector_db(age_minutes=20)
|
||||
|
||||
def must_not_compute():
|
||||
raise AssertionError("computed in-process although a fresh forecast is stored")
|
||||
|
||||
monkeypatch.setattr("src.ml.predict.get_latest_forecasts", must_not_compute)
|
||||
assert asyncio.run(web_api.get_flood_forecasts()) == ROWS
|
||||
# served from the short cache on the next request
|
||||
assert "stored" in web_api.FORECAST_CACHE
|
||||
|
||||
|
||||
def test_stale_stored_forecast_falls_back_to_computing(collector_db, monkeypatch):
|
||||
collector_db(age_minutes=Config.FORECAST_MAX_AGE_SECONDS / 60 + 5)
|
||||
computed = [dict(ROWS[0], source="heuristic")]
|
||||
monkeypatch.setattr("src.ml.predict.get_latest_forecasts", lambda: computed)
|
||||
assert asyncio.run(web_api.get_flood_forecasts()) == computed
|
||||
|
||||
|
||||
def test_missing_table_falls_back_to_computing(tmp_path, monkeypatch):
|
||||
engine = create_engine(f"sqlite:///{tmp_path / 'empty.sqlite'}")
|
||||
scraper = types.SimpleNamespace(db_adapter=types.SimpleNamespace(engine=engine))
|
||||
monkeypatch.setitem(web_api.app_state, "scraper", scraper)
|
||||
monkeypatch.setattr(Config, "FORECAST_IN_COLLECTOR", True)
|
||||
monkeypatch.setattr(web_api, "FORECAST_CACHE", {})
|
||||
monkeypatch.setattr("src.ml.predict.get_latest_forecasts", lambda: ROWS)
|
||||
assert asyncio.run(web_api.get_flood_forecasts()) == ROWS
|
||||
|
||||
|
||||
def test_this_app_neither_forecasts_nor_notifies(monkeypatch):
|
||||
published = []
|
||||
|
||||
class Publisher:
|
||||
def publish(self, n):
|
||||
published.append(n)
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(Config, "FORECAST_IN_COLLECTOR", True)
|
||||
monkeypatch.setitem(web_api.app_state, "notify", (Publisher(), object()))
|
||||
monkeypatch.setattr(
|
||||
"src.ml.predict.get_latest_forecasts",
|
||||
lambda: pytest.fail("forecast precompute ran in the API"),
|
||||
)
|
||||
asyncio.run(web_api._precompute_forecasts())
|
||||
asyncio.run(web_api._notify_transitions())
|
||||
assert published == []
|
||||
Reference in new issue
Block a user