grabowski
6ae67dec63
CI / Format & lint (push) Successful in 22s
CI / Test suite (push) Successful in 59s
Docs / Validate documentation (push) Successful in 25s
Rust / fmt, clippy, test, build (push) Successful in 10m2s
Security / Dependency vulnerabilities (push) Successful in 2m11s
Security / Static analysis (push) Successful in 22s
Security / License report (push) Successful in 1m37s
Phase 2b/2c of docs/RUST_REWRITE.md. With FORECAST_IN_COLLECTOR=true (shared .env) ping-collector, after each cycle that brought newer RID or HII readings and at least hourly: loads the last 336 h (RID + HII mirror fill), fetches the rain forecast, runs every exported model (rust/forecast), stores forecast_history and forecast_current, and sends the ntfy notifications. The Python API then serves forecast_current as /forecast (computing in-process only when it is older than FORECAST_MAX_AGE_SECONDS) and neither forecasts nor notifies. ntfy (collector/src/notify.rs): a port of src/notify.py with the same topics, texts, hysteresis, capacity guard and notification_state keys, checked word for word on 751 messages generated by Python (testdata/gen_notify_cases.py). The change: for stations whose ThaiWater twin is the same sensor, a reading newer than RID's drives the level alerts (marked as ThaiWater telemetry), and the feed counts as stale only when neither source has a reading for 3 h. RID alone lagging is reported once after RID_STALE_ALERT_HOURS (18). This ends the nightly "gauge feed stale" / "recovered" pairs caused by RID's late reports. Commands: `forecast [--json] [--save] [--now] [--rain-file]`, `notify --dry-run`. scripts/forecast_parity.py compares Python and the collector on the live database (same moment, same rain); locally 48/48 rows identical. install_collector.sh: models export + that check on every install, --cutover-forecast (refuses on a failed check) and --rollback-forecast. Trees are packed into 24-byte nodes: 70 MB peak with all 15 models (was 140). Forecast values are written to NUMERIC as shortest round-trip text, so they round as psycopg2's do. DB test schemas get a counter (parallel tests collided on Windows' coarse clock).
108 lines
3.7 KiB
Python
108 lines
3.7 KiB
Python
"""FORECAST_IN_COLLECTOR=true: ping-collector forecasts and notifies; the API
|
|
serves its stored forecast and falls back to computing when that is stale."""
|
|
|
|
import asyncio
|
|
import datetime
|
|
import json
|
|
import types
|
|
|
|
import pytest
|
|
from sqlalchemy import create_engine, text
|
|
|
|
from src import web_api
|
|
from src.config import Config
|
|
|
|
ROWS = [
|
|
{
|
|
"station_code": "P.1",
|
|
"horizon_hours": 24,
|
|
"p_warning": 0.012,
|
|
"p_danger": 0.001,
|
|
"predicted_max_level": 1.31,
|
|
"current_level": 1.23,
|
|
"as_of": "2026-10-18T13:00:00",
|
|
"model_version": "hgb-v3+abc",
|
|
"trained_at": "2026-10-01T03:42:00",
|
|
"source": "model",
|
|
"threshold_warning": 3.7,
|
|
"threshold_danger": 4.2,
|
|
}
|
|
]
|
|
|
|
|
|
@pytest.fixture
|
|
def collector_db(tmp_path, monkeypatch):
|
|
engine = create_engine(f"sqlite:///{tmp_path / 'db.sqlite'}")
|
|
with engine.begin() as conn:
|
|
conn.execute(
|
|
text(
|
|
"CREATE TABLE forecast_current (id VARCHAR(16) PRIMARY KEY, "
|
|
"computed_at TIMESTAMP NOT NULL, model_versions TEXT, payload TEXT NOT NULL)"
|
|
)
|
|
)
|
|
scraper = types.SimpleNamespace(db_adapter=types.SimpleNamespace(engine=engine))
|
|
monkeypatch.setitem(web_api.app_state, "scraper", scraper)
|
|
monkeypatch.setattr(Config, "FORECAST_IN_COLLECTOR", True)
|
|
monkeypatch.setattr(web_api, "FORECAST_CACHE", {})
|
|
|
|
def store(age_minutes):
|
|
computed = datetime.datetime.now() - datetime.timedelta(minutes=age_minutes)
|
|
with engine.begin() as conn:
|
|
conn.execute(text("DELETE FROM forecast_current"))
|
|
conn.execute(
|
|
text(
|
|
"INSERT INTO forecast_current VALUES ('all', :t, 'hgb-v3+abc', :p)"
|
|
),
|
|
{"t": computed, "p": json.dumps(ROWS)},
|
|
)
|
|
|
|
return store
|
|
|
|
|
|
def test_serves_the_collectors_forecast(collector_db, monkeypatch):
|
|
collector_db(age_minutes=20)
|
|
|
|
def must_not_compute():
|
|
raise AssertionError("computed in-process although a fresh forecast is stored")
|
|
|
|
monkeypatch.setattr("src.ml.predict.get_latest_forecasts", must_not_compute)
|
|
assert asyncio.run(web_api.get_flood_forecasts()) == ROWS
|
|
# served from the short cache on the next request
|
|
assert "stored" in web_api.FORECAST_CACHE
|
|
|
|
|
|
def test_stale_stored_forecast_falls_back_to_computing(collector_db, monkeypatch):
|
|
collector_db(age_minutes=Config.FORECAST_MAX_AGE_SECONDS / 60 + 5)
|
|
computed = [dict(ROWS[0], source="heuristic")]
|
|
monkeypatch.setattr("src.ml.predict.get_latest_forecasts", lambda: computed)
|
|
assert asyncio.run(web_api.get_flood_forecasts()) == computed
|
|
|
|
|
|
def test_missing_table_falls_back_to_computing(tmp_path, monkeypatch):
|
|
engine = create_engine(f"sqlite:///{tmp_path / 'empty.sqlite'}")
|
|
scraper = types.SimpleNamespace(db_adapter=types.SimpleNamespace(engine=engine))
|
|
monkeypatch.setitem(web_api.app_state, "scraper", scraper)
|
|
monkeypatch.setattr(Config, "FORECAST_IN_COLLECTOR", True)
|
|
monkeypatch.setattr(web_api, "FORECAST_CACHE", {})
|
|
monkeypatch.setattr("src.ml.predict.get_latest_forecasts", lambda: ROWS)
|
|
assert asyncio.run(web_api.get_flood_forecasts()) == ROWS
|
|
|
|
|
|
def test_this_app_neither_forecasts_nor_notifies(monkeypatch):
|
|
published = []
|
|
|
|
class Publisher:
|
|
def publish(self, n):
|
|
published.append(n)
|
|
return True
|
|
|
|
monkeypatch.setattr(Config, "FORECAST_IN_COLLECTOR", True)
|
|
monkeypatch.setitem(web_api.app_state, "notify", (Publisher(), object()))
|
|
monkeypatch.setattr(
|
|
"src.ml.predict.get_latest_forecasts",
|
|
lambda: pytest.fail("forecast precompute ran in the API"),
|
|
)
|
|
asyncio.run(web_api._precompute_forecasts())
|
|
asyncio.run(web_api._notify_transitions())
|
|
assert published == []
|