grabowski
7e642555b4
Phase 4 of docs/RUST_REWRITE.md. Since 2026-10-18 ping-collector (rust/) collects, forecasts, notifies (ntfy + Matrix) and serves the website/API; nothing in production ran the Python service code any more. Removed (~17k lines): the FastAPI app (web_api, schemas, health_check, metrics, postgres_history, forecast_history, thaiwater), the Python RID collector and CLI (water_scraper_v3, main, run.py), the notifier and Matrix alerting (notify, alerting), the multi-backend adapters (SQLite, MySQL, InfluxDB, VictoriaMetrics) with demo databases, models/validators/exceptions/ logging/rate limiter, ml/skill.py (served by Rust now), the migration tooling (api_parity, collector_parity, e2e_notify, install_collector.sh, gen_notify_cases.py, spikes/rust-forecast), Docker/compose/Grafana files, sql/, the release workflow and the docs for all of that. Their tests went with them. Kept in Python: src/ml (training, evaluation harness, export; predict.py as the reference the Rust forecaster is tested against), the HII and RID-dam clients/stores for the archive backfills (their live collectors removed), the backfill scripts, retrain.sh + timer, forecast_parity.py. - PostgreSQL only: src/config.py is database_url() from POSTGRES_* / .env; the stores lost their sqlite/mysql branches (and the 2026-08 column migration). The database tests run against a throwaway schema from TEST_DATABASE_URL (tests/conftest.py pg_url, skipped without it), as the Rust tests do; CI gets a PostgreSQL service. 100 tests (66 without a DB). - The notifier settings moved from src/notify.py into src/ml/export.py; flood_config.json regenerates byte-identical. - tests/test_dashboard.py checks every path the dashboard fetches against ping-collector's router. - Dependencies: fastapi, uvicorn, pydantic, influxdb, pymysql, psutil, schedule, sphinx, pyinstaller, mypy, pytest-asyncio dropped; python-dotenv 1.1.1 explicit (it came in through uvicorn[standard]). - Formatting contract extended to scripts/ and tests/ (reformatted once). - scripts/install.sh is the one installer (fresh host or update, in place in a checkout without touching its ownership): binary, units, uv sync, .env defaults, removes the retired water-monitor.service and any old --alert-check cron job, dry run + forecast check, restart, readiness wait. - Docs, CLAUDE.md, CONTRIBUTING.md, .env.example, Makefile, unit comments and Rust doc comments updated to the Rust service.
310 lines
11 KiB
Python
310 lines
11 KiB
Python
"""Tests for the HII/ThaiWater api-v3 client/store (parsing + persistence)."""
|
|
|
|
import datetime
|
|
|
|
import pytest
|
|
|
|
from src.hii_backfill import chunk_date_range, parse_graph_rows, select_stations
|
|
from src.hii_collector import (
|
|
HiiStore,
|
|
parse_rain_records,
|
|
parse_waterlevel_records,
|
|
rid_code_from_oldcode,
|
|
)
|
|
|
|
|
|
def _rain_payload():
|
|
return {
|
|
"result": "OK",
|
|
"data": [
|
|
{
|
|
"id": 306091240,
|
|
"rain_1h": 0,
|
|
"rain_24h": "49.6",
|
|
"rainfall_datetime": "2026-08-11 13:00",
|
|
"agency": {"agency_shortname": {"en": "HII", "th": "สสน."}},
|
|
"basin": {"basin_code": 6, "basin_name": {"en": "Ping Basin"}},
|
|
"station": {
|
|
"id": 418,
|
|
"sub_basin_id": "0604",
|
|
"tele_station_lat": 19.12207,
|
|
"tele_station_long": 98.94447,
|
|
"tele_station_name": {"en": "Chiang Mai 5", "th": "แม่แตง"},
|
|
"tele_station_oldcode": "CHM005",
|
|
},
|
|
},
|
|
# Wrong basin -> filtered out
|
|
{
|
|
"id": 2,
|
|
"rain_24h": 10,
|
|
"rainfall_datetime": "2026-08-11 13:00",
|
|
"basin": {"basin_code": 7},
|
|
"station": {"id": 99},
|
|
},
|
|
# No timestamp -> skipped
|
|
{
|
|
"id": 3,
|
|
"rain_24h": 5,
|
|
"rainfall_datetime": None,
|
|
"basin": {"basin_code": 6},
|
|
"station": {"id": 100},
|
|
},
|
|
],
|
|
}
|
|
|
|
|
|
def _waterlevel_payload():
|
|
return {
|
|
"waterlevel_data": {
|
|
"result": "OK",
|
|
"data": [
|
|
{
|
|
"id": 1286124160,
|
|
"waterlevel_datetime": "2026-08-11 13:00",
|
|
"waterlevel_m": None,
|
|
"waterlevel_msl": "303.27",
|
|
"discharge": "335.00",
|
|
"flow_rate": None,
|
|
"storage_percent": "81.21",
|
|
"situation_level": 4,
|
|
"diff_wl_bank": "0.93",
|
|
"river_name": "แม่น้ำปิง",
|
|
"agency": {"agency_shortname": {"en": "RID"}},
|
|
"basin": {"basin_code": 6},
|
|
"station": {
|
|
"id": 3226,
|
|
"tele_station_lat": 18.786961,
|
|
"tele_station_long": 99.005089,
|
|
"tele_station_name": {"th": "สะพานนวรัฐ"},
|
|
"tele_station_oldcode": "P.1",
|
|
"offset": 300.5,
|
|
"ground_level": 299.25,
|
|
"min_bank": 304.2,
|
|
"critical_level_msl": 304.2,
|
|
"critical_level_m": 3.7,
|
|
"qmax": 425,
|
|
"is_key_station": True,
|
|
},
|
|
},
|
|
{
|
|
"id": 2,
|
|
"waterlevel_datetime": "2026-08-11 13:00",
|
|
"waterlevel_msl": "200.0",
|
|
"basin": {"basin_code": 6},
|
|
"station": {"id": 4000, "tele_station_oldcode": "ridhydro_P.67"},
|
|
},
|
|
# Wrong basin -> filtered out
|
|
{
|
|
"id": 3,
|
|
"waterlevel_datetime": "2026-08-11 13:00",
|
|
"basin": {"basin_code": 10},
|
|
"station": {"id": 5000},
|
|
},
|
|
],
|
|
}
|
|
}
|
|
|
|
|
|
class TestRidCodeNormalization:
|
|
def test_plain_code(self):
|
|
assert rid_code_from_oldcode("P.1") == "P.1"
|
|
|
|
def test_ridhydro_prefix(self):
|
|
assert rid_code_from_oldcode("ridhydro_P.67") == "P.67"
|
|
|
|
def test_letter_suffix(self):
|
|
assert rid_code_from_oldcode("ridhydro_P.4A") == "P.4A"
|
|
|
|
def test_non_rid_codes(self):
|
|
assert rid_code_from_oldcode("CHM005") is None
|
|
assert rid_code_from_oldcode("ridtele_TUP.14") is None
|
|
assert rid_code_from_oldcode(None) is None
|
|
|
|
|
|
class TestParseRain:
|
|
def test_filters_and_parses(self):
|
|
records = parse_rain_records(_rain_payload())
|
|
assert len(records) == 1
|
|
r = records[0]
|
|
assert r["station_id"] == 418
|
|
assert r["oldcode"] == "CHM005"
|
|
assert r["name_en"] == "Chiang Mai 5"
|
|
assert r["rain_1h"] == 0.0
|
|
assert r["rain_24h"] == 49.6
|
|
assert r["timestamp"] == datetime.datetime(2026, 8, 11, 13, 0)
|
|
assert r["agency"] == "HII"
|
|
|
|
def test_empty_payload(self):
|
|
assert parse_rain_records({}) == []
|
|
|
|
|
|
class TestParseWaterlevel:
|
|
def test_filters_and_parses(self):
|
|
records = parse_waterlevel_records(_waterlevel_payload())
|
|
assert len(records) == 2
|
|
p1 = records[0]
|
|
assert p1["station_id"] == 3226
|
|
assert p1["rid_code"] == "P.1"
|
|
assert p1["wl_msl"] == 303.27
|
|
assert p1["discharge"] == 335.0
|
|
assert p1["flow_rate"] is None
|
|
assert p1["storage_percent"] == 81.21
|
|
assert p1["situation_level"] == 4
|
|
assert p1["offset_msl"] == 300.5
|
|
assert p1["is_key_station"] is True
|
|
# MSL minus station offset recovers the familiar gauge level
|
|
assert p1["wl_msl"] - p1["offset_msl"] == pytest.approx(2.77)
|
|
assert records[1]["rid_code"] == "P.67"
|
|
|
|
def test_empty_payload(self):
|
|
assert parse_waterlevel_records({}) == []
|
|
|
|
|
|
class TestHiiStore:
|
|
@pytest.fixture
|
|
def store(self, pg_url):
|
|
store = HiiStore(pg_url)
|
|
assert store.connect()
|
|
yield store
|
|
store.engine.dispose()
|
|
|
|
def test_rain_roundtrip_and_upsert(self, store):
|
|
records = parse_rain_records(_rain_payload())
|
|
assert store.save_rain(records) == 1
|
|
# Same snapshot again -> upsert, still one row
|
|
assert store.save_rain(records) == 1
|
|
|
|
from sqlalchemy import text
|
|
|
|
with store.engine.connect() as conn:
|
|
rows = conn.execute(text("SELECT COUNT(*) FROM hii_rainfall")).scalar()
|
|
stations = conn.execute(
|
|
text("SELECT oldcode FROM hii_rain_stations")
|
|
).fetchall()
|
|
assert rows == 1
|
|
assert stations == [("CHM005",)]
|
|
|
|
def test_waterlevel_roundtrip(self, store):
|
|
records = parse_waterlevel_records(_waterlevel_payload())
|
|
assert store.save_waterlevel(records) == 2
|
|
|
|
from sqlalchemy import text
|
|
|
|
with store.engine.connect() as conn:
|
|
row = conn.execute(
|
|
text(
|
|
"SELECT s.rid_code, m.wl_msl, m.situation_level "
|
|
"FROM hii_waterlevel m JOIN hii_wl_stations s ON s.id = m.station_id "
|
|
"WHERE s.oldcode = 'P.1'"
|
|
)
|
|
).fetchone()
|
|
assert row is not None
|
|
assert row[0] == "P.1"
|
|
assert float(row[1]) == 303.27
|
|
assert row[2] == 4
|
|
|
|
def test_save_empty(self, store):
|
|
assert store.save_rain([]) == 0
|
|
|
|
def test_history_upsert_preserves_snapshot_columns(self, store):
|
|
# A live snapshot row exists with extra columns populated
|
|
store.save_waterlevel(parse_waterlevel_records(_waterlevel_payload()))
|
|
# Backfill collides on the same (station, timestamp) with new values
|
|
rows = [
|
|
{
|
|
"timestamp": datetime.datetime(2026, 8, 11, 13, 0),
|
|
"wl_msl": 303.30,
|
|
"discharge": 340.0,
|
|
},
|
|
{
|
|
"timestamp": datetime.datetime(2019, 8, 1, 1, 0),
|
|
"wl_msl": 301.71,
|
|
"discharge": 11.7,
|
|
},
|
|
]
|
|
assert store.save_waterlevel_history(3226, rows) == 2
|
|
|
|
from sqlalchemy import text
|
|
|
|
with store.engine.connect() as conn:
|
|
collided = conn.execute(
|
|
text(
|
|
"SELECT wl_msl, discharge, storage_percent, situation_level "
|
|
"FROM hii_waterlevel WHERE station_id = 3226 "
|
|
"AND timestamp = '2026-08-11 13:00:00'"
|
|
)
|
|
).fetchone()
|
|
historical = conn.execute(
|
|
text(
|
|
"SELECT wl_msl FROM hii_waterlevel WHERE station_id = 3226 "
|
|
"AND timestamp = '2019-08-01 01:00:00'"
|
|
)
|
|
).fetchone()
|
|
# wl_msl/discharge updated, snapshot-only columns untouched
|
|
assert float(collided[0]) == 303.30
|
|
assert float(collided[1]) == 340.0
|
|
assert float(collided[2]) == 81.21
|
|
assert collided[3] == 4
|
|
assert float(historical[0]) == 301.71
|
|
|
|
|
|
class TestParseGraphRows:
|
|
def test_parses_and_skips_empty(self):
|
|
payload = {
|
|
"data": {
|
|
"graph_data": [
|
|
{"datetime": "2024-10-05 12:00", "value": 305.8, "discharge": 656},
|
|
{"datetime": "2024-10-05 13:00", "value": None, "discharge": None},
|
|
{"datetime": None, "value": 300.0, "discharge": 1},
|
|
]
|
|
}
|
|
}
|
|
rows = parse_graph_rows(payload)
|
|
assert rows == [
|
|
{
|
|
"timestamp": datetime.datetime(2024, 10, 5, 12, 0),
|
|
"wl_msl": 305.8,
|
|
"discharge": 656.0,
|
|
}
|
|
]
|
|
|
|
def test_empty_payload(self):
|
|
assert parse_graph_rows({}) == []
|
|
|
|
|
|
class TestBackfillHelpers:
|
|
def test_chunk_date_range(self):
|
|
chunks = chunk_date_range(
|
|
datetime.date(2024, 1, 1), datetime.date(2024, 3, 1), 31
|
|
)
|
|
assert chunks[0] == (datetime.date(2024, 1, 1), datetime.date(2024, 1, 31))
|
|
assert chunks[-1][1] == datetime.date(2024, 3, 1)
|
|
# Contiguous, no overlap
|
|
for (_, prev_end), (next_start, _) in zip(chunks, chunks[1:]):
|
|
assert next_start == prev_end + datetime.timedelta(days=1)
|
|
|
|
def test_chunk_single_day(self):
|
|
d = datetime.date(2024, 1, 1)
|
|
assert chunk_date_range(d, d, 365) == [(d, d)]
|
|
|
|
def test_select_default_keeps_rid_and_key_stations(self):
|
|
records = [
|
|
{"station_id": 1, "rid_code": "P.1", "is_key_station": True},
|
|
{"station_id": 2, "rid_code": None, "is_key_station": False},
|
|
{"station_id": 3, "rid_code": None, "is_key_station": True},
|
|
]
|
|
assert [r["station_id"] for r in select_stations(records)] == [1, 3]
|
|
|
|
def test_select_by_code_matches_rid_code_and_oldcode(self):
|
|
records = [
|
|
{"station_id": 1, "rid_code": "P.1", "oldcode": "ridhydro_P.1"},
|
|
{"station_id": 2, "rid_code": None, "oldcode": "CHM004"},
|
|
{"station_id": 3, "rid_code": "P.67", "oldcode": "P.67"},
|
|
]
|
|
selected = select_stations(records, codes=["p.1", "chm004"])
|
|
assert [r["station_id"] for r in selected] == [1, 2]
|
|
|
|
def test_select_all(self):
|
|
records = [{"station_id": 1}, {"station_id": 2}]
|
|
assert select_stations(records, all_stations=True) == records
|