chore: remove the Python service; Python is the training stack only
Phase 4 of docs/RUST_REWRITE.md. Since 2026-10-18 ping-collector (rust/) collects, forecasts, notifies (ntfy + Matrix) and serves the website/API; nothing in production ran the Python service code any more. Removed (~17k lines): the FastAPI app (web_api, schemas, health_check, metrics, postgres_history, forecast_history, thaiwater), the Python RID collector and CLI (water_scraper_v3, main, run.py), the notifier and Matrix alerting (notify, alerting), the multi-backend adapters (SQLite, MySQL, InfluxDB, VictoriaMetrics) with demo databases, models/validators/exceptions/ logging/rate limiter, ml/skill.py (served by Rust now), the migration tooling (api_parity, collector_parity, e2e_notify, install_collector.sh, gen_notify_cases.py, spikes/rust-forecast), Docker/compose/Grafana files, sql/, the release workflow and the docs for all of that. Their tests went with them. Kept in Python: src/ml (training, evaluation harness, export; predict.py as the reference the Rust forecaster is tested against), the HII and RID-dam clients/stores for the archive backfills (their live collectors removed), the backfill scripts, retrain.sh + timer, forecast_parity.py. - PostgreSQL only: src/config.py is database_url() from POSTGRES_* / .env; the stores lost their sqlite/mysql branches (and the 2026-08 column migration). The database tests run against a throwaway schema from TEST_DATABASE_URL (tests/conftest.py pg_url, skipped without it), as the Rust tests do; CI gets a PostgreSQL service. 100 tests (66 without a DB). - The notifier settings moved from src/notify.py into src/ml/export.py; flood_config.json regenerates byte-identical. - tests/test_dashboard.py checks every path the dashboard fetches against ping-collector's router. - Dependencies: fastapi, uvicorn, pydantic, influxdb, pymysql, psutil, schedule, sphinx, pyinstaller, mypy, pytest-asyncio dropped; python-dotenv 1.1.1 explicit (it came in through uvicorn[standard]). - Formatting contract extended to scripts/ and tests/ (reformatted once). - scripts/install.sh is the one installer (fresh host or update, in place in a checkout without touching its ownership): binary, units, uv sync, .env defaults, removes the retired water-monitor.service and any old --alert-check cron job, dry run + forecast check, restart, readiness wait. - Docs, CLAUDE.md, CONTRIBUTING.md, .env.example, Makefile, unit comments and Rust doc comments updated to the Rust service.
This commit is contained in:
grabowski committed
1 parent
330fdf7804
commit
7e642555b4
125 files changed
+1300
-19558
No files matched your search
+5
-184
@@ -1,4 +1,4 @@
|
||||
"""Tests for the HII/ThaiWater api-v3 collector (parsing + persistence)."""
|
||||
"""Tests for the HII/ThaiWater api-v3 client/store (parsing + persistence)."""
|
||||
|
||||
import datetime
|
||||
|
||||
@@ -162,14 +162,11 @@ class TestParseWaterlevel:
|
||||
|
||||
class TestHiiStore:
|
||||
@pytest.fixture
|
||||
def store(self, tmp_path):
|
||||
store = HiiStore(f"sqlite:///{tmp_path}/hii_test.db", "sqlite")
|
||||
def store(self, pg_url):
|
||||
store = HiiStore(pg_url)
|
||||
assert store.connect()
|
||||
return store
|
||||
|
||||
def test_rejects_non_sql_backend(self):
|
||||
with pytest.raises(ValueError):
|
||||
HiiStore("http://localhost:8428", "victoriametrics")
|
||||
yield store
|
||||
store.engine.dispose()
|
||||
|
||||
def test_rain_roundtrip_and_upsert(self, store):
|
||||
records = parse_rain_records(_rain_payload())
|
||||
@@ -275,182 +272,6 @@ class TestParseGraphRows:
|
||||
assert parse_graph_rows({}) == []
|
||||
|
||||
|
||||
class TestHiiApiEndpoints:
|
||||
"""Call the endpoint coroutines directly (the venv's httpx/starlette
|
||||
combination is incompatible with TestClient)."""
|
||||
|
||||
@pytest.fixture
|
||||
def web_api(self, tmp_path, monkeypatch):
|
||||
from src import web_api
|
||||
from src.hii_collector import HiiCollector
|
||||
|
||||
collector = HiiCollector(
|
||||
{"type": "sqlite", "connection_string": f"sqlite:///{tmp_path}/api.db"}
|
||||
)
|
||||
now = datetime.datetime.now().replace(microsecond=0)
|
||||
rain = parse_rain_records(_rain_payload())
|
||||
wl = parse_waterlevel_records(_waterlevel_payload())
|
||||
for record in rain + wl:
|
||||
record["timestamp"] = now
|
||||
assert collector.store.save_rain(rain) == 1
|
||||
assert collector.store.save_waterlevel(wl) == 2
|
||||
monkeypatch.setitem(web_api.app_state, "hii_collector", collector)
|
||||
web_api.HII_CACHE.clear() # response cache would leak across tests
|
||||
web_api._REFRESH_IN_FLIGHT.clear()
|
||||
return web_api
|
||||
|
||||
@staticmethod
|
||||
def _get(web_api_module, endpoint, **kwargs):
|
||||
import asyncio
|
||||
|
||||
from fastapi import Response
|
||||
|
||||
response = Response()
|
||||
rows = asyncio.run(
|
||||
getattr(web_api_module, endpoint)(response=response, **kwargs)
|
||||
)
|
||||
return rows, response
|
||||
|
||||
def test_rainfall_latest(self, web_api):
|
||||
rows, _ = self._get(web_api, "get_hii_rainfall_latest", hours=26)
|
||||
assert len(rows) == 1
|
||||
assert rows[0]["oldcode"] == "CHM005"
|
||||
assert rows[0]["rain_24h"] == 49.6
|
||||
assert rows[0]["latitude"] == pytest.approx(19.12207)
|
||||
|
||||
def test_waterlevel_latest(self, web_api):
|
||||
rows, _ = self._get(web_api, "get_hii_waterlevel_latest", hours=26)
|
||||
assert len(rows) == 2
|
||||
p1 = next(r for r in rows if r["oldcode"] == "P.1")
|
||||
assert p1["rid_code"] == "P.1"
|
||||
assert p1["wl_msl"] == 303.27
|
||||
assert p1["offset_msl"] == 300.5
|
||||
assert p1["situation_level"] == 4
|
||||
|
||||
def test_empty_when_collector_disabled(self, monkeypatch):
|
||||
from src import web_api
|
||||
|
||||
monkeypatch.setitem(web_api.app_state, "hii_collector", None)
|
||||
web_api.HII_CACHE.clear()
|
||||
web_api._REFRESH_IN_FLIGHT.clear()
|
||||
assert self._get(web_api, "get_hii_rainfall_latest", hours=26)[0] == []
|
||||
assert self._get(web_api, "get_hii_waterlevel_latest", hours=26)[0] == []
|
||||
|
||||
def test_latest_responses_are_cached(self, web_api, monkeypatch):
|
||||
calls = {"n": 0}
|
||||
real = web_api._hii_rows
|
||||
|
||||
def counting(sql, params):
|
||||
calls["n"] += 1
|
||||
return real(sql, params)
|
||||
|
||||
monkeypatch.setattr(web_api, "_hii_rows", counting)
|
||||
first, _ = self._get(web_api, "get_hii_rainfall_latest", hours=26)
|
||||
second, _ = self._get(web_api, "get_hii_rainfall_latest", hours=26)
|
||||
assert first == second and len(first) == 1
|
||||
assert calls["n"] == 1 # second call served from the TTL cache
|
||||
# different hours -> different cache key -> fresh query
|
||||
self._get(web_api, "get_hii_rainfall_latest", hours=48)
|
||||
assert calls["n"] == 2
|
||||
|
||||
def test_rainfall_catchment(self, web_api):
|
||||
"""One gauge in the box (CHM005, 19.12N 98.94E) is below the
|
||||
MIN_GAUGES_PER_HOUR floor, so the catchment mean is NaN -> null, the
|
||||
openmeteo_rain table does not exist in this store, and the comparison
|
||||
reports no overlap. Shape is what matters: the endpoint must not 500
|
||||
on a fresh database."""
|
||||
payload, response = self._get(web_api, "get_hii_rainfall_catchment", days=7)
|
||||
assert "x-data-stale" not in response.headers
|
||||
assert list(payload) == ["box", "gauge", "openmeteo", "comparison_24h_sums"]
|
||||
assert payload["openmeteo"] == []
|
||||
assert payload["comparison_24h_sums"] == {"overlap_hours": 0}
|
||||
assert len(payload["gauge"]) == 1
|
||||
assert payload["gauge"][0]["rain_mm"] is None # < MIN_GAUGES_PER_HOUR
|
||||
|
||||
def test_rainfall_catchment_disabled(self, monkeypatch):
|
||||
from src import web_api
|
||||
|
||||
monkeypatch.setitem(web_api.app_state, "hii_collector", None)
|
||||
web_api.HII_CACHE.clear()
|
||||
web_api._REFRESH_IN_FLIGHT.clear()
|
||||
payload, _ = self._get(web_api, "get_hii_rainfall_catchment", days=7)
|
||||
assert payload["gauge"] == [] and payload["openmeteo"] == []
|
||||
|
||||
def test_stale_served_on_recompute_failure(self, web_api, monkeypatch):
|
||||
# Prime the cache, expire it, break the DB: the stale copy is served
|
||||
# and flagged via the X-Data-Stale header.
|
||||
good, response = self._get(web_api, "get_hii_rainfall_latest", hours=26)
|
||||
assert good and "x-data-stale" not in response.headers
|
||||
|
||||
from src.config import Config
|
||||
|
||||
monkeypatch.setattr(Config, "HII_CACHE_TTL_SECONDS", 0)
|
||||
|
||||
def broken(sql, params):
|
||||
raise RuntimeError("db unreachable")
|
||||
|
||||
monkeypatch.setattr(web_api, "_hii_rows", broken)
|
||||
rows, response = self._get(web_api, "get_hii_rainfall_latest", hours=26)
|
||||
assert rows == good
|
||||
assert response.headers["X-Data-Stale"] == "true"
|
||||
|
||||
def test_empty_results_are_not_cached(self, tmp_path, monkeypatch):
|
||||
from src import web_api
|
||||
from src.hii_collector import HiiCollector
|
||||
|
||||
collector = HiiCollector(
|
||||
{"type": "sqlite", "connection_string": f"sqlite:///{tmp_path}/empty.db"}
|
||||
)
|
||||
monkeypatch.setitem(web_api.app_state, "hii_collector", collector)
|
||||
web_api.HII_CACHE.clear()
|
||||
web_api._REFRESH_IN_FLIGHT.clear()
|
||||
assert self._get(web_api, "get_hii_rainfall_latest", hours=26)[0] == []
|
||||
assert web_api.HII_CACHE == {} # empty response left uncached
|
||||
|
||||
def test_measurements_latest_cached_and_stale(self, monkeypatch):
|
||||
from types import SimpleNamespace
|
||||
|
||||
from src import web_api
|
||||
from src.config import Config
|
||||
|
||||
calls = {"n": 0}
|
||||
|
||||
def get_latest_data(limit=100):
|
||||
calls["n"] += 1
|
||||
return [
|
||||
{
|
||||
"timestamp": "2026-08-12 10:00:00",
|
||||
"station_code": "P.1",
|
||||
"station_name_en": "Nawarat Bridge",
|
||||
"station_name_th": "สะพานนวรัฐ",
|
||||
"water_level": 2.76,
|
||||
"discharge": 331.0,
|
||||
"discharge_percent": 77.9,
|
||||
}
|
||||
]
|
||||
|
||||
scraper = SimpleNamespace(db_adapter=True, get_latest_data=get_latest_data)
|
||||
monkeypatch.setitem(web_api.app_state, "scraper", scraper)
|
||||
web_api.LATEST_CACHE.clear()
|
||||
web_api._REFRESH_IN_FLIGHT.clear()
|
||||
|
||||
rows, response = self._get(web_api, "get_latest_measurements", limit=500)
|
||||
rows2, _ = self._get(web_api, "get_latest_measurements", limit=500)
|
||||
assert calls["n"] == 1 # second call cached
|
||||
assert rows2[0].station_code == "P.1"
|
||||
|
||||
# DB failure after expiry -> stale copy + header
|
||||
monkeypatch.setattr(Config, "LATEST_CACHE_TTL_SECONDS", 0)
|
||||
|
||||
def broken(limit=100):
|
||||
raise RuntimeError("db unreachable")
|
||||
|
||||
scraper.get_latest_data = broken
|
||||
rows3, response = self._get(web_api, "get_latest_measurements", limit=500)
|
||||
assert rows3[0].water_level == 2.76
|
||||
assert response.headers["X-Data-Stale"] == "true"
|
||||
|
||||
|
||||
class TestBackfillHelpers:
|
||||
def test_chunk_date_range(self):
|
||||
chunks = chunk_date_range(
|
||||
|
||||
Reference in new issue
Block a user