chore: remove the Python service; Python is the training stack only
Phase 4 of docs/RUST_REWRITE.md. Since 2026-10-18 ping-collector (rust/) collects, forecasts, notifies (ntfy + Matrix) and serves the website/API; nothing in production ran the Python service code any more. Removed (~17k lines): the FastAPI app (web_api, schemas, health_check, metrics, postgres_history, forecast_history, thaiwater), the Python RID collector and CLI (water_scraper_v3, main, run.py), the notifier and Matrix alerting (notify, alerting), the multi-backend adapters (SQLite, MySQL, InfluxDB, VictoriaMetrics) with demo databases, models/validators/exceptions/ logging/rate limiter, ml/skill.py (served by Rust now), the migration tooling (api_parity, collector_parity, e2e_notify, install_collector.sh, gen_notify_cases.py, spikes/rust-forecast), Docker/compose/Grafana files, sql/, the release workflow and the docs for all of that. Their tests went with them. Kept in Python: src/ml (training, evaluation harness, export; predict.py as the reference the Rust forecaster is tested against), the HII and RID-dam clients/stores for the archive backfills (their live collectors removed), the backfill scripts, retrain.sh + timer, forecast_parity.py. - PostgreSQL only: src/config.py is database_url() from POSTGRES_* / .env; the stores lost their sqlite/mysql branches (and the 2026-08 column migration). The database tests run against a throwaway schema from TEST_DATABASE_URL (tests/conftest.py pg_url, skipped without it), as the Rust tests do; CI gets a PostgreSQL service. 100 tests (66 without a DB). - The notifier settings moved from src/notify.py into src/ml/export.py; flood_config.json regenerates byte-identical. - tests/test_dashboard.py checks every path the dashboard fetches against ping-collector's router. - Dependencies: fastapi, uvicorn, pydantic, influxdb, pymysql, psutil, schedule, sphinx, pyinstaller, mypy, pytest-asyncio dropped; python-dotenv 1.1.1 explicit (it came in through uvicorn[standard]). - Formatting contract extended to scripts/ and tests/ (reformatted once). - scripts/install.sh is the one installer (fresh host or update, in place in a checkout without touching its ownership): binary, units, uv sync, .env defaults, removes the retired water-monitor.service and any old --alert-check cron job, dry run + forecast check, restart, readiness wait. - Docs, CLAUDE.md, CONTRIBUTING.md, .env.example, Makefile, unit comments and Rust doc comments updated to the Rust service.
This commit is contained in:
1 parent
330fdf7804
commit
7e642555b4
125 files changed
+1300
-19558
No files matched your search
+32
-49
@@ -1,4 +1,4 @@
|
||||
"""Tests for the RID large-dam daily collector (parsing + persistence)."""
|
||||
"""Tests for the RID large-dam client/store/backfill (parsing + persistence)."""
|
||||
|
||||
import datetime
|
||||
|
||||
@@ -6,7 +6,6 @@ import pytest
|
||||
|
||||
from src.rid_reservoir import (
|
||||
MAE_NGAT_DAM_ID,
|
||||
RidReservoirCollector,
|
||||
RidReservoirStore,
|
||||
backfill,
|
||||
backfill_dam,
|
||||
@@ -131,8 +130,9 @@ class FakeRangeClient:
|
||||
start + datetime.timedelta(days=i) for i in range((end - start).days + 1)
|
||||
]
|
||||
return parse_dam_range_records(
|
||||
_range_payload(rows=[_range_row(d.isoformat()) for d in days
|
||||
if d not in self.gaps])
|
||||
_range_payload(
|
||||
rows=[_range_row(d.isoformat()) for d in days if d not in self.gaps]
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -177,7 +177,9 @@ class TestParsing:
|
||||
)
|
||||
|
||||
def test_parse_dam_range_zero_level_is_missing_not_a_reading(self):
|
||||
payload = _range_payload(rows=[_range_row("2024-09-24", DMD_ULevel_curr="0.00")])
|
||||
payload = _range_payload(
|
||||
rows=[_range_row("2024-09-24", DMD_ULevel_curr="0.00")]
|
||||
)
|
||||
assert parse_dam_range_records(payload)[0]["level_msl"] is None
|
||||
|
||||
def test_parse_dam_range_skips_broken_rows(self):
|
||||
@@ -196,14 +198,11 @@ class TestParsing:
|
||||
|
||||
class TestStore:
|
||||
@pytest.fixture
|
||||
def store(self, tmp_path):
|
||||
store = RidReservoirStore(f"sqlite:///{tmp_path}/rsv_test.db", "sqlite")
|
||||
def store(self, pg_url):
|
||||
store = RidReservoirStore(pg_url)
|
||||
assert store.connect()
|
||||
return store
|
||||
|
||||
def test_rejects_non_sql(self):
|
||||
with pytest.raises(ValueError):
|
||||
RidReservoirStore("http://localhost:8428", "victoriametrics")
|
||||
yield store
|
||||
store.engine.dispose()
|
||||
|
||||
def test_roundtrip_and_upsert(self, store):
|
||||
records = parse_dam_records(_dams_payload())
|
||||
@@ -269,8 +268,11 @@ class TestStore:
|
||||
date = "2026-08-13"
|
||||
assert store.save(parse_dam_range_records(_range_payload((date,)))) == 1
|
||||
# api/dams has no level column at all; its rewrite must not clear one
|
||||
dams_row = [r for r in parse_dam_records(_dams_payload(date))
|
||||
if r["dam_id"] == MAE_NGAT_DAM_ID]
|
||||
dams_row = [
|
||||
r
|
||||
for r in parse_dam_records(_dams_payload(date))
|
||||
if r["dam_id"] == MAE_NGAT_DAM_ID
|
||||
]
|
||||
assert dams_row[0]["level_msl"] is None
|
||||
store.save(dams_row)
|
||||
from sqlalchemy import text
|
||||
@@ -315,27 +317,15 @@ class TestStore:
|
||||
assert row[2] == "เหนือ"
|
||||
|
||||
|
||||
class TestCollectorAndBackfill:
|
||||
def test_run_cycle_today_and_yesterday(self, tmp_path):
|
||||
collector = RidReservoirCollector(
|
||||
{"type": "sqlite", "connection_string": f"sqlite:///{tmp_path}/c.db"},
|
||||
client=FakeClient(),
|
||||
)
|
||||
saved = collector.run_cycle()
|
||||
assert saved == 4 # 2 dams x (today + yesterday)
|
||||
assert collector.client.calls[0] is None # today via empty date
|
||||
|
||||
def test_run_cycle_survives_fetch_failure(self, tmp_path):
|
||||
yesterday = datetime.date.today() - datetime.timedelta(days=1)
|
||||
collector = RidReservoirCollector(
|
||||
{"type": "sqlite", "connection_string": f"sqlite:///{tmp_path}/f.db"},
|
||||
client=FakeClient(fail_dates={yesterday}),
|
||||
)
|
||||
assert collector.run_cycle() == 2 # today only
|
||||
|
||||
def test_backfill_range(self, tmp_path):
|
||||
store = RidReservoirStore(f"sqlite:///{tmp_path}/b.db", "sqlite")
|
||||
class TestBackfill:
|
||||
@pytest.fixture
|
||||
def store(self, pg_url):
|
||||
store = RidReservoirStore(pg_url)
|
||||
assert store.connect()
|
||||
yield store
|
||||
store.engine.dispose()
|
||||
|
||||
def test_backfill_range(self, store):
|
||||
client = FakeClient()
|
||||
saved = backfill(
|
||||
store,
|
||||
@@ -351,9 +341,7 @@ class TestCollectorAndBackfill:
|
||||
datetime.date(2024, 9, 26),
|
||||
]
|
||||
|
||||
def test_backfill_skips_present_days_so_reruns_repair_holes(self, tmp_path):
|
||||
store = RidReservoirStore(f"sqlite:///{tmp_path}/skip.db", "sqlite")
|
||||
assert store.connect()
|
||||
def test_backfill_skips_present_days_so_reruns_repair_holes(self, store):
|
||||
# Day 25 already stored (e.g. by the live collector)
|
||||
store.save(parse_dam_records(_dams_payload("2024-09-25")))
|
||||
client = FakeClient()
|
||||
@@ -370,9 +358,7 @@ class TestCollectorAndBackfill:
|
||||
]
|
||||
assert saved == 4
|
||||
|
||||
def test_backfill_aborts_when_db_saves_nothing(self, tmp_path, monkeypatch):
|
||||
store = RidReservoirStore(f"sqlite:///{tmp_path}/deaddb.db", "sqlite")
|
||||
assert store.connect()
|
||||
def test_backfill_aborts_when_db_saves_nothing(self, store, monkeypatch):
|
||||
monkeypatch.setattr(store, "save", lambda records: 0) # broken DB
|
||||
client = FakeClient()
|
||||
backfill(
|
||||
@@ -384,9 +370,7 @@ class TestCollectorAndBackfill:
|
||||
)
|
||||
assert len(client.calls) == 5 # aborted, not one request per day
|
||||
|
||||
def test_backfill_aborts_after_consecutive_failures(self, tmp_path):
|
||||
store = RidReservoirStore(f"sqlite:///{tmp_path}/a.db", "sqlite")
|
||||
assert store.connect()
|
||||
def test_backfill_aborts_after_consecutive_failures(self, store):
|
||||
start = datetime.date(2024, 1, 1)
|
||||
fail_dates = {start + datetime.timedelta(days=i) for i in range(2, 30)}
|
||||
client = FakeClient(fail_dates=fail_dates)
|
||||
@@ -403,10 +387,11 @@ class TestCollectorAndBackfill:
|
||||
|
||||
class TestBackfillDam:
|
||||
@pytest.fixture
|
||||
def store(self, tmp_path):
|
||||
store = RidReservoirStore(f"sqlite:///{tmp_path}/range.db", "sqlite")
|
||||
def store(self, pg_url):
|
||||
store = RidReservoirStore(pg_url)
|
||||
assert store.connect()
|
||||
return store
|
||||
yield store
|
||||
store.engine.dispose()
|
||||
|
||||
def test_whole_range_in_one_request(self, store):
|
||||
client = FakeRangeClient()
|
||||
@@ -451,9 +436,7 @@ class TestBackfillDam:
|
||||
assert saved == 4 # the day the source never published stays absent
|
||||
assert store.present_dates(
|
||||
datetime.date(2024, 1, 1), datetime.date(2024, 1, 5)
|
||||
) == {
|
||||
datetime.date(2024, 1, d) for d in (1, 2, 4, 5)
|
||||
}
|
||||
) == {datetime.date(2024, 1, d) for d in (1, 2, 4, 5)}
|
||||
|
||||
def test_stored_days_are_skipped_and_whole_chunks_cost_no_request(self, store):
|
||||
client = FakeRangeClient()
|
||||
|
||||
Reference in new issue
Block a user