chore: remove the Python service; Python is the training stack only
Phase 4 of docs/RUST_REWRITE.md. Since 2026-10-18 ping-collector (rust/) collects, forecasts, notifies (ntfy + Matrix) and serves the website/API; nothing in production ran the Python service code any more. Removed (~17k lines): the FastAPI app (web_api, schemas, health_check, metrics, postgres_history, forecast_history, thaiwater), the Python RID collector and CLI (water_scraper_v3, main, run.py), the notifier and Matrix alerting (notify, alerting), the multi-backend adapters (SQLite, MySQL, InfluxDB, VictoriaMetrics) with demo databases, models/validators/exceptions/ logging/rate limiter, ml/skill.py (served by Rust now), the migration tooling (api_parity, collector_parity, e2e_notify, install_collector.sh, gen_notify_cases.py, spikes/rust-forecast), Docker/compose/Grafana files, sql/, the release workflow and the docs for all of that. Their tests went with them. Kept in Python: src/ml (training, evaluation harness, export; predict.py as the reference the Rust forecaster is tested against), the HII and RID-dam clients/stores for the archive backfills (their live collectors removed), the backfill scripts, retrain.sh + timer, forecast_parity.py. - PostgreSQL only: src/config.py is database_url() from POSTGRES_* / .env; the stores lost their sqlite/mysql branches (and the 2026-08 column migration). The database tests run against a throwaway schema from TEST_DATABASE_URL (tests/conftest.py pg_url, skipped without it), as the Rust tests do; CI gets a PostgreSQL service. 100 tests (66 without a DB). - The notifier settings moved from src/notify.py into src/ml/export.py; flood_config.json regenerates byte-identical. - tests/test_dashboard.py checks every path the dashboard fetches against ping-collector's router. - Dependencies: fastapi, uvicorn, pydantic, influxdb, pymysql, psutil, schedule, sphinx, pyinstaller, mypy, pytest-asyncio dropped; python-dotenv 1.1.1 explicit (it came in through uvicorn[standard]). - Formatting contract extended to scripts/ and tests/ (reformatted once). - scripts/install.sh is the one installer (fresh host or update, in place in a checkout without touching its ownership): binary, units, uv sync, .env defaults, removes the retired water-monitor.service and any old --alert-check cron job, dry run + forecast check, restart, readiness wait. - Docs, CLAUDE.md, CONTRIBUTING.md, .env.example, Makefile, unit comments and Rust doc comments updated to the Rust service.
This commit is contained in:
1 parent
330fdf7804
commit
7e642555b4
125 files changed
+1300
-19558
No files matched your search
-127
@@ -1,127 +0,0 @@
|
||||
"""Notification cases for rust/collector/src/notify.rs, from src/notify.py itself.
|
||||
|
||||
uv run python rust/collector/testdata/gen_notify_cases.py
|
||||
|
||||
Random hourly sequences around the thresholds (capacity guard, hysteresis,
|
||||
danger escalation, the P.1 outlook, feed gaps, ntfy outages for some topics)
|
||||
run through notify.evaluate with an in-memory state; every delivered
|
||||
notification is recorded. The Rust test replays the same steps (RID readings
|
||||
only, so the ThaiWater additions stay out of play) and must deliver exactly
|
||||
the same messages.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import gzip
|
||||
import json
|
||||
import random
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE.parents[2]))
|
||||
|
||||
from src import notify # noqa: E402
|
||||
|
||||
|
||||
class Publisher(notify.NtfyPublisher):
|
||||
def __init__(self):
|
||||
super().__init__("http://ntfy.invalid", prefix="ping")
|
||||
self.down = set()
|
||||
|
||||
def publish(self, n):
|
||||
return n.topic not in self.down
|
||||
|
||||
|
||||
STATIONS = { # around (warning, danger)
|
||||
"P.1": (3.7, 4.2),
|
||||
"P.77": (4.3, 4.9),
|
||||
"P.21": (3.2, 3.6),
|
||||
"P.20": (2.35, 2.8),
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
rng = random.Random(20261018)
|
||||
sequences = []
|
||||
for _ in range(40):
|
||||
state = notify.InMemoryState()
|
||||
pub = Publisher()
|
||||
t = datetime.datetime(2026, 9, 1) + datetime.timedelta(
|
||||
hours=rng.randrange(2000)
|
||||
)
|
||||
codes = rng.sample(sorted(STATIONS), rng.randint(1, 4))
|
||||
levels = {c: STATIONS[c][0] + rng.uniform(-0.6, 0.6) for c in codes}
|
||||
p_out = rng.random()
|
||||
steps = []
|
||||
for _ in range(rng.randint(8, 25)):
|
||||
gap = rng.choice([1, 1, 1, 1, 2, 4, 6])
|
||||
t += datetime.timedelta(hours=gap)
|
||||
# the clock sometimes runs ahead of the data (a stale feed)
|
||||
now = t + datetime.timedelta(minutes=rng.choice([5, 5, 30, 200, 300]))
|
||||
readings = []
|
||||
for c in codes:
|
||||
levels[c] += rng.uniform(-0.25, 0.25)
|
||||
pct = rng.choice([None, rng.uniform(5, 120)])
|
||||
level = round(levels[c], 2) if rng.random() > 0.05 else None
|
||||
readings.append(
|
||||
{
|
||||
"station_code": c,
|
||||
"timestamp": t,
|
||||
"water_level": level,
|
||||
"discharge_percent": None if pct is None else round(pct, 2),
|
||||
}
|
||||
)
|
||||
p_out = min(1.0, max(0.0, p_out + rng.uniform(-0.3, 0.3)))
|
||||
forecasts = [
|
||||
{
|
||||
"station_code": "P.1",
|
||||
"horizon_hours": rng.choice([24, 24, 24, 12]),
|
||||
"p_warning": p_out,
|
||||
"predicted_max_level": round(rng.uniform(2.5, 5.0), 3),
|
||||
"source": rng.choice(["model", "model", "model", "heuristic"]),
|
||||
}
|
||||
]
|
||||
pub.down = set(
|
||||
rng.sample(
|
||||
[
|
||||
"ping-p1-warning",
|
||||
"ping-warning",
|
||||
"ping-status",
|
||||
"ping-p1-outlook",
|
||||
],
|
||||
rng.choice([0, 0, 0, 1]),
|
||||
)
|
||||
)
|
||||
sent = notify.evaluate(readings, forecasts, state, pub, now=now)
|
||||
steps.append(
|
||||
{
|
||||
"now": now.isoformat(),
|
||||
"readings": [
|
||||
{**r, "timestamp": r["timestamp"].isoformat()} for r in readings
|
||||
],
|
||||
"forecasts": forecasts,
|
||||
"down": sorted(pub.down),
|
||||
"sent": [
|
||||
{
|
||||
"topic": n.topic,
|
||||
"title": n.title,
|
||||
"message": n.message,
|
||||
"priority": n.priority,
|
||||
"tags": n.tags or [],
|
||||
}
|
||||
for n in sent
|
||||
],
|
||||
}
|
||||
)
|
||||
sequences.append(steps)
|
||||
out = HERE / "notify_python_cases.json.gz"
|
||||
with open(out, "wb") as raw, gzip.GzipFile(fileobj=raw, mode="wb", mtime=0) as gz:
|
||||
gz.write(json.dumps({"sequences": sequences}).encode())
|
||||
total = sum(len(s["sent"]) for seq in sequences for s in seq)
|
||||
print(
|
||||
f"{len(sequences)} sequences, {sum(map(len, sequences))} steps, {total} notifications"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user