chore: remove the Python service; Python is the training stack only
Phase 4 of docs/RUST_REWRITE.md. Since 2026-10-18 ping-collector (rust/) collects, forecasts, notifies (ntfy + Matrix) and serves the website/API; nothing in production ran the Python service code any more. Removed (~17k lines): the FastAPI app (web_api, schemas, health_check, metrics, postgres_history, forecast_history, thaiwater), the Python RID collector and CLI (water_scraper_v3, main, run.py), the notifier and Matrix alerting (notify, alerting), the multi-backend adapters (SQLite, MySQL, InfluxDB, VictoriaMetrics) with demo databases, models/validators/exceptions/ logging/rate limiter, ml/skill.py (served by Rust now), the migration tooling (api_parity, collector_parity, e2e_notify, install_collector.sh, gen_notify_cases.py, spikes/rust-forecast), Docker/compose/Grafana files, sql/, the release workflow and the docs for all of that. Their tests went with them. Kept in Python: src/ml (training, evaluation harness, export; predict.py as the reference the Rust forecaster is tested against), the HII and RID-dam clients/stores for the archive backfills (their live collectors removed), the backfill scripts, retrain.sh + timer, forecast_parity.py. - PostgreSQL only: src/config.py is database_url() from POSTGRES_* / .env; the stores lost their sqlite/mysql branches (and the 2026-08 column migration). The database tests run against a throwaway schema from TEST_DATABASE_URL (tests/conftest.py pg_url, skipped without it), as the Rust tests do; CI gets a PostgreSQL service. 100 tests (66 without a DB). - The notifier settings moved from src/notify.py into src/ml/export.py; flood_config.json regenerates byte-identical. - tests/test_dashboard.py checks every path the dashboard fetches against ping-collector's router. - Dependencies: fastapi, uvicorn, pydantic, influxdb, pymysql, psutil, schedule, sphinx, pyinstaller, mypy, pytest-asyncio dropped; python-dotenv 1.1.1 explicit (it came in through uvicorn[standard]). - Formatting contract extended to scripts/ and tests/ (reformatted once). - scripts/install.sh is the one installer (fresh host or update, in place in a checkout without touching its ownership): binary, units, uv sync, .env defaults, removes the retired water-monitor.service and any old --alert-check cron job, dry run + forecast check, restart, readiness wait. - Docs, CLAUDE.md, CONTRIBUTING.md, .env.example, Makefile, unit comments and Rust doc comments updated to the Rust service.
This commit is contained in:
grabowski committed
1 parent
330fdf7804
commit
7e642555b4
125 files changed
+1300
-19558
No files matched your search
@@ -1,5 +1,7 @@
|
||||
//! Configuration from the same environment / `.env` file as the Python app,
|
||||
//! so both services can share `/opt/thailand-water-monitor/.env`.
|
||||
//! Configuration from the environment / `.env` file
|
||||
//! (`/opt/thailand-water-monitor/.env`), which the Python training tools
|
||||
//! (src/config.py) read too. Names and defaults are the ones the Python app
|
||||
//! used.
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use std::env;
|
||||
@@ -33,10 +35,10 @@ pub struct Config {
|
||||
/// Days of RID holes re-fetched at startup (0 = off).
|
||||
pub gap_fill_days_on_start: u32,
|
||||
pub station_config_path: PathBuf,
|
||||
/// Status JSON read by the Python API's /scraping/status.
|
||||
/// Status JSON for /scraping/status when the website runs in a separate
|
||||
/// process (`ping-collector serve`).
|
||||
pub status_file: Option<PathBuf>,
|
||||
/// FORECAST_IN_COLLECTOR: forecast and notify here; the Python API then
|
||||
/// serves the stored forecast and does neither.
|
||||
/// FORECAST_IN_COLLECTOR: forecast and notify (off: collect only).
|
||||
pub forecast_enabled: bool,
|
||||
/// Exported bundles (`flood_<station>.json.gz`, src/ml/export.py).
|
||||
pub models_dir: PathBuf,
|
||||
@@ -62,7 +64,7 @@ pub struct MatrixConfig {
|
||||
pub room_id: String,
|
||||
}
|
||||
|
||||
/// The website and read API (src/web_api.py's settings, same names).
|
||||
/// The website and read API (setting names as the retired Python app had them).
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct WebConfig {
|
||||
/// WEB_LISTEN: `run` also serves the website on this address
|
||||
|
||||
@@ -30,8 +30,9 @@ pub async fn connect(url: &str, max_connections: u32) -> Result<PgPool> {
|
||||
.context("connecting to PostgreSQL")
|
||||
}
|
||||
|
||||
/// Idempotent DDL, identical to what the Python app creates (so either side
|
||||
/// can start first against an empty database).
|
||||
/// Idempotent DDL, identical to what the Python app created and the Python
|
||||
/// backfill stores (src/hii_collector.py, src/rid_reservoir.py) still create,
|
||||
/// so either can start first against an empty database.
|
||||
const SCHEMA: &[&str] = &[
|
||||
"CREATE TABLE IF NOT EXISTS stations (
|
||||
id SERIAL PRIMARY KEY,
|
||||
@@ -145,12 +146,12 @@ const SCHEMA: &[&str] = &[
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (as_of, station_code, horizon_hours))",
|
||||
"CREATE INDEX IF NOT EXISTS idx_forecast_history_station ON forecast_history(station_code, as_of)",
|
||||
// src/notify.py
|
||||
// notify.rs
|
||||
"CREATE TABLE IF NOT EXISTS notification_state (
|
||||
key VARCHAR(64) PRIMARY KEY, state VARCHAR(16) NOT NULL,
|
||||
value NUMERIC(8,3), updated_at TIMESTAMP NOT NULL)",
|
||||
// The current /forecast payload, read by the Python API when
|
||||
// FORECAST_IN_COLLECTOR=true (one row, id 'all').
|
||||
// The current /forecast payload (one row, id 'all'), served by
|
||||
// `ping-collector serve` when the website runs as its own process.
|
||||
"CREATE TABLE IF NOT EXISTS forecast_current (
|
||||
id VARCHAR(16) PRIMARY KEY,
|
||||
computed_at TIMESTAMP NOT NULL,
|
||||
@@ -727,7 +728,7 @@ pub async fn save_forecast_history(
|
||||
Ok(rows.len())
|
||||
}
|
||||
|
||||
/// The payload the Python API serves as /forecast.
|
||||
/// The payload /forecast serves (see `forecast_current` above).
|
||||
pub async fn save_forecast_current(
|
||||
pool: &PgPool,
|
||||
computed_at: NaiveDateTime,
|
||||
|
||||
@@ -2,7 +2,8 @@
|
||||
//! interpolate `${VAR}` / `${VAR:-default}`, never override variables that
|
||||
//! are already set).
|
||||
//!
|
||||
//! The Python app reads the same file, so the two must agree on every value.
|
||||
//! The Python tools (src/config.py) read the same file, so the two must agree
|
||||
//! on every value.
|
||||
//! dotenvy (the usual crate) is stricter: a stray `'` or `"` inside an
|
||||
//! unquoted value makes it swallow the rest of the file and abort, and its
|
||||
//! error message echoes the line, password included. Here an unparsable
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
//! Forecasting and notifications in the collector (FORECAST_IN_COLLECTOR=true),
|
||||
//! replacing the Python API's forecast_notify_task.
|
||||
//! Forecasting and notifications in the collector (FORECAST_IN_COLLECTOR=true);
|
||||
//! the port of the retired Python app's forecast_notify_task.
|
||||
//!
|
||||
//! After each collection cycle that brought newer RID or HII readings, and at
|
||||
//! least once per clock hour: load the last 336 h (RID plus the HII mirror
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
//! ping-collector: the Ping River Monitor's data collection service.
|
||||
//! ping-collector: the Ping River Monitor service.
|
||||
//!
|
||||
//! Replaces the collection half of the Python web app (RID hourly gauges,
|
||||
//! HII/ThaiWater rain + water level, RID dams, Open-Meteo rain) and, with
|
||||
//! FORECAST_IN_COLLECTOR=true, the flood forecast (rust/forecast) and the
|
||||
//! ntfy/Matrix notifications, and with WEB_LISTEN the website and read API
|
||||
//! (the port of the FastAPI app). Writes the same PostgreSQL tables as the
|
||||
//! Python app did. See docs/RUST_REWRITE.md.
|
||||
//! Collects RID hourly gauges, HII/ThaiWater rain + water level, RID dams
|
||||
//! and Open-Meteo rain into PostgreSQL; with FORECAST_IN_COLLECTOR=true runs
|
||||
//! the flood forecast (rust/forecast, models trained in Python) and the
|
||||
//! ntfy/Matrix notifications; with WEB_LISTEN serves the website and API.
|
||||
//! It replaced the Python app in 2026-10 (same tables, routes and messages);
|
||||
//! see docs/RUST_REWRITE.md.
|
||||
|
||||
mod config;
|
||||
mod dams;
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
//! Public flood notifications over ntfy, and the same transitions to a
|
||||
//! Matrix room: the port of src/notify.py, run by the collector when
|
||||
//! FORECAST_IN_COLLECTOR=true. (Matrix replaces the old `run.py
|
||||
//! --alert-check` cron job, which repeated an alert on every run while a
|
||||
//! level stayed high.)
|
||||
//! Matrix room, run by the collector when FORECAST_IN_COLLECTOR=true. A port
|
||||
//! of the retired Python notifier (src/notify.py, removed after the 2026-10
|
||||
//! cutover); its settings still come from src/ml/export.py via
|
||||
//! flood_config.json. (Matrix replaced the old `run.py --alert-check` cron
|
||||
//! job, which repeated an alert on every run while a level stayed high.)
|
||||
//!
|
||||
//! Same topics, texts, hysteresis, capacity guard, outlook and persisted
|
||||
//! state keys (`notification_state`) as Python, so a cutover neither repeats
|
||||
//! nor drops a message. One change: for stations whose HII/ThaiWater twin is
|
||||
//! state keys (`notification_state`) as Python had, so the cutover neither
|
||||
//! repeated nor dropped a message. One change: for stations whose HII/ThaiWater twin is
|
||||
//! the same sensor, a reading newer than RID's (RID's report often lags 2-10 h
|
||||
//! at night) is used for the level alerts and the feed check. The feed is
|
||||
//! "stale" only when neither source has a reading for 3 h; RID lagging alone
|
||||
@@ -1011,8 +1012,10 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// 652 hourly steps from src/notify.py itself (testdata/gen_notify_cases.py):
|
||||
/// the same notifications, word for word, in the same order.
|
||||
/// 652 hourly steps recorded from the retired Python notifier (generator:
|
||||
/// testdata/gen_notify_cases.py, in git history before the cleanup): the
|
||||
/// same notifications, word for word, in the same order. A deliberate
|
||||
/// text change updates the fixture with the new expected messages.
|
||||
#[test]
|
||||
fn matches_python_notify_on_generated_sequences() {
|
||||
use std::io::Read;
|
||||
|
||||
@@ -403,7 +403,7 @@ impl Collector {
|
||||
pub async fn notify(&self, dry_run: bool) -> Result<()> {
|
||||
if !dry_run && !self.cfg.forecast_enabled {
|
||||
bail!(
|
||||
"the Python app notifies while FORECAST_IN_COLLECTOR is not true; \
|
||||
"notifications are off while FORECAST_IN_COLLECTOR is not true; \
|
||||
use --dry-run"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
//! Stale-while-revalidate response cache, as `_cached_swr` in src/web_api.py:
|
||||
//! Stale-while-revalidate response cache, as the retired FastAPI app's
|
||||
//! `_cached_swr`:
|
||||
//!
|
||||
//! - fresh entry: served;
|
||||
//! - expired entry (no older than `max_stale`): served at once, marked stale,
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
//! The website and read API: the port of the FastAPI app (src/web_api.py).
|
||||
//! The website and read API: the port of the retired FastAPI app.
|
||||
//!
|
||||
//! Same routes, parameters, JSON fields, status codes, cache headers and
|
||||
//! response caches as the Python app, checked route by route against it by
|
||||
//! scripts/api_parity.py. It runs inside `ping-collector run` (WEB_LISTEN)
|
||||
//! response caches as the Python app had; before the cutover (2026-10-18) a
|
||||
//! route-by-route comparison of the two on production data found no
|
||||
//! difference. It runs inside `ping-collector run` (WEB_LISTEN)
|
||||
//! or alone (`ping-collector serve`). /docs is the same Swagger UI, fed by an
|
||||
//! OpenAPI document built from the route table that also validates the
|
||||
//! parameters (openapi.rs, params.rs).
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
//! Route tests: the router as the server runs it, against a throwaway
|
||||
//! schema (TEST_DATABASE_URL; see db_tests.rs) or, for the routes that do
|
||||
//! not touch the database, a pool that never connects. Output formats are
|
||||
//! the FastAPI app's (scripts/api_parity.py compares the two live).
|
||||
//! the retired FastAPI app's (field order, isoformat, errors).
|
||||
|
||||
use super::{AppState, router};
|
||||
use crate::config::Config;
|
||||
|
||||
-127
@@ -1,127 +0,0 @@
|
||||
"""Notification cases for rust/collector/src/notify.rs, from src/notify.py itself.
|
||||
|
||||
uv run python rust/collector/testdata/gen_notify_cases.py
|
||||
|
||||
Random hourly sequences around the thresholds (capacity guard, hysteresis,
|
||||
danger escalation, the P.1 outlook, feed gaps, ntfy outages for some topics)
|
||||
run through notify.evaluate with an in-memory state; every delivered
|
||||
notification is recorded. The Rust test replays the same steps (RID readings
|
||||
only, so the ThaiWater additions stay out of play) and must deliver exactly
|
||||
the same messages.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import gzip
|
||||
import json
|
||||
import random
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE.parents[2]))
|
||||
|
||||
from src import notify # noqa: E402
|
||||
|
||||
|
||||
class Publisher(notify.NtfyPublisher):
|
||||
def __init__(self):
|
||||
super().__init__("http://ntfy.invalid", prefix="ping")
|
||||
self.down = set()
|
||||
|
||||
def publish(self, n):
|
||||
return n.topic not in self.down
|
||||
|
||||
|
||||
STATIONS = { # around (warning, danger)
|
||||
"P.1": (3.7, 4.2),
|
||||
"P.77": (4.3, 4.9),
|
||||
"P.21": (3.2, 3.6),
|
||||
"P.20": (2.35, 2.8),
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
rng = random.Random(20261018)
|
||||
sequences = []
|
||||
for _ in range(40):
|
||||
state = notify.InMemoryState()
|
||||
pub = Publisher()
|
||||
t = datetime.datetime(2026, 9, 1) + datetime.timedelta(
|
||||
hours=rng.randrange(2000)
|
||||
)
|
||||
codes = rng.sample(sorted(STATIONS), rng.randint(1, 4))
|
||||
levels = {c: STATIONS[c][0] + rng.uniform(-0.6, 0.6) for c in codes}
|
||||
p_out = rng.random()
|
||||
steps = []
|
||||
for _ in range(rng.randint(8, 25)):
|
||||
gap = rng.choice([1, 1, 1, 1, 2, 4, 6])
|
||||
t += datetime.timedelta(hours=gap)
|
||||
# the clock sometimes runs ahead of the data (a stale feed)
|
||||
now = t + datetime.timedelta(minutes=rng.choice([5, 5, 30, 200, 300]))
|
||||
readings = []
|
||||
for c in codes:
|
||||
levels[c] += rng.uniform(-0.25, 0.25)
|
||||
pct = rng.choice([None, rng.uniform(5, 120)])
|
||||
level = round(levels[c], 2) if rng.random() > 0.05 else None
|
||||
readings.append(
|
||||
{
|
||||
"station_code": c,
|
||||
"timestamp": t,
|
||||
"water_level": level,
|
||||
"discharge_percent": None if pct is None else round(pct, 2),
|
||||
}
|
||||
)
|
||||
p_out = min(1.0, max(0.0, p_out + rng.uniform(-0.3, 0.3)))
|
||||
forecasts = [
|
||||
{
|
||||
"station_code": "P.1",
|
||||
"horizon_hours": rng.choice([24, 24, 24, 12]),
|
||||
"p_warning": p_out,
|
||||
"predicted_max_level": round(rng.uniform(2.5, 5.0), 3),
|
||||
"source": rng.choice(["model", "model", "model", "heuristic"]),
|
||||
}
|
||||
]
|
||||
pub.down = set(
|
||||
rng.sample(
|
||||
[
|
||||
"ping-p1-warning",
|
||||
"ping-warning",
|
||||
"ping-status",
|
||||
"ping-p1-outlook",
|
||||
],
|
||||
rng.choice([0, 0, 0, 1]),
|
||||
)
|
||||
)
|
||||
sent = notify.evaluate(readings, forecasts, state, pub, now=now)
|
||||
steps.append(
|
||||
{
|
||||
"now": now.isoformat(),
|
||||
"readings": [
|
||||
{**r, "timestamp": r["timestamp"].isoformat()} for r in readings
|
||||
],
|
||||
"forecasts": forecasts,
|
||||
"down": sorted(pub.down),
|
||||
"sent": [
|
||||
{
|
||||
"topic": n.topic,
|
||||
"title": n.title,
|
||||
"message": n.message,
|
||||
"priority": n.priority,
|
||||
"tags": n.tags or [],
|
||||
}
|
||||
for n in sent
|
||||
],
|
||||
}
|
||||
)
|
||||
sequences.append(steps)
|
||||
out = HERE / "notify_python_cases.json.gz"
|
||||
with open(out, "wb") as raw, gzip.GzipFile(fileobj=raw, mode="wb", mtime=0) as gz:
|
||||
gz.write(json.dumps({"sequences": sequences}).encode())
|
||||
total = sum(len(s["sent"]) for seq in sequences for s in seq)
|
||||
print(
|
||||
f"{len(sequences)} sequences, {sum(map(len, sequences))} steps, {total} notifications"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user