feat(rust): ping-collector, phase 1 of the Rust rewrite

rust/collector replaces the collection half of the Python web app: RID hourly
gauges (with the late-report semantics: store everything, retry every 10 min
while RID is behind, re-fetch yesterday's holes, startup gap fill), HII rain +
water level, RID dams (today + yesterday) and Open-Meteo rain. Same tables and
upsert semantics, one UNNEST statement per table, Bangkok wall-clock time
regardless of the server time zone.

Python side: COLLECTION_ENABLED=false makes the API leader stop collecting and
instead re-forecast + notify when the newest RID reading in the database
advances (polled every FORECAST_POLL_SECONDS) and once per clock hour;
/scraping/status then serves the collector's status file; /scrape/trigger
answers 409.

Verified: 20 Rust tests (parsers, gap/schedule logic, database tests against
PostgreSQL); live parity run of both collectors into two empty databases:
all 9 tables row-for-row identical (scripts/collector_parity.py); the Python
API served health, measurements, stats, status and model forecasts from a
database filled only by ping-collector. 177 Python tests pass.

reqwest uses native-tls: app.rid.go.th offers only TLS 1.2 CBC suites, which
rustls refuses.

Deploy/cutover/rollback: scripts/install_collector.sh, scripts/ping-collector.service;
CI: .gitea/workflows/rust.yml (fmt, clippy, tests with a PostgreSQL service,
release artifact).

scripts/collector_parity.py

@@ -0,0 +1,150 @@
+"""Live parity check: Python collectors vs the Rust ping-collector.
+
+Runs one collection of every source with the Python code into PY_DB and with
+`ping-collector` into RS_DB (both should start empty), back to back against the
+real upstream APIs, then diffs all nine collector tables row by row (ids and
+created/updated timestamps ignored). Exit status 1 on any difference.
+
+    uv run python scripts/collector_parity.py \
+        --py-db postgresql://postgres:ping@localhost:55432/py \
+        --rs-db postgresql://postgres:ping@localhost:55432/rs \
+        --collector rust/target/debug/ping-collector
+
+Upstream snapshots can move between the two runs (a new HII hour, a revised
+RID value); rerun before treating a small difference as a bug.
+"""
+
+import argparse
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+REPO = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(REPO))
+
+TABLES = {
+    "stations": (
+        "id",
+        "id, station_code, thai_name, english_name, latitude, longitude, geohash",
+    ),
+    "water_measurements": (
+        "timestamp, station_id",
+        "timestamp, station_id, water_level, discharge, discharge_percent, status",
+    ),
+    "hii_rain_stations": (
+        "id",
+        "id, oldcode, name_th, name_en, latitude, longitude, sub_basin_id, agency",
+    ),
+    "hii_rainfall": (
+        "station_id, timestamp",
+        "station_id, timestamp, rain_1h, rain_24h",
+    ),
+    "hii_wl_stations": (
+        "id",
+        "id, oldcode, rid_code, name_th, name_en, latitude, longitude, agency, "
+        "river_name, offset_msl, ground_level_msl, min_bank_msl, "
+        "critical_level_msl, critical_level_m, qmax, is_key_station",
+    ),
+    "hii_waterlevel": (
+        "station_id, timestamp",
+        "station_id, timestamp, wl_msl, wl_m, discharge, flow_rate, "
+        "storage_percent, situation_level, diff_wl_bank",
+    ),
+    "rid_dams": (
+        "dam_id",
+        "dam_id, region, name_th, latitude, longitude, capacity_max_mcm, "
+        "capacity_normal_mcm",
+    ),
+    "rid_reservoir_daily": (
+        "dam_id, date",
+        "dam_id, date, storage_mcm, storage_pct, inflow_mcm, outflow_mcm, level_msl",
+    ),
+    "openmeteo_rain": (
+        "timestamp",
+        "timestamp, chiang_dao, mae_taeng, mae_ngat, mae_rim, chiang_mai, "
+        "catchment_mean",
+    ),
+}
+
+
+def run_python(db_url: str, gap_days: int):
+    os.environ.update(
+        DB_TYPE="postgresql", POSTGRES_CONNECTION_STRING=db_url, LOG_LEVEL="WARNING"
+    )
+    from sqlalchemy import create_engine
+
+    from src.config import Config
+    from src.hii_collector import HiiCollector
+    from src.ml import rain
+    from src.rid_reservoir import RidReservoirCollector
+    from src.water_scraper_v3 import EnhancedWaterMonitorScraper
+
+    cfg = Config.get_database_config()
+    scraper = EnhancedWaterMonitorScraper(cfg)
+    scraper.fill_data_gaps(gap_days)
+    scraper.run_scraping_cycle()
+    HiiCollector(cfg).run_cycle()
+    RidReservoirCollector(cfg).run_cycle()
+    rain.save_to_db(rain.fetch_forecast(), create_engine(db_url), "postgresql")
+
+
+def run_rust(db_url: str, collector: str, gap_days: int):
+    env = {**os.environ, "POSTGRES_CONNECTION_STRING": db_url, "DB_TYPE": "postgresql"}
+    for args in (["fill-gaps", str(gap_days)], ["once"]):
+        subprocess.run([collector, *args], env=env, check=True)
+
+
+def table_rows(db_url: str, table: str, key: str, cols: str) -> dict:
+    import psycopg2
+
+    with psycopg2.connect(db_url) as conn:
+        cur = conn.cursor()
+        cur.execute(f"SELECT {cols} FROM {table} ORDER BY {key}")
+        nkey = len(key.split(","))
+        return {row[:nkey]: row for row in cur.fetchall()}
+
+
+def diff(py_db: str, rs_db: str) -> bool:
+    identical = True
+    for table, (key, cols) in TABLES.items():
+        a = table_rows(py_db, table, key, cols)
+        b = table_rows(rs_db, table, key, cols)
+        only_py, only_rs = sorted(set(a) - set(b)), sorted(set(b) - set(a))
+        differ = sorted(k for k in set(a) & set(b) if a[k] != b[k])
+        same = not (only_py or only_rs or differ)
+        identical &= same
+        print(
+            f"{table:<20} py {len(a):>6}  rs {len(b):>6}  only-py {len(only_py):>4}  "
+            f"only-rs {len(only_rs):>4}  differing {len(differ):>4}  "
+            f"{'identical' if same else 'DIFFERENT'}"
+        )
+        for k in only_py[:3]:
+            print("   only py:", a[k])
+        for k in only_rs[:3]:
+            print("   only rs:", b[k])
+        for k in differ[:3]:
+            print("   py:", a[k], "\n   rs:", b[k])
+    return identical
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument("--py-db", required=True)
+    parser.add_argument("--rs-db", required=True)
+    parser.add_argument(
+        "--collector", default=str(REPO / "rust/target/debug/ping-collector")
+    )
+    parser.add_argument("--gap-days", type=int, default=2)
+    parser.add_argument("--diff-only", action="store_true")
+    args = parser.parse_args()
+    if not args.diff_only:
+        run_python(args.py_db, args.gap_days)
+        run_rust(args.rs_db, args.collector, args.gap_days)
+    ok = diff(args.py_db, args.rs_db)
+    print("ALL IDENTICAL" if ok else "differences found")
+    sys.exit(0 if ok else 1)
+
+
+if __name__ == "__main__":
+    main()

scripts/install_collector.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+# Install the Rust collector (rust/collector, `ping-collector`) and switch
+# data collection over to it, or back.
+#
+#   sudo bash scripts/install_collector.sh             # install binary + unit, dry-run check
+#   sudo bash scripts/install_collector.sh --cutover   # install, then Rust collects:
+#                                                      #   COLLECTION_ENABLED=false, start
+#                                                      #   ping-collector, restart water-monitor
+#   sudo bash scripts/install_collector.sh --rollback  # Python collects again
+#
+# The binary comes from BIN=/path/to/ping-collector (e.g. the "ping-collector-
+# linux-x86_64" CI artifact) or from rust/target/release/ping-collector, built
+# beforehand as a normal user:
+#   cargo build --release --locked --manifest-path rust/Cargo.toml
+# (needs rustup, pkg-config and libssl-dev).
+set -euo pipefail
+
+APP_DIR="${APP_DIR:-/opt/thailand-water-monitor}"
+ENV_FILE="${APP_DIR}/.env"
+STATUS_FILE="${APP_DIR}/collector-status.json"
+REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+MODE="${1:-install}"
+
+log() { printf '\033[1;32m==>\033[0m %s\n' "$*"; }
+die() { printf '\033[1;31m[error]\033[0m %s\n' "$*" >&2; exit 1; }
+
+[ "$(id -u)" -eq 0 ] || die "run as root (sudo)"
+[ -f "$ENV_FILE" ] || die "$ENV_FILE not found"
+
+set_env() {
+    local key="$1" value="$2"
+    if grep -q "^${key}=" "$ENV_FILE"; then
+        sed -i "s|^${key}=.*|${key}=${value}|" "$ENV_FILE"
+    else
+        printf '%s=%s\n' "$key" "$value" >>"$ENV_FILE"
+    fi
+}
+
+if [ "$MODE" = "--rollback" ]; then
+    log "Stopping ping-collector; the Python app collects again"
+    systemctl disable --now ping-collector.service 2>/dev/null || true
+    set_env COLLECTION_ENABLED true
+    systemctl restart water-monitor.service
+    log "Done. Check: journalctl -u water-monitor -n 50"
+    exit 0
+fi
+case "$MODE" in install | --cutover) ;; *) die "unknown option: $MODE" ;; esac
+
+BIN="${BIN:-${REPO_DIR}/rust/target/release/ping-collector}"
+[ -x "$BIN" ] || die "no binary at $BIN: build it (see header) or pass BIN=/path/to/ping-collector"
+
+log "Installing $BIN -> ${APP_DIR}/bin/ping-collector ($("$BIN" --version))"
+install -D -m 0755 -o root -g root "$BIN" "${APP_DIR}/bin/ping-collector"
+install -m 0644 "${REPO_DIR}/scripts/ping-collector.service" /etc/systemd/system/ping-collector.service
+systemctl daemon-reload
+
+# Both services read and write /scraping/status through this file (inside
+# the app dir: the units use PrivateTmp).
+set_env SCRAPING_STATUS_FILE "$STATUS_FILE"
+touch "$STATUS_FILE"
+chown "$(stat -c %U:%G "$APP_DIR")" "$STATUS_FILE"
+
+log "Dry run against the configured database (writes nothing)"
+(cd "$APP_DIR" && "${APP_DIR}/bin/ping-collector" --env-file "$ENV_FILE" once --dry-run)
+
+if [ "$MODE" = "--cutover" ]; then
+    log "Cutover: ping-collector collects, the Python app forecasts and notifies"
+    set_env COLLECTION_ENABLED false
+    systemctl enable --now ping-collector.service
+    systemctl restart water-monitor.service
+    sleep 20
+    systemctl --no-pager --lines=0 status ping-collector.service || true
+    journalctl -u ping-collector -n 20 --no-pager || true
+    log "Rollback if needed: sudo bash scripts/install_collector.sh --rollback"
+else
+    log "Installed, not started. Switch over with: sudo bash scripts/install_collector.sh --cutover"
+fi

scripts/ping-collector.service

@@ -0,0 +1,35 @@
+[Unit]
+Description=Ping River Monitor data collector (Rust: RID, HII, dams, Open-Meteo)
+Documentation=https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/src/branch/master/rust
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=water-monitor
+Group=water-monitor
+WorkingDirectory=/opt/thailand-water-monitor
+ExecStart=/opt/thailand-water-monitor/bin/ping-collector run
+Restart=always
+RestartSec=30
+TimeoutStopSec=20
+
+# Same .env as water-monitor.service (POSTGRES_*, RID_*, GAP_FILL_*,
+# SCRAPING_STATUS_FILE). The Python app must run with COLLECTION_ENABLED=false.
+EnvironmentFile=/opt/thailand-water-monitor/.env
+
+# Security settings
+NoNewPrivileges=true
+PrivateTmp=true
+ProtectSystem=strict
+ProtectHome=true
+ReadWritePaths=/opt/thailand-water-monitor
+CapabilityBoundingSet=
+MemoryMax=256M
+
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=ping-collector
+
+[Install]
+WantedBy=multi-user.target