chore: remove the Python service; Python is the training stack only

Phase 4 of docs/RUST_REWRITE.md. Since 2026-10-18 ping-collector (rust/)
collects, forecasts, notifies (ntfy + Matrix) and serves the website/API;
nothing in production ran the Python service code any more.

Removed (~17k lines): the FastAPI app (web_api, schemas, health_check,
metrics, postgres_history, forecast_history, thaiwater), the Python RID
collector and CLI (water_scraper_v3, main, run.py), the notifier and Matrix
alerting (notify, alerting), the multi-backend adapters (SQLite, MySQL,
InfluxDB, VictoriaMetrics) with demo databases, models/validators/exceptions/
logging/rate limiter, ml/skill.py (served by Rust now), the migration tooling
(api_parity, collector_parity, e2e_notify, install_collector.sh,
gen_notify_cases.py, spikes/rust-forecast), Docker/compose/Grafana files,
sql/, the release workflow and the docs for all of that. Their tests went
with them.

Kept in Python: src/ml (training, evaluation harness, export; predict.py as
the reference the Rust forecaster is tested against), the HII and RID-dam
clients/stores for the archive backfills (their live collectors removed),
the backfill scripts, retrain.sh + timer, forecast_parity.py.

- PostgreSQL only: src/config.py is database_url() from POSTGRES_* / .env;
  the stores lost their sqlite/mysql branches (and the 2026-08 column
  migration). The database tests run against a throwaway schema from
  TEST_DATABASE_URL (tests/conftest.py pg_url, skipped without it), as the
  Rust tests do; CI gets a PostgreSQL service. 100 tests (66 without a DB).
- The notifier settings moved from src/notify.py into src/ml/export.py;
  flood_config.json regenerates byte-identical.
- tests/test_dashboard.py checks every path the dashboard fetches against
  ping-collector's router.
- Dependencies: fastapi, uvicorn, pydantic, influxdb, pymysql, psutil,
  schedule, sphinx, pyinstaller, mypy, pytest-asyncio dropped; python-dotenv
  1.1.1 explicit (it came in through uvicorn[standard]).
- Formatting contract extended to scripts/ and tests/ (reformatted once).
- scripts/install.sh is the one installer (fresh host or update, in place in
  a checkout without touching its ownership): binary, units, uv sync, .env
  defaults, removes the retired water-monitor.service and any old
  --alert-check cron job, dry run + forecast check, restart, readiness wait.
- Docs, CLAUDE.md, CONTRIBUTING.md, .env.example, Makefile, unit comments
  and Rust doc comments updated to the Rust service.
This commit is contained in:
grabowski committed 2026-10-18 17:15:07 +07:00
1 parent 330fdf7804
commit 7e642555b4
125 files changed
+1300 -19558

No files matched your search

+31 -342
View File
@@ -1,354 +1,43 @@
import os
from typing import Any, Dict
"""Database settings for the Python tools (training, evaluation, backfills).
The service itself is ping-collector (rust/collector); it reads the same
environment / `.env` file (rust/collector/src/config.rs). PostgreSQL only:
POSTGRES_CONNECTION_STRING, or POSTGRES_HOST / POSTGRES_PORT / POSTGRES_DB /
POSTGRES_USER / POSTGRES_PASSWORD with the same defaults as ping-collector.
"""
import os
import urllib.parse
from typing import Optional
# Load environment variables from .env file
try:
from dotenv import load_dotenv
load_dotenv()
except ImportError:
# python-dotenv not installed, continue without it
except ImportError: # the systemd units pass the variables in directly
pass
try:
from .exceptions import ConfigurationError
from .models import DatabaseType
except ImportError:
# Handle case when running as standalone script
class ConfigurationError(Exception):
pass
from enum import Enum
class DatabaseType(Enum):
SQLITE = "sqlite"
MYSQL = "mysql"
POSTGRESQL = "postgresql"
INFLUXDB = "influxdb"
VICTORIAMETRICS = "victoriametrics"
class Config:
"""Configuration class for the Water Level Monitor"""
# Database settings
DATABASE_PATH = os.getenv("WATER_DB_PATH", "water_levels.db")
# Website settings
TARGET_URL = "https://hyd-app-db.rid.go.th/hydro1h.html"
API_URL = "https://hyd-app-db.rid.go.th/webservice/getGroupHourlyWaterLevelReportAllHL.ashx"
THAIWATER_API_KEY = os.getenv("THAIWATER_API_KEY")
# Public flood notifications (ntfy). Off unless NTFY_SERVER is set.
# NTFY_SERVER is what subscribers use (public https URL, shown on the
# dashboard). NTFY_PUBLISH_URL is where the monitor POSTs; defaults to
# NTFY_SERVER, set it to http://127.0.0.1:2586 when ntfy runs on the same
# host so publishing never depends on DNS/proxy/tunnel being up.
NTFY_SERVER = os.getenv("NTFY_SERVER", "").strip()
NTFY_PUBLISH_URL = os.getenv("NTFY_PUBLISH_URL", "").strip() or NTFY_SERVER
NTFY_TOPIC_PREFIX = os.getenv("NTFY_TOPIC_PREFIX", "ping").strip()
NTFY_TOKEN = os.getenv("NTFY_TOKEN", "").strip() # publish token if ACL enabled
PUBLIC_URL = os.getenv("PUBLIC_URL", "https://water.buildfor.life/").strip()
REQUEST_TIMEOUT = int(os.getenv("REQUEST_TIMEOUT", "30"))
USER_AGENT = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
def database_url() -> Optional[str]:
"""The PostgreSQL connection string, or None when none is configured."""
url = os.getenv("POSTGRES_CONNECTION_STRING", "").strip()
if url:
return url
password = os.getenv("POSTGRES_PASSWORD")
if not password:
return None
user = urllib.parse.quote(os.getenv("POSTGRES_USER", "postgres"), safe="")
host = os.getenv("POSTGRES_HOST", "localhost")
port = os.getenv("POSTGRES_PORT", "5432")
name = os.getenv("POSTGRES_DB", "water_monitoring")
return (
f"postgresql://{user}:{urllib.parse.quote(password, safe='')}"
f"@{host}:{port}/{name}"
)
# Database configuration
# When DB_TYPE is not set explicitly, a configured Postgres connection wins over the sqlite default
DB_TYPE = os.getenv(
"DB_TYPE",
"postgresql" if os.getenv("POSTGRES_CONNECTION_STRING") else "sqlite",
).lower()
# VictoriaMetrics settings
# Default to localhost; set VM_HOST in the environment for real deployments
# (avoids committing infrastructure hostnames to the repo).
VM_HOST = os.getenv("VM_HOST", "localhost")
VM_PORT = int(os.getenv("VM_PORT", "443"))
# Support for HTTPS URLs (e.g., behind reverse proxy)
VM_URL = os.getenv("VM_URL") # Full URL override (e.g., https://vm.example.com)
# InfluxDB settings
INFLUX_HOST = os.getenv("INFLUX_HOST", "localhost")
INFLUX_PORT = int(os.getenv("INFLUX_PORT", "8086"))
INFLUX_DATABASE = os.getenv("INFLUX_DATABASE", "water_monitoring")
INFLUX_USERNAME = os.getenv("INFLUX_USERNAME")
INFLUX_PASSWORD = os.getenv("INFLUX_PASSWORD")
# PostgreSQL settings
POSTGRES_CONNECTION_STRING = os.getenv("POSTGRES_CONNECTION_STRING")
POSTGRES_HOST = os.getenv("POSTGRES_HOST", "localhost")
POSTGRES_PORT = int(os.getenv("POSTGRES_PORT", "5432"))
POSTGRES_DB = os.getenv("POSTGRES_DB", "water_monitoring")
POSTGRES_USER = os.getenv("POSTGRES_USER", "postgres")
POSTGRES_PASSWORD = os.getenv("POSTGRES_PASSWORD")
# MySQL settings
MYSQL_CONNECTION_STRING = os.getenv("MYSQL_CONNECTION_STRING")
# HII/ThaiWater open api-v3 collection (rainfall + backup water level)
# See docs/DATA_SOURCES.md. Requires a SQL DB_TYPE (sqlite/postgresql/mysql).
ENABLE_HII_COLLECTION = os.getenv("ENABLE_HII_COLLECTION", "true").lower() in (
"1",
"true",
"yes",
)
HII_BASIN_CODE = int(os.getenv("HII_BASIN_CODE", "6")) # 6 = Ping Basin
# RID large-dam daily status (app.rid.go.th/reservoir) — Mae Ngat et al.
ENABLE_RESERVOIR_COLLECTION = os.getenv(
"ENABLE_RESERVOIR_COLLECTION", "true"
).lower() in ("1", "true", "yes")
# TTL for the /api/hii/*/latest response cache; source data changes hourly
HII_CACHE_TTL_SECONDS = int(os.getenv("HII_CACHE_TTL_SECONDS", "120"))
# TTL for the /measurements/latest response cache (hottest endpoint)
LATEST_CACHE_TTL_SECONDS = int(os.getenv("LATEST_CACHE_TTL_SECONDS", "45"))
# TTL for /health check results (includes an external RID-API probe)
HEALTH_CACHE_TTL_SECONDS = int(os.getenv("HEALTH_CACHE_TTL_SECONDS", "30"))
# Thread-pool size for blocking work in the web process (DB queries,
# inference, health probes). Waiting threads are cheap; starving the pool
# stalls every endpoint that needs a thread.
EXECUTOR_THREADS = int(os.getenv("EXECUTOR_THREADS", "48"))
# Web server worker processes. Above 1, uvicorn forks workers and a
# localhost lock port elects a single background-collection leader.
WEB_WORKERS = int(os.getenv("WEB_WORKERS", "2"))
COLLECTION_LEADER_PORT = int(os.getenv("COLLECTION_LEADER_PORT", "8901"))
# Umami analytics (self-hosted). The website id is public (it ships in the
# dashboard <script> tag); server-side API tracking posts to /api/send.
UMAMI_API_URL = os.getenv("UMAMI_API_URL", "https://stats.buildfor.life/api/send")
UMAMI_WEBSITE_ID = os.getenv(
"UMAMI_WEBSITE_ID", "00b2be73-8f5f-4400-9029-3be852eb08f7"
)
UMAMI_TRACK_API = os.getenv("UMAMI_TRACK_API", "true").lower() in (
"1",
"true",
"yes",
)
# Scheduler settings
SCRAPING_INTERVAL_HOURS = int(os.getenv("SCRAPING_INTERVAL_HOURS", "1"))
# RID is polled this many minutes past each hour; while the current hour is
# not published yet (the evening report often lags 1-2 h) it is retried
# every RID_RETRY_MINUTES. Late readings are stored either way.
RID_POLL_OFFSET_MINUTES = int(os.getenv("RID_POLL_OFFSET_MINUTES", "5"))
RID_RETRY_MINUTES = int(os.getenv("RID_RETRY_MINUTES", "10"))
# On startup the collection leader re-fetches RID reports with holes from
# the last N days (0 = off). Older history: `run.py --fill-gaps all`.
GAP_FILL_DAYS_ON_START = int(os.getenv("GAP_FILL_DAYS_ON_START", "14"))
# Leader -> other workers hand-off for /scraping/status (default: temp dir).
# With ping-collector, point both services at the same file inside the app
# dir (the systemd units use PrivateTmp).
SCRAPING_STATUS_FILE = os.getenv("SCRAPING_STATUS_FILE", "")
# false: data collection runs in the Rust ping-collector service
# (rust/collector); this app then only serves, forecasts and notifies,
# re-forecasting when the newest RID reading advances (polled every
# FORECAST_POLL_SECONDS) and at least once per clock hour.
COLLECTION_ENABLED = os.getenv("COLLECTION_ENABLED", "true").lower() in (
"1",
"true",
"yes",
)
FORECAST_POLL_SECONDS = int(os.getenv("FORECAST_POLL_SECONDS", "60"))
# true: ping-collector also forecasts (rust/forecast) and sends the ntfy
# notifications; this app serves its stored forecast (forecast_current)
# and does neither. Read by both services from the shared .env.
FORECAST_IN_COLLECTOR = os.getenv("FORECAST_IN_COLLECTOR", "false").lower() in (
"1",
"true",
"yes",
)
# A stored forecast older than this is not served; /forecast then computes
# in-process as before (the collector is down or has no models).
FORECAST_MAX_AGE_SECONDS = int(os.getenv("FORECAST_MAX_AGE_SECONDS", "7200"))
# Logging settings
LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO")
LOG_FILE = os.getenv("LOG_FILE", "water_monitor.log")
LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s"
# Data retention
DATA_RETENTION_DAYS = int(os.getenv("DATA_RETENTION_DAYS", "365"))
# Retry settings
MAX_RETRIES = int(os.getenv("MAX_RETRIES", "3"))
RETRY_DELAY_SECONDS = int(os.getenv("RETRY_DELAY_SECONDS", "60"))
# Station configuration
# Runtime-writable JSON file that persists the station mapping across restarts
# (station CRUD via the API writes here). If it does not exist, the bundled
# defaults in src/data/stations.json are used to seed it.
STATION_CONFIG_PATH = os.getenv("STATION_CONFIG_PATH", "stations.json")
# Web API / CORS settings
# Comma-separated list of allowed origins. Defaults to none (same-origin only);
# set CORS_ALLOW_ORIGINS to a specific list of front-end origins in production.
# Credentials are only enabled when explicit (non-wildcard) origins are set,
# because "*" + credentials is rejected by browsers and unsafe.
CORS_ALLOW_ORIGINS = [
origin.strip()
for origin in os.getenv("CORS_ALLOW_ORIGINS", "").split(",")
if origin.strip()
]
@classmethod
def validate_config(cls) -> bool:
"""Validate configuration settings"""
errors = []
# Validate database type
try:
DatabaseType(cls.DB_TYPE)
except ValueError:
errors.append(f"Invalid DB_TYPE: {cls.DB_TYPE}")
# Validate database-specific settings
if cls.DB_TYPE == "victoriametrics":
if not cls.VM_HOST:
errors.append("VM_HOST is required for VictoriaMetrics")
if not isinstance(cls.VM_PORT, int) or cls.VM_PORT <= 0:
errors.append("VM_PORT must be a positive integer")
elif cls.DB_TYPE == "influxdb":
if not cls.INFLUX_HOST:
errors.append("INFLUX_HOST is required for InfluxDB")
if not cls.INFLUX_DATABASE:
errors.append("INFLUX_DATABASE is required for InfluxDB")
elif cls.DB_TYPE in ["postgresql", "mysql"]:
if cls.DB_TYPE == "postgresql":
# Check if either connection string or individual components are provided
if not cls.POSTGRES_CONNECTION_STRING:
# If no connection string, check individual components
if not cls.POSTGRES_HOST:
errors.append("POSTGRES_HOST is required for PostgreSQL")
if not cls.POSTGRES_USER:
errors.append("POSTGRES_USER is required for PostgreSQL")
if not cls.POSTGRES_PASSWORD:
errors.append("POSTGRES_PASSWORD is required for PostgreSQL")
if not cls.POSTGRES_DB:
errors.append("POSTGRES_DB is required for PostgreSQL")
else: # mysql
if not cls.MYSQL_CONNECTION_STRING:
errors.append("MYSQL_CONNECTION_STRING is required for MySQL")
# Validate numeric settings
if cls.SCRAPING_INTERVAL_HOURS <= 0:
errors.append("SCRAPING_INTERVAL_HOURS must be positive")
if not 0 <= cls.RID_POLL_OFFSET_MINUTES < 60:
errors.append("RID_POLL_OFFSET_MINUTES must be between 0 and 59")
if cls.RID_RETRY_MINUTES <= 0:
errors.append("RID_RETRY_MINUTES must be positive")
if cls.DATA_RETENTION_DAYS <= 0:
errors.append("DATA_RETENTION_DAYS must be positive")
if errors:
raise ConfigurationError(f"Configuration errors: {'; '.join(errors)}")
return True
@classmethod
def get_database_config(cls) -> Dict[str, Any]:
"""Returns database configuration based on DB_TYPE"""
if cls.DB_TYPE == "victoriametrics":
return {"type": "victoriametrics", "host": cls.VM_HOST, "port": cls.VM_PORT}
elif cls.DB_TYPE == "influxdb":
return {
"type": "influxdb",
"host": cls.INFLUX_HOST,
"port": cls.INFLUX_PORT,
"database": cls.INFLUX_DATABASE,
"username": cls.INFLUX_USERNAME,
"password": cls.INFLUX_PASSWORD,
}
elif cls.DB_TYPE == "postgresql":
# Use individual components if POSTGRES_CONNECTION_STRING is not provided
if cls.POSTGRES_CONNECTION_STRING:
return {
"type": "postgresql",
"connection_string": cls.POSTGRES_CONNECTION_STRING,
}
else:
# Build connection string from components (automatically URL-encodes password)
import urllib.parse
if not cls.POSTGRES_PASSWORD:
raise ConfigurationError(
"POSTGRES_PASSWORD is required for PostgreSQL (no default is provided)"
)
password = urllib.parse.quote(cls.POSTGRES_PASSWORD, safe="")
connection_string = (
f"postgresql://{cls.POSTGRES_USER}:{password}"
f"@{cls.POSTGRES_HOST}:{cls.POSTGRES_PORT}/{cls.POSTGRES_DB}"
)
return {"type": "postgresql", "connection_string": connection_string}
elif cls.DB_TYPE == "mysql":
if not cls.MYSQL_CONNECTION_STRING:
raise ConfigurationError(
"MYSQL_CONNECTION_STRING is required for MySQL (no default is provided)"
)
return {"type": "mysql", "connection_string": cls.MYSQL_CONNECTION_STRING}
else: # sqlite
return {
"type": "sqlite",
"connection_string": f"sqlite:///{cls.DATABASE_PATH}",
}
@classmethod
def get_all_settings(cls) -> Dict[str, Any]:
"""Returns all configuration settings"""
return {
"DB_TYPE": cls.DB_TYPE,
"DATABASE_PATH": cls.DATABASE_PATH,
"TARGET_URL": cls.TARGET_URL,
"API_URL": cls.API_URL,
"REQUEST_TIMEOUT": cls.REQUEST_TIMEOUT,
"SCRAPING_INTERVAL_HOURS": cls.SCRAPING_INTERVAL_HOURS,
"RID_POLL_OFFSET_MINUTES": cls.RID_POLL_OFFSET_MINUTES,
"RID_RETRY_MINUTES": cls.RID_RETRY_MINUTES,
"GAP_FILL_DAYS_ON_START": cls.GAP_FILL_DAYS_ON_START,
"COLLECTION_ENABLED": cls.COLLECTION_ENABLED,
"FORECAST_IN_COLLECTOR": cls.FORECAST_IN_COLLECTOR,
"LOG_LEVEL": cls.LOG_LEVEL,
"LOG_FILE": cls.LOG_FILE,
"DATA_RETENTION_DAYS": cls.DATA_RETENTION_DAYS,
"MAX_RETRIES": cls.MAX_RETRIES,
"RETRY_DELAY_SECONDS": cls.RETRY_DELAY_SECONDS,
"VM_HOST": cls.VM_HOST,
"VM_PORT": cls.VM_PORT,
"INFLUX_HOST": cls.INFLUX_HOST,
"INFLUX_PORT": cls.INFLUX_PORT,
"INFLUX_DATABASE": cls.INFLUX_DATABASE,
}
@classmethod
def print_settings(cls):
"""Prints all current settings"""
print("=== Water Level Monitor Configuration ===")
for key, value in cls.get_all_settings().items():
# Hide sensitive information
if "PASSWORD" in key and value:
value = "*" * len(str(value))
print(f"{key}: {value}")
print("=" * 45)
print("\nDatabase Configuration:")
db_config = cls.get_database_config()
for key, value in db_config.items():
if "password" in key and value:
value = "*" * len(str(value))
print(f" {key}: {value}")
print("=" * 45)
if __name__ == "__main__":
Config.print_settings()
url = database_url()
if url and "@" in url:
url = url.split("://", 1)[0] + "://***@" + url.rsplit("@", 1)[1]
print(url or "no database configured")