chore: remove the Python service; Python is the training stack only
Phase 4 of docs/RUST_REWRITE.md. Since 2026-10-18 ping-collector (rust/) collects, forecasts, notifies (ntfy + Matrix) and serves the website/API; nothing in production ran the Python service code any more. Removed (~17k lines): the FastAPI app (web_api, schemas, health_check, metrics, postgres_history, forecast_history, thaiwater), the Python RID collector and CLI (water_scraper_v3, main, run.py), the notifier and Matrix alerting (notify, alerting), the multi-backend adapters (SQLite, MySQL, InfluxDB, VictoriaMetrics) with demo databases, models/validators/exceptions/ logging/rate limiter, ml/skill.py (served by Rust now), the migration tooling (api_parity, collector_parity, e2e_notify, install_collector.sh, gen_notify_cases.py, spikes/rust-forecast), Docker/compose/Grafana files, sql/, the release workflow and the docs for all of that. Their tests went with them. Kept in Python: src/ml (training, evaluation harness, export; predict.py as the reference the Rust forecaster is tested against), the HII and RID-dam clients/stores for the archive backfills (their live collectors removed), the backfill scripts, retrain.sh + timer, forecast_parity.py. - PostgreSQL only: src/config.py is database_url() from POSTGRES_* / .env; the stores lost their sqlite/mysql branches (and the 2026-08 column migration). The database tests run against a throwaway schema from TEST_DATABASE_URL (tests/conftest.py pg_url, skipped without it), as the Rust tests do; CI gets a PostgreSQL service. 100 tests (66 without a DB). - The notifier settings moved from src/notify.py into src/ml/export.py; flood_config.json regenerates byte-identical. - tests/test_dashboard.py checks every path the dashboard fetches against ping-collector's router. - Dependencies: fastapi, uvicorn, pydantic, influxdb, pymysql, psutil, schedule, sphinx, pyinstaller, mypy, pytest-asyncio dropped; python-dotenv 1.1.1 explicit (it came in through uvicorn[standard]). - Formatting contract extended to scripts/ and tests/ (reformatted once). - scripts/install.sh is the one installer (fresh host or update, in place in a checkout without touching its ownership): binary, units, uv sync, .env defaults, removes the retired water-monitor.service and any old --alert-check cron job, dry run + forecast check, restart, readiness wait. - Docs, CLAUDE.md, CONTRIBUTING.md, .env.example, Makefile, unit comments and Rust doc comments updated to the Rust service.
This commit is contained in:
1 parent
330fdf7804
commit
7e642555b4
125 files changed
+1300
-19558
No files matched your search
+31
-342
@@ -1,354 +1,43 @@
|
||||
import os
|
||||
from typing import Any, Dict
|
||||
"""Database settings for the Python tools (training, evaluation, backfills).
|
||||
|
||||
The service itself is ping-collector (rust/collector); it reads the same
|
||||
environment / `.env` file (rust/collector/src/config.rs). PostgreSQL only:
|
||||
POSTGRES_CONNECTION_STRING, or POSTGRES_HOST / POSTGRES_PORT / POSTGRES_DB /
|
||||
POSTGRES_USER / POSTGRES_PASSWORD with the same defaults as ping-collector.
|
||||
"""
|
||||
|
||||
import os
|
||||
import urllib.parse
|
||||
from typing import Optional
|
||||
|
||||
# Load environment variables from .env file
|
||||
try:
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
except ImportError:
|
||||
# python-dotenv not installed, continue without it
|
||||
except ImportError: # the systemd units pass the variables in directly
|
||||
pass
|
||||
|
||||
try:
|
||||
from .exceptions import ConfigurationError
|
||||
from .models import DatabaseType
|
||||
except ImportError:
|
||||
# Handle case when running as standalone script
|
||||
class ConfigurationError(Exception):
|
||||
pass
|
||||
|
||||
from enum import Enum
|
||||
|
||||
class DatabaseType(Enum):
|
||||
SQLITE = "sqlite"
|
||||
MYSQL = "mysql"
|
||||
POSTGRESQL = "postgresql"
|
||||
INFLUXDB = "influxdb"
|
||||
VICTORIAMETRICS = "victoriametrics"
|
||||
|
||||
|
||||
class Config:
|
||||
"""Configuration class for the Water Level Monitor"""
|
||||
|
||||
# Database settings
|
||||
DATABASE_PATH = os.getenv("WATER_DB_PATH", "water_levels.db")
|
||||
|
||||
# Website settings
|
||||
TARGET_URL = "https://hyd-app-db.rid.go.th/hydro1h.html"
|
||||
API_URL = "https://hyd-app-db.rid.go.th/webservice/getGroupHourlyWaterLevelReportAllHL.ashx"
|
||||
THAIWATER_API_KEY = os.getenv("THAIWATER_API_KEY")
|
||||
|
||||
# Public flood notifications (ntfy). Off unless NTFY_SERVER is set.
|
||||
# NTFY_SERVER is what subscribers use (public https URL, shown on the
|
||||
# dashboard). NTFY_PUBLISH_URL is where the monitor POSTs; defaults to
|
||||
# NTFY_SERVER, set it to http://127.0.0.1:2586 when ntfy runs on the same
|
||||
# host so publishing never depends on DNS/proxy/tunnel being up.
|
||||
NTFY_SERVER = os.getenv("NTFY_SERVER", "").strip()
|
||||
NTFY_PUBLISH_URL = os.getenv("NTFY_PUBLISH_URL", "").strip() or NTFY_SERVER
|
||||
NTFY_TOPIC_PREFIX = os.getenv("NTFY_TOPIC_PREFIX", "ping").strip()
|
||||
NTFY_TOKEN = os.getenv("NTFY_TOKEN", "").strip() # publish token if ACL enabled
|
||||
PUBLIC_URL = os.getenv("PUBLIC_URL", "https://water.buildfor.life/").strip()
|
||||
REQUEST_TIMEOUT = int(os.getenv("REQUEST_TIMEOUT", "30"))
|
||||
USER_AGENT = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
|
||||
def database_url() -> Optional[str]:
|
||||
"""The PostgreSQL connection string, or None when none is configured."""
|
||||
url = os.getenv("POSTGRES_CONNECTION_STRING", "").strip()
|
||||
if url:
|
||||
return url
|
||||
password = os.getenv("POSTGRES_PASSWORD")
|
||||
if not password:
|
||||
return None
|
||||
user = urllib.parse.quote(os.getenv("POSTGRES_USER", "postgres"), safe="")
|
||||
host = os.getenv("POSTGRES_HOST", "localhost")
|
||||
port = os.getenv("POSTGRES_PORT", "5432")
|
||||
name = os.getenv("POSTGRES_DB", "water_monitoring")
|
||||
return (
|
||||
f"postgresql://{user}:{urllib.parse.quote(password, safe='')}"
|
||||
f"@{host}:{port}/{name}"
|
||||
)
|
||||
|
||||
# Database configuration
|
||||
# When DB_TYPE is not set explicitly, a configured Postgres connection wins over the sqlite default
|
||||
DB_TYPE = os.getenv(
|
||||
"DB_TYPE",
|
||||
"postgresql" if os.getenv("POSTGRES_CONNECTION_STRING") else "sqlite",
|
||||
).lower()
|
||||
|
||||
# VictoriaMetrics settings
|
||||
# Default to localhost; set VM_HOST in the environment for real deployments
|
||||
# (avoids committing infrastructure hostnames to the repo).
|
||||
VM_HOST = os.getenv("VM_HOST", "localhost")
|
||||
VM_PORT = int(os.getenv("VM_PORT", "443"))
|
||||
|
||||
# Support for HTTPS URLs (e.g., behind reverse proxy)
|
||||
VM_URL = os.getenv("VM_URL") # Full URL override (e.g., https://vm.example.com)
|
||||
|
||||
# InfluxDB settings
|
||||
INFLUX_HOST = os.getenv("INFLUX_HOST", "localhost")
|
||||
INFLUX_PORT = int(os.getenv("INFLUX_PORT", "8086"))
|
||||
INFLUX_DATABASE = os.getenv("INFLUX_DATABASE", "water_monitoring")
|
||||
INFLUX_USERNAME = os.getenv("INFLUX_USERNAME")
|
||||
INFLUX_PASSWORD = os.getenv("INFLUX_PASSWORD")
|
||||
|
||||
# PostgreSQL settings
|
||||
POSTGRES_CONNECTION_STRING = os.getenv("POSTGRES_CONNECTION_STRING")
|
||||
POSTGRES_HOST = os.getenv("POSTGRES_HOST", "localhost")
|
||||
POSTGRES_PORT = int(os.getenv("POSTGRES_PORT", "5432"))
|
||||
POSTGRES_DB = os.getenv("POSTGRES_DB", "water_monitoring")
|
||||
POSTGRES_USER = os.getenv("POSTGRES_USER", "postgres")
|
||||
POSTGRES_PASSWORD = os.getenv("POSTGRES_PASSWORD")
|
||||
|
||||
# MySQL settings
|
||||
MYSQL_CONNECTION_STRING = os.getenv("MYSQL_CONNECTION_STRING")
|
||||
|
||||
# HII/ThaiWater open api-v3 collection (rainfall + backup water level)
|
||||
# See docs/DATA_SOURCES.md. Requires a SQL DB_TYPE (sqlite/postgresql/mysql).
|
||||
ENABLE_HII_COLLECTION = os.getenv("ENABLE_HII_COLLECTION", "true").lower() in (
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
)
|
||||
HII_BASIN_CODE = int(os.getenv("HII_BASIN_CODE", "6")) # 6 = Ping Basin
|
||||
|
||||
# RID large-dam daily status (app.rid.go.th/reservoir) — Mae Ngat et al.
|
||||
ENABLE_RESERVOIR_COLLECTION = os.getenv(
|
||||
"ENABLE_RESERVOIR_COLLECTION", "true"
|
||||
).lower() in ("1", "true", "yes")
|
||||
# TTL for the /api/hii/*/latest response cache; source data changes hourly
|
||||
HII_CACHE_TTL_SECONDS = int(os.getenv("HII_CACHE_TTL_SECONDS", "120"))
|
||||
# TTL for the /measurements/latest response cache (hottest endpoint)
|
||||
LATEST_CACHE_TTL_SECONDS = int(os.getenv("LATEST_CACHE_TTL_SECONDS", "45"))
|
||||
|
||||
# TTL for /health check results (includes an external RID-API probe)
|
||||
HEALTH_CACHE_TTL_SECONDS = int(os.getenv("HEALTH_CACHE_TTL_SECONDS", "30"))
|
||||
|
||||
# Thread-pool size for blocking work in the web process (DB queries,
|
||||
# inference, health probes). Waiting threads are cheap; starving the pool
|
||||
# stalls every endpoint that needs a thread.
|
||||
EXECUTOR_THREADS = int(os.getenv("EXECUTOR_THREADS", "48"))
|
||||
|
||||
# Web server worker processes. Above 1, uvicorn forks workers and a
|
||||
# localhost lock port elects a single background-collection leader.
|
||||
WEB_WORKERS = int(os.getenv("WEB_WORKERS", "2"))
|
||||
COLLECTION_LEADER_PORT = int(os.getenv("COLLECTION_LEADER_PORT", "8901"))
|
||||
|
||||
# Umami analytics (self-hosted). The website id is public (it ships in the
|
||||
# dashboard <script> tag); server-side API tracking posts to /api/send.
|
||||
UMAMI_API_URL = os.getenv("UMAMI_API_URL", "https://stats.buildfor.life/api/send")
|
||||
UMAMI_WEBSITE_ID = os.getenv(
|
||||
"UMAMI_WEBSITE_ID", "00b2be73-8f5f-4400-9029-3be852eb08f7"
|
||||
)
|
||||
UMAMI_TRACK_API = os.getenv("UMAMI_TRACK_API", "true").lower() in (
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
)
|
||||
|
||||
# Scheduler settings
|
||||
SCRAPING_INTERVAL_HOURS = int(os.getenv("SCRAPING_INTERVAL_HOURS", "1"))
|
||||
# RID is polled this many minutes past each hour; while the current hour is
|
||||
# not published yet (the evening report often lags 1-2 h) it is retried
|
||||
# every RID_RETRY_MINUTES. Late readings are stored either way.
|
||||
RID_POLL_OFFSET_MINUTES = int(os.getenv("RID_POLL_OFFSET_MINUTES", "5"))
|
||||
RID_RETRY_MINUTES = int(os.getenv("RID_RETRY_MINUTES", "10"))
|
||||
# On startup the collection leader re-fetches RID reports with holes from
|
||||
# the last N days (0 = off). Older history: `run.py --fill-gaps all`.
|
||||
GAP_FILL_DAYS_ON_START = int(os.getenv("GAP_FILL_DAYS_ON_START", "14"))
|
||||
# Leader -> other workers hand-off for /scraping/status (default: temp dir).
|
||||
# With ping-collector, point both services at the same file inside the app
|
||||
# dir (the systemd units use PrivateTmp).
|
||||
SCRAPING_STATUS_FILE = os.getenv("SCRAPING_STATUS_FILE", "")
|
||||
# false: data collection runs in the Rust ping-collector service
|
||||
# (rust/collector); this app then only serves, forecasts and notifies,
|
||||
# re-forecasting when the newest RID reading advances (polled every
|
||||
# FORECAST_POLL_SECONDS) and at least once per clock hour.
|
||||
COLLECTION_ENABLED = os.getenv("COLLECTION_ENABLED", "true").lower() in (
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
)
|
||||
FORECAST_POLL_SECONDS = int(os.getenv("FORECAST_POLL_SECONDS", "60"))
|
||||
# true: ping-collector also forecasts (rust/forecast) and sends the ntfy
|
||||
# notifications; this app serves its stored forecast (forecast_current)
|
||||
# and does neither. Read by both services from the shared .env.
|
||||
FORECAST_IN_COLLECTOR = os.getenv("FORECAST_IN_COLLECTOR", "false").lower() in (
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
)
|
||||
# A stored forecast older than this is not served; /forecast then computes
|
||||
# in-process as before (the collector is down or has no models).
|
||||
FORECAST_MAX_AGE_SECONDS = int(os.getenv("FORECAST_MAX_AGE_SECONDS", "7200"))
|
||||
|
||||
# Logging settings
|
||||
LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO")
|
||||
LOG_FILE = os.getenv("LOG_FILE", "water_monitor.log")
|
||||
LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s"
|
||||
|
||||
# Data retention
|
||||
DATA_RETENTION_DAYS = int(os.getenv("DATA_RETENTION_DAYS", "365"))
|
||||
|
||||
# Retry settings
|
||||
MAX_RETRIES = int(os.getenv("MAX_RETRIES", "3"))
|
||||
RETRY_DELAY_SECONDS = int(os.getenv("RETRY_DELAY_SECONDS", "60"))
|
||||
|
||||
# Station configuration
|
||||
# Runtime-writable JSON file that persists the station mapping across restarts
|
||||
# (station CRUD via the API writes here). If it does not exist, the bundled
|
||||
# defaults in src/data/stations.json are used to seed it.
|
||||
STATION_CONFIG_PATH = os.getenv("STATION_CONFIG_PATH", "stations.json")
|
||||
|
||||
# Web API / CORS settings
|
||||
# Comma-separated list of allowed origins. Defaults to none (same-origin only);
|
||||
# set CORS_ALLOW_ORIGINS to a specific list of front-end origins in production.
|
||||
# Credentials are only enabled when explicit (non-wildcard) origins are set,
|
||||
# because "*" + credentials is rejected by browsers and unsafe.
|
||||
CORS_ALLOW_ORIGINS = [
|
||||
origin.strip()
|
||||
for origin in os.getenv("CORS_ALLOW_ORIGINS", "").split(",")
|
||||
if origin.strip()
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def validate_config(cls) -> bool:
|
||||
"""Validate configuration settings"""
|
||||
errors = []
|
||||
|
||||
# Validate database type
|
||||
try:
|
||||
DatabaseType(cls.DB_TYPE)
|
||||
except ValueError:
|
||||
errors.append(f"Invalid DB_TYPE: {cls.DB_TYPE}")
|
||||
|
||||
# Validate database-specific settings
|
||||
if cls.DB_TYPE == "victoriametrics":
|
||||
if not cls.VM_HOST:
|
||||
errors.append("VM_HOST is required for VictoriaMetrics")
|
||||
if not isinstance(cls.VM_PORT, int) or cls.VM_PORT <= 0:
|
||||
errors.append("VM_PORT must be a positive integer")
|
||||
|
||||
elif cls.DB_TYPE == "influxdb":
|
||||
if not cls.INFLUX_HOST:
|
||||
errors.append("INFLUX_HOST is required for InfluxDB")
|
||||
if not cls.INFLUX_DATABASE:
|
||||
errors.append("INFLUX_DATABASE is required for InfluxDB")
|
||||
|
||||
elif cls.DB_TYPE in ["postgresql", "mysql"]:
|
||||
if cls.DB_TYPE == "postgresql":
|
||||
# Check if either connection string or individual components are provided
|
||||
if not cls.POSTGRES_CONNECTION_STRING:
|
||||
# If no connection string, check individual components
|
||||
if not cls.POSTGRES_HOST:
|
||||
errors.append("POSTGRES_HOST is required for PostgreSQL")
|
||||
if not cls.POSTGRES_USER:
|
||||
errors.append("POSTGRES_USER is required for PostgreSQL")
|
||||
if not cls.POSTGRES_PASSWORD:
|
||||
errors.append("POSTGRES_PASSWORD is required for PostgreSQL")
|
||||
if not cls.POSTGRES_DB:
|
||||
errors.append("POSTGRES_DB is required for PostgreSQL")
|
||||
else: # mysql
|
||||
if not cls.MYSQL_CONNECTION_STRING:
|
||||
errors.append("MYSQL_CONNECTION_STRING is required for MySQL")
|
||||
|
||||
# Validate numeric settings
|
||||
if cls.SCRAPING_INTERVAL_HOURS <= 0:
|
||||
errors.append("SCRAPING_INTERVAL_HOURS must be positive")
|
||||
|
||||
if not 0 <= cls.RID_POLL_OFFSET_MINUTES < 60:
|
||||
errors.append("RID_POLL_OFFSET_MINUTES must be between 0 and 59")
|
||||
|
||||
if cls.RID_RETRY_MINUTES <= 0:
|
||||
errors.append("RID_RETRY_MINUTES must be positive")
|
||||
|
||||
if cls.DATA_RETENTION_DAYS <= 0:
|
||||
errors.append("DATA_RETENTION_DAYS must be positive")
|
||||
|
||||
if errors:
|
||||
raise ConfigurationError(f"Configuration errors: {'; '.join(errors)}")
|
||||
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def get_database_config(cls) -> Dict[str, Any]:
|
||||
"""Returns database configuration based on DB_TYPE"""
|
||||
if cls.DB_TYPE == "victoriametrics":
|
||||
return {"type": "victoriametrics", "host": cls.VM_HOST, "port": cls.VM_PORT}
|
||||
elif cls.DB_TYPE == "influxdb":
|
||||
return {
|
||||
"type": "influxdb",
|
||||
"host": cls.INFLUX_HOST,
|
||||
"port": cls.INFLUX_PORT,
|
||||
"database": cls.INFLUX_DATABASE,
|
||||
"username": cls.INFLUX_USERNAME,
|
||||
"password": cls.INFLUX_PASSWORD,
|
||||
}
|
||||
elif cls.DB_TYPE == "postgresql":
|
||||
# Use individual components if POSTGRES_CONNECTION_STRING is not provided
|
||||
if cls.POSTGRES_CONNECTION_STRING:
|
||||
return {
|
||||
"type": "postgresql",
|
||||
"connection_string": cls.POSTGRES_CONNECTION_STRING,
|
||||
}
|
||||
else:
|
||||
# Build connection string from components (automatically URL-encodes password)
|
||||
import urllib.parse
|
||||
|
||||
if not cls.POSTGRES_PASSWORD:
|
||||
raise ConfigurationError(
|
||||
"POSTGRES_PASSWORD is required for PostgreSQL (no default is provided)"
|
||||
)
|
||||
password = urllib.parse.quote(cls.POSTGRES_PASSWORD, safe="")
|
||||
connection_string = (
|
||||
f"postgresql://{cls.POSTGRES_USER}:{password}"
|
||||
f"@{cls.POSTGRES_HOST}:{cls.POSTGRES_PORT}/{cls.POSTGRES_DB}"
|
||||
)
|
||||
return {"type": "postgresql", "connection_string": connection_string}
|
||||
elif cls.DB_TYPE == "mysql":
|
||||
if not cls.MYSQL_CONNECTION_STRING:
|
||||
raise ConfigurationError(
|
||||
"MYSQL_CONNECTION_STRING is required for MySQL (no default is provided)"
|
||||
)
|
||||
return {"type": "mysql", "connection_string": cls.MYSQL_CONNECTION_STRING}
|
||||
else: # sqlite
|
||||
return {
|
||||
"type": "sqlite",
|
||||
"connection_string": f"sqlite:///{cls.DATABASE_PATH}",
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def get_all_settings(cls) -> Dict[str, Any]:
|
||||
"""Returns all configuration settings"""
|
||||
return {
|
||||
"DB_TYPE": cls.DB_TYPE,
|
||||
"DATABASE_PATH": cls.DATABASE_PATH,
|
||||
"TARGET_URL": cls.TARGET_URL,
|
||||
"API_URL": cls.API_URL,
|
||||
"REQUEST_TIMEOUT": cls.REQUEST_TIMEOUT,
|
||||
"SCRAPING_INTERVAL_HOURS": cls.SCRAPING_INTERVAL_HOURS,
|
||||
"RID_POLL_OFFSET_MINUTES": cls.RID_POLL_OFFSET_MINUTES,
|
||||
"RID_RETRY_MINUTES": cls.RID_RETRY_MINUTES,
|
||||
"GAP_FILL_DAYS_ON_START": cls.GAP_FILL_DAYS_ON_START,
|
||||
"COLLECTION_ENABLED": cls.COLLECTION_ENABLED,
|
||||
"FORECAST_IN_COLLECTOR": cls.FORECAST_IN_COLLECTOR,
|
||||
"LOG_LEVEL": cls.LOG_LEVEL,
|
||||
"LOG_FILE": cls.LOG_FILE,
|
||||
"DATA_RETENTION_DAYS": cls.DATA_RETENTION_DAYS,
|
||||
"MAX_RETRIES": cls.MAX_RETRIES,
|
||||
"RETRY_DELAY_SECONDS": cls.RETRY_DELAY_SECONDS,
|
||||
"VM_HOST": cls.VM_HOST,
|
||||
"VM_PORT": cls.VM_PORT,
|
||||
"INFLUX_HOST": cls.INFLUX_HOST,
|
||||
"INFLUX_PORT": cls.INFLUX_PORT,
|
||||
"INFLUX_DATABASE": cls.INFLUX_DATABASE,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def print_settings(cls):
|
||||
"""Prints all current settings"""
|
||||
print("=== Water Level Monitor Configuration ===")
|
||||
for key, value in cls.get_all_settings().items():
|
||||
# Hide sensitive information
|
||||
if "PASSWORD" in key and value:
|
||||
value = "*" * len(str(value))
|
||||
print(f"{key}: {value}")
|
||||
print("=" * 45)
|
||||
|
||||
print("\nDatabase Configuration:")
|
||||
db_config = cls.get_database_config()
|
||||
for key, value in db_config.items():
|
||||
if "password" in key and value:
|
||||
value = "*" * len(str(value))
|
||||
print(f" {key}: {value}")
|
||||
print("=" * 45)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
Config.print_settings()
|
||||
url = database_url()
|
||||
if url and "@" in url:
|
||||
url = url.split("://", 1)[0] + "://***@" + url.rsplit("@", 1)[1]
|
||||
print(url or "no database configured")
|
||||
Reference in new issue
Block a user