chore: remove the Python service; Python is the training stack only
Phase 4 of docs/RUST_REWRITE.md. Since 2026-10-18 ping-collector (rust/) collects, forecasts, notifies (ntfy + Matrix) and serves the website/API; nothing in production ran the Python service code any more. Removed (~17k lines): the FastAPI app (web_api, schemas, health_check, metrics, postgres_history, forecast_history, thaiwater), the Python RID collector and CLI (water_scraper_v3, main, run.py), the notifier and Matrix alerting (notify, alerting), the multi-backend adapters (SQLite, MySQL, InfluxDB, VictoriaMetrics) with demo databases, models/validators/exceptions/ logging/rate limiter, ml/skill.py (served by Rust now), the migration tooling (api_parity, collector_parity, e2e_notify, install_collector.sh, gen_notify_cases.py, spikes/rust-forecast), Docker/compose/Grafana files, sql/, the release workflow and the docs for all of that. Their tests went with them. Kept in Python: src/ml (training, evaluation harness, export; predict.py as the reference the Rust forecaster is tested against), the HII and RID-dam clients/stores for the archive backfills (their live collectors removed), the backfill scripts, retrain.sh + timer, forecast_parity.py. - PostgreSQL only: src/config.py is database_url() from POSTGRES_* / .env; the stores lost their sqlite/mysql branches (and the 2026-08 column migration). The database tests run against a throwaway schema from TEST_DATABASE_URL (tests/conftest.py pg_url, skipped without it), as the Rust tests do; CI gets a PostgreSQL service. 100 tests (66 without a DB). - The notifier settings moved from src/notify.py into src/ml/export.py; flood_config.json regenerates byte-identical. - tests/test_dashboard.py checks every path the dashboard fetches against ping-collector's router. - Dependencies: fastapi, uvicorn, pydantic, influxdb, pymysql, psutil, schedule, sphinx, pyinstaller, mypy, pytest-asyncio dropped; python-dotenv 1.1.1 explicit (it came in through uvicorn[standard]). - Formatting contract extended to scripts/ and tests/ (reformatted once). - scripts/install.sh is the one installer (fresh host or update, in place in a checkout without touching its ownership): binary, units, uv sync, .env defaults, removes the retired water-monitor.service and any old --alert-check cron job, dry run + forecast check, restart, readiness wait. - Docs, CLAUDE.md, CONTRIBUTING.md, .env.example, Makefile, unit comments and Rust doc comments updated to the Rust service.
This commit is contained in:
grabowski committed
1 parent
330fdf7804
commit
7e642555b4
125 files changed
+1300
-19558
No files matched your search
+3
-36
@@ -1,38 +1,5 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Northern Thailand Ping River Monitor Package
|
||||
|
||||
A comprehensive real-time water level monitoring system for the Ping River Basin
|
||||
in Northern Thailand, covering Royal Irrigation Department (RID) stations.
|
||||
"""
|
||||
"""Ping River Monitor: the Python side, i.e. the flood-model training and
|
||||
evaluation stack (src/ml) and the archive backfills. The service (collection,
|
||||
forecast, notifications, website/API) is ping-collector in rust/."""
|
||||
|
||||
__version__ = "3.1.3"
|
||||
__author__ = "Ping River Monitor Team"
|
||||
__description__ = "Northern Thailand Ping River Monitoring System"
|
||||
|
||||
from .config import Config
|
||||
from .database_adapters import DatabaseAdapter, create_database_adapter
|
||||
from .exceptions import (
|
||||
APIConnectionError,
|
||||
ConfigurationError,
|
||||
DatabaseConnectionError,
|
||||
DataValidationError,
|
||||
WaterMonitorException,
|
||||
)
|
||||
from .models import DatabaseConfig, StationInfo, WaterMeasurement
|
||||
from .water_scraper_v3 import EnhancedWaterMonitorScraper
|
||||
|
||||
__all__ = [
|
||||
"EnhancedWaterMonitorScraper",
|
||||
"create_database_adapter",
|
||||
"DatabaseAdapter",
|
||||
"Config",
|
||||
"WaterMeasurement",
|
||||
"StationInfo",
|
||||
"DatabaseConfig",
|
||||
"WaterMonitorException",
|
||||
"DatabaseConnectionError",
|
||||
"APIConnectionError",
|
||||
"DataValidationError",
|
||||
"ConfigurationError",
|
||||
]
|
||||
-602
@@ -1,602 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Water Level Alerting System with Matrix Integration
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import html
|
||||
import os
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
import requests
|
||||
|
||||
try:
|
||||
from .config import Config
|
||||
from .database_adapters import create_database_adapter
|
||||
from .logging_config import get_logger
|
||||
except ImportError:
|
||||
import logging
|
||||
|
||||
from config import Config
|
||||
from database_adapters import create_database_adapter
|
||||
|
||||
def get_logger(name):
|
||||
return logging.getLogger(name)
|
||||
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
_BOLD_RE = re.compile(r"\*\*(.+?)\*\*")
|
||||
_URL_RE = re.compile(r"(https?://[^\s<]+)")
|
||||
|
||||
|
||||
def markdown_to_matrix_html(text: str) -> str:
|
||||
"""Convert the small Markdown subset we emit into Matrix-compatible HTML.
|
||||
|
||||
Matrix clients do NOT render Markdown in the plain ``body`` field; formatting
|
||||
only shows when an HTML ``formatted_body`` is sent alongside it. We only use
|
||||
``**bold**``, bare URLs and newlines, so a minimal converter is sufficient and
|
||||
avoids adding a Markdown dependency.
|
||||
"""
|
||||
# Escape HTML special chars first so station/message data can't inject markup.
|
||||
result = html.escape(text, quote=False)
|
||||
result = _BOLD_RE.sub(r"<strong>\1</strong>", result)
|
||||
result = _URL_RE.sub(r'<a href="\1">\1</a>', result)
|
||||
result = result.replace("\n", "<br/>")
|
||||
return result
|
||||
|
||||
|
||||
def strip_markdown(text: str) -> str:
|
||||
"""Produce a clean plain-text fallback for the Matrix ``body`` field."""
|
||||
return _BOLD_RE.sub(r"\1", text)
|
||||
|
||||
|
||||
class AlertLevel(Enum):
|
||||
INFO = "info"
|
||||
WARNING = "warning"
|
||||
CRITICAL = "critical"
|
||||
EMERGENCY = "emergency"
|
||||
|
||||
|
||||
@dataclass
|
||||
class WaterAlert:
|
||||
station_code: str
|
||||
station_name: str
|
||||
alert_type: str
|
||||
level: AlertLevel
|
||||
water_level: float
|
||||
threshold: float
|
||||
discharge: Optional[float] = None
|
||||
timestamp: Optional[datetime.datetime] = None
|
||||
message: Optional[str] = None
|
||||
|
||||
|
||||
class MatrixNotifier:
|
||||
def __init__(self, homeserver: str, access_token: str, room_id: str):
|
||||
self.homeserver = homeserver.rstrip("/")
|
||||
self.access_token = access_token
|
||||
self.room_id = room_id
|
||||
self.session = requests.Session()
|
||||
|
||||
def send_message(
|
||||
self, message: str, msgtype: str = "m.text", markdown: bool = True
|
||||
) -> bool:
|
||||
"""Send a message to the Matrix room.
|
||||
|
||||
When ``markdown`` is True (default) the ``message`` is treated as Markdown:
|
||||
a rendered HTML ``formatted_body`` is sent so clients show real formatting,
|
||||
with a plain-text ``body`` fallback for clients that ignore HTML.
|
||||
"""
|
||||
try:
|
||||
# Add transaction ID to prevent duplicates
|
||||
txn_id = datetime.datetime.now().strftime("%Y%m%d_%H%M%S_%f")
|
||||
url = f"{self.homeserver}/_matrix/client/v3/rooms/{self.room_id}/send/m.room.message/{txn_id}"
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.access_token}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
if markdown:
|
||||
data = {
|
||||
"msgtype": msgtype,
|
||||
"body": strip_markdown(message),
|
||||
"format": "org.matrix.custom.html",
|
||||
"formatted_body": markdown_to_matrix_html(message),
|
||||
}
|
||||
else:
|
||||
data = {"msgtype": msgtype, "body": message}
|
||||
|
||||
# Matrix API requires PUT when transaction ID is in the URL path
|
||||
response = self.session.put(url, headers=headers, json=data, timeout=10)
|
||||
response.raise_for_status()
|
||||
|
||||
logger.info(
|
||||
f"Matrix message sent successfully: {response.json().get('event_id')}"
|
||||
)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to send Matrix message: {e}")
|
||||
return False
|
||||
|
||||
def send_alert(self, alert: WaterAlert) -> bool:
|
||||
"""Send formatted water alert to Matrix"""
|
||||
emoji_map = {
|
||||
AlertLevel.INFO: "ℹ️",
|
||||
AlertLevel.WARNING: "⚠️",
|
||||
AlertLevel.CRITICAL: "🚨",
|
||||
AlertLevel.EMERGENCY: "🆘",
|
||||
}
|
||||
|
||||
emoji = emoji_map.get(alert.level, "📊")
|
||||
|
||||
message = f"""{emoji} **WATER LEVEL ALERT**
|
||||
|
||||
**Station:** {alert.station_code} ({alert.station_name})
|
||||
**Alert Type:** {alert.alert_type}
|
||||
**Severity:** {alert.level.value.upper()}
|
||||
|
||||
**Current Level:** {alert.water_level:.2f}m
|
||||
**Threshold:** {alert.threshold:.2f}m
|
||||
**Difference:** {(alert.water_level - alert.threshold):+.2f}m
|
||||
"""
|
||||
|
||||
if alert.discharge:
|
||||
message += f"**Discharge:** {alert.discharge:.1f} cms\n"
|
||||
|
||||
if alert.timestamp:
|
||||
message += f"**Time:** {alert.timestamp.strftime('%Y-%m-%d %H:%M:%S')}\n"
|
||||
|
||||
if alert.message:
|
||||
message += f"\n**Details:** {alert.message}\n"
|
||||
|
||||
# Add live dashboard link
|
||||
dashboard_url = os.getenv("ALERT_DASHBOARD_URL", "https://water.buildfor.life/")
|
||||
message += f"\n📈 **View Dashboard:** {dashboard_url}"
|
||||
|
||||
return self.send_message(message)
|
||||
|
||||
|
||||
class WaterLevelAlertSystem:
|
||||
# Stations upstream of Chiang Mai (and CNX itself) to monitor
|
||||
UPSTREAM_STATIONS = {
|
||||
"P.20", # Ban Chiang Dao
|
||||
"P.75", # Ban Chai Lat
|
||||
"P.92", # Ban Muang Aut
|
||||
"P.4A", # Ban Mae Taeng
|
||||
"P.67", # Ban Tae
|
||||
"P.21", # Ban Rim Tai
|
||||
"P.103", # Ring Bridge 3
|
||||
"P.1", # Nawarat Bridge (Chiang Mai)
|
||||
}
|
||||
|
||||
def __init__(self):
|
||||
self.db_adapter = None
|
||||
self.matrix_notifier = None
|
||||
self.thresholds = self._load_thresholds()
|
||||
|
||||
# Matrix configuration from environment
|
||||
matrix_homeserver = os.getenv("MATRIX_HOMESERVER", "https://matrix.org")
|
||||
matrix_token = os.getenv("MATRIX_ACCESS_TOKEN")
|
||||
matrix_room = os.getenv("MATRIX_ROOM_ID")
|
||||
|
||||
if matrix_token and matrix_room:
|
||||
self.matrix_notifier = MatrixNotifier(
|
||||
matrix_homeserver, matrix_token, matrix_room
|
||||
)
|
||||
logger.info("Matrix notifications enabled")
|
||||
else:
|
||||
logger.warning("Matrix configuration missing - notifications disabled")
|
||||
|
||||
def _load_thresholds(self) -> Dict[str, Dict[str, float]]:
|
||||
"""Load alert thresholds from config or database"""
|
||||
# Default thresholds for Northern Thailand stations
|
||||
return {
|
||||
"P.1": {
|
||||
# Zone-based thresholds for Nawarat Bridge (P.1)
|
||||
"zone_1": 3.7,
|
||||
"zone_2": 3.9,
|
||||
"zone_3": 4.0,
|
||||
"zone_4": 4.1,
|
||||
"zone_5": 4.2,
|
||||
"zone_6": 4.3,
|
||||
"zone_7": 4.6,
|
||||
"zone_8": 4.8,
|
||||
"newedge": 4.8, # Same as zone 8 or adjust as needed
|
||||
# Keep legacy thresholds for compatibility
|
||||
"warning": 3.7,
|
||||
"critical": 4.3,
|
||||
"emergency": 4.8,
|
||||
},
|
||||
"P.4A": {"warning": 4.5, "critical": 6.0, "emergency": 7.5},
|
||||
"P.20": {"warning": 3.0, "critical": 4.5, "emergency": 6.0},
|
||||
"P.21": {"warning": 4.0, "critical": 5.5, "emergency": 7.0},
|
||||
"P.67": {"warning": 6.0, "critical": 8.0, "emergency": 10.0},
|
||||
"P.75": {"warning": 5.5, "critical": 7.5, "emergency": 9.5},
|
||||
"P.103": {"warning": 7.0, "critical": 9.0, "emergency": 11.0},
|
||||
# Default for unknown stations
|
||||
"default": {"warning": 4.0, "critical": 6.0, "emergency": 8.0},
|
||||
}
|
||||
|
||||
def connect_database(self):
|
||||
"""Initialize database connection"""
|
||||
try:
|
||||
db_config = Config.get_database_config()
|
||||
self.db_adapter = create_database_adapter(
|
||||
db_config["type"], connection_string=db_config["connection_string"]
|
||||
)
|
||||
|
||||
if self.db_adapter.connect():
|
||||
logger.info("Database connection established for alerting")
|
||||
return True
|
||||
else:
|
||||
logger.error("Failed to connect to database")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Database connection error: {e}")
|
||||
return False
|
||||
|
||||
def check_water_levels(self) -> List[WaterAlert]:
|
||||
"""Check current water levels against thresholds"""
|
||||
alerts = []
|
||||
|
||||
if not self.db_adapter:
|
||||
logger.error("Database not connected")
|
||||
return alerts
|
||||
|
||||
try:
|
||||
# Get latest measurements
|
||||
measurements = self.db_adapter.get_latest_measurements(limit=50)
|
||||
|
||||
for measurement in measurements:
|
||||
station_code = measurement.get("station_code", "UNKNOWN")
|
||||
water_level = measurement.get("water_level")
|
||||
|
||||
if not water_level:
|
||||
continue
|
||||
|
||||
# Only alert for upstream stations and Chiang Mai
|
||||
if station_code not in self.UPSTREAM_STATIONS:
|
||||
continue
|
||||
|
||||
# Get thresholds for this station
|
||||
station_thresholds = self.thresholds.get(
|
||||
station_code, self.thresholds["default"]
|
||||
)
|
||||
|
||||
# Check each threshold level
|
||||
alert_level = None
|
||||
threshold_value = None
|
||||
alert_type = None
|
||||
|
||||
# Special handling for P.1 with zone-based thresholds
|
||||
if station_code == "P.1" and "zone_1" in station_thresholds:
|
||||
# Check all zones in reverse order (highest to lowest)
|
||||
zones = [
|
||||
("zone_8", 4.8, AlertLevel.EMERGENCY, "Zone 8 - Emergency"),
|
||||
("newedge", 4.8, AlertLevel.EMERGENCY, "NewEdge Alert Level"),
|
||||
("zone_7", 4.6, AlertLevel.CRITICAL, "Zone 7 - Critical"),
|
||||
("zone_6", 4.3, AlertLevel.CRITICAL, "Zone 6 - Critical"),
|
||||
("zone_5", 4.2, AlertLevel.WARNING, "Zone 5 - Warning"),
|
||||
("zone_4", 4.1, AlertLevel.WARNING, "Zone 4 - Warning"),
|
||||
("zone_3", 4.0, AlertLevel.WARNING, "Zone 3 - Warning"),
|
||||
("zone_2", 3.9, AlertLevel.INFO, "Zone 2 - Info"),
|
||||
("zone_1", 3.7, AlertLevel.INFO, "Zone 1 - Info"),
|
||||
]
|
||||
|
||||
for (
|
||||
zone_name,
|
||||
zone_threshold,
|
||||
zone_alert_level,
|
||||
zone_description,
|
||||
) in zones:
|
||||
if water_level >= zone_threshold:
|
||||
alert_level = zone_alert_level
|
||||
threshold_value = zone_threshold
|
||||
alert_type = zone_description
|
||||
break
|
||||
|
||||
else:
|
||||
# Standard threshold checking for other stations
|
||||
if water_level >= station_thresholds.get("emergency", float("inf")):
|
||||
alert_level = AlertLevel.EMERGENCY
|
||||
threshold_value = station_thresholds["emergency"]
|
||||
alert_type = "Emergency Water Level"
|
||||
elif water_level >= station_thresholds.get(
|
||||
"critical", float("inf")
|
||||
):
|
||||
alert_level = AlertLevel.CRITICAL
|
||||
threshold_value = station_thresholds["critical"]
|
||||
alert_type = "Critical Water Level"
|
||||
elif water_level >= station_thresholds.get("warning", float("inf")):
|
||||
alert_level = AlertLevel.WARNING
|
||||
threshold_value = station_thresholds["warning"]
|
||||
alert_type = "High Water Level"
|
||||
|
||||
if alert_level:
|
||||
alert = WaterAlert(
|
||||
station_code=station_code,
|
||||
station_name=measurement.get(
|
||||
"station_name_th", f"Station {station_code}"
|
||||
),
|
||||
alert_type=alert_type,
|
||||
level=alert_level,
|
||||
water_level=water_level,
|
||||
threshold=threshold_value,
|
||||
discharge=measurement.get("discharge"),
|
||||
timestamp=measurement.get("timestamp"),
|
||||
)
|
||||
alerts.append(alert)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error checking water levels: {e}")
|
||||
|
||||
return alerts
|
||||
|
||||
def check_data_freshness(self, max_age_hours: int = 12) -> List[WaterAlert]:
|
||||
"""Check if data is fresh enough"""
|
||||
alerts = []
|
||||
|
||||
if not self.db_adapter:
|
||||
return alerts
|
||||
|
||||
try:
|
||||
measurements = self.db_adapter.get_latest_measurements(limit=20)
|
||||
cutoff_time = datetime.datetime.now() - datetime.timedelta(
|
||||
hours=max_age_hours
|
||||
)
|
||||
|
||||
for measurement in measurements:
|
||||
timestamp = measurement.get("timestamp")
|
||||
if timestamp and timestamp < cutoff_time:
|
||||
station_code = measurement.get("station_code", "UNKNOWN")
|
||||
|
||||
age_hours = (
|
||||
datetime.datetime.now() - timestamp
|
||||
).total_seconds() / 3600
|
||||
|
||||
alert = WaterAlert(
|
||||
station_code=station_code,
|
||||
station_name=measurement.get(
|
||||
"station_name_th", f"Station {station_code}"
|
||||
),
|
||||
alert_type="Stale Data",
|
||||
level=AlertLevel.WARNING,
|
||||
water_level=measurement.get("water_level", 0),
|
||||
threshold=max_age_hours,
|
||||
timestamp=timestamp,
|
||||
message=f"No fresh data for {age_hours:.1f} hours",
|
||||
)
|
||||
alerts.append(alert)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error checking data freshness: {e}")
|
||||
|
||||
return alerts
|
||||
|
||||
def check_rate_of_change(self, lookback_hours: int = 3) -> List[WaterAlert]:
|
||||
"""Check for rapid water level changes over recent hours"""
|
||||
alerts = []
|
||||
|
||||
if not self.db_adapter:
|
||||
return alerts
|
||||
|
||||
try:
|
||||
# Define rate-of-change thresholds (meters per hour)
|
||||
rate_thresholds = {
|
||||
"P.1": {
|
||||
"warning": 0.15, # 15cm/hour - moderate rise
|
||||
"critical": 0.25, # 25cm/hour - rapid rise
|
||||
"emergency": 0.40, # 40cm/hour - very rapid rise
|
||||
},
|
||||
"default": {"warning": 0.20, "critical": 0.35, "emergency": 0.50},
|
||||
}
|
||||
|
||||
# Get recent measurements for each station
|
||||
cutoff_time = datetime.datetime.now() - datetime.timedelta(
|
||||
hours=lookback_hours
|
||||
)
|
||||
|
||||
# Get unique stations from latest data
|
||||
latest = self.db_adapter.get_latest_measurements(limit=20)
|
||||
station_codes = set(
|
||||
m.get("station_code") for m in latest if m.get("station_code")
|
||||
)
|
||||
|
||||
for station_code in station_codes:
|
||||
try:
|
||||
# Only alert for upstream stations and Chiang Mai
|
||||
if station_code not in self.UPSTREAM_STATIONS:
|
||||
continue
|
||||
|
||||
# Get measurements for this station in the time window
|
||||
current_time = datetime.datetime.now()
|
||||
measurements = self.db_adapter.get_measurements_by_timerange(
|
||||
start_time=cutoff_time,
|
||||
end_time=current_time,
|
||||
station_codes=[station_code],
|
||||
)
|
||||
|
||||
if len(measurements) < 2:
|
||||
continue # Need at least 2 points to calculate rate
|
||||
|
||||
# Sort by timestamp
|
||||
measurements = sorted(
|
||||
measurements, key=lambda m: m.get("timestamp")
|
||||
)
|
||||
|
||||
# Get oldest and newest measurements
|
||||
oldest = measurements[0]
|
||||
newest = measurements[-1]
|
||||
|
||||
oldest_time = oldest.get("timestamp")
|
||||
oldest_level = oldest.get("water_level")
|
||||
newest_time = newest.get("timestamp")
|
||||
newest_level = newest.get("water_level")
|
||||
|
||||
# Convert timestamp strings to datetime if needed
|
||||
if isinstance(oldest_time, str):
|
||||
oldest_time = datetime.datetime.fromisoformat(oldest_time)
|
||||
if isinstance(newest_time, str):
|
||||
newest_time = datetime.datetime.fromisoformat(newest_time)
|
||||
|
||||
# Calculate rate of change
|
||||
time_diff_hours = (newest_time - oldest_time).total_seconds() / 3600
|
||||
if time_diff_hours == 0:
|
||||
continue
|
||||
|
||||
level_change = newest_level - oldest_level
|
||||
rate_per_hour = level_change / time_diff_hours
|
||||
|
||||
# Only alert on rising water (positive rate)
|
||||
if rate_per_hour <= 0:
|
||||
continue
|
||||
|
||||
# Get station info from latest data
|
||||
station_info = next(
|
||||
(m for m in latest if m.get("station_code") == station_code), {}
|
||||
)
|
||||
station_name = station_info.get("station_name_th", station_code)
|
||||
|
||||
# Get thresholds for this station
|
||||
station_rate_threshold = rate_thresholds.get(
|
||||
station_code, rate_thresholds["default"]
|
||||
)
|
||||
|
||||
alert_level = None
|
||||
threshold_value = None
|
||||
alert_type = None
|
||||
|
||||
if rate_per_hour >= station_rate_threshold["emergency"]:
|
||||
alert_level = AlertLevel.EMERGENCY
|
||||
threshold_value = station_rate_threshold["emergency"]
|
||||
alert_type = "Very Rapid Water Level Rise"
|
||||
elif rate_per_hour >= station_rate_threshold["critical"]:
|
||||
alert_level = AlertLevel.CRITICAL
|
||||
threshold_value = station_rate_threshold["critical"]
|
||||
alert_type = "Rapid Water Level Rise"
|
||||
elif rate_per_hour >= station_rate_threshold["warning"]:
|
||||
alert_level = AlertLevel.WARNING
|
||||
threshold_value = station_rate_threshold["warning"]
|
||||
alert_type = "Moderate Water Level Rise"
|
||||
|
||||
if alert_level:
|
||||
message = (
|
||||
f"Rising at {rate_per_hour:.2f}m/h over last {time_diff_hours:.1f}h "
|
||||
f"(change: {level_change:+.2f}m)"
|
||||
)
|
||||
|
||||
alert = WaterAlert(
|
||||
station_code=station_code,
|
||||
station_name=station_name or f"Station {station_code}",
|
||||
alert_type=alert_type,
|
||||
level=alert_level,
|
||||
water_level=newest_level,
|
||||
threshold=threshold_value,
|
||||
timestamp=newest_time,
|
||||
message=message,
|
||||
)
|
||||
alerts.append(alert)
|
||||
|
||||
except Exception as station_error:
|
||||
logger.debug(
|
||||
f"Error checking rate of change for station {station_code}: {station_error}"
|
||||
)
|
||||
continue
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error checking rate of change: {e}")
|
||||
|
||||
return alerts
|
||||
|
||||
def send_alerts(self, alerts: List[WaterAlert]) -> int:
|
||||
"""Send alerts via configured channels"""
|
||||
sent_count = 0
|
||||
|
||||
if not alerts:
|
||||
return sent_count
|
||||
|
||||
if self.matrix_notifier:
|
||||
for alert in alerts:
|
||||
if self.matrix_notifier.send_alert(alert):
|
||||
sent_count += 1
|
||||
|
||||
# Could add other notification channels here:
|
||||
# - Email
|
||||
# - Discord
|
||||
# - Telegram
|
||||
# - SMS
|
||||
|
||||
return sent_count
|
||||
|
||||
def run_alert_check(self) -> Dict[str, int]:
|
||||
"""Run complete alert check cycle"""
|
||||
if not self.connect_database():
|
||||
return {"error": 1}
|
||||
|
||||
# Check water levels
|
||||
water_alerts = self.check_water_levels()
|
||||
|
||||
# Check data freshness
|
||||
data_alerts = self.check_data_freshness()
|
||||
|
||||
# Check rate of change (rapid rises)
|
||||
rate_alerts = self.check_rate_of_change()
|
||||
|
||||
# Combine alerts
|
||||
all_alerts = water_alerts + rate_alerts
|
||||
|
||||
# Send alerts
|
||||
sent_count = self.send_alerts(all_alerts)
|
||||
|
||||
logger.info(
|
||||
f"Alert check complete: {len(all_alerts)} alerts, {sent_count} sent"
|
||||
)
|
||||
|
||||
return {
|
||||
"water_alerts": len(water_alerts),
|
||||
"data_alerts": len(data_alerts),
|
||||
"rate_alerts": len(rate_alerts),
|
||||
"total_alerts": len(all_alerts),
|
||||
"sent": sent_count,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
"""Standalone alerting check"""
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description="Water Level Alert System")
|
||||
parser.add_argument("--check", action="store_true", help="Run alert check")
|
||||
parser.add_argument("--test", action="store_true", help="Send test message")
|
||||
args = parser.parse_args()
|
||||
|
||||
alerting = WaterLevelAlertSystem()
|
||||
|
||||
if args.test:
|
||||
if alerting.matrix_notifier:
|
||||
test_message = (
|
||||
"🧪 **Test Alert**\n\nThis is a test message from the Water Level Alert System.\n\n"
|
||||
"If you received this, Matrix notifications are working correctly!"
|
||||
)
|
||||
success = alerting.matrix_notifier.send_message(test_message)
|
||||
print(f"Test message sent: {success}")
|
||||
else:
|
||||
print("Matrix notifier not configured")
|
||||
|
||||
elif args.check:
|
||||
results = alerting.run_alert_check()
|
||||
print(f"Alert check results: {results}")
|
||||
|
||||
else:
|
||||
print("Use --check or --test")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+31
-342
@@ -1,354 +1,43 @@
|
||||
import os
|
||||
from typing import Any, Dict
|
||||
"""Database settings for the Python tools (training, evaluation, backfills).
|
||||
|
||||
The service itself is ping-collector (rust/collector); it reads the same
|
||||
environment / `.env` file (rust/collector/src/config.rs). PostgreSQL only:
|
||||
POSTGRES_CONNECTION_STRING, or POSTGRES_HOST / POSTGRES_PORT / POSTGRES_DB /
|
||||
POSTGRES_USER / POSTGRES_PASSWORD with the same defaults as ping-collector.
|
||||
"""
|
||||
|
||||
import os
|
||||
import urllib.parse
|
||||
from typing import Optional
|
||||
|
||||
# Load environment variables from .env file
|
||||
try:
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
except ImportError:
|
||||
# python-dotenv not installed, continue without it
|
||||
except ImportError: # the systemd units pass the variables in directly
|
||||
pass
|
||||
|
||||
try:
|
||||
from .exceptions import ConfigurationError
|
||||
from .models import DatabaseType
|
||||
except ImportError:
|
||||
# Handle case when running as standalone script
|
||||
class ConfigurationError(Exception):
|
||||
pass
|
||||
|
||||
from enum import Enum
|
||||
|
||||
class DatabaseType(Enum):
|
||||
SQLITE = "sqlite"
|
||||
MYSQL = "mysql"
|
||||
POSTGRESQL = "postgresql"
|
||||
INFLUXDB = "influxdb"
|
||||
VICTORIAMETRICS = "victoriametrics"
|
||||
|
||||
|
||||
class Config:
|
||||
"""Configuration class for the Water Level Monitor"""
|
||||
|
||||
# Database settings
|
||||
DATABASE_PATH = os.getenv("WATER_DB_PATH", "water_levels.db")
|
||||
|
||||
# Website settings
|
||||
TARGET_URL = "https://hyd-app-db.rid.go.th/hydro1h.html"
|
||||
API_URL = "https://hyd-app-db.rid.go.th/webservice/getGroupHourlyWaterLevelReportAllHL.ashx"
|
||||
THAIWATER_API_KEY = os.getenv("THAIWATER_API_KEY")
|
||||
|
||||
# Public flood notifications (ntfy). Off unless NTFY_SERVER is set.
|
||||
# NTFY_SERVER is what subscribers use (public https URL, shown on the
|
||||
# dashboard). NTFY_PUBLISH_URL is where the monitor POSTs; defaults to
|
||||
# NTFY_SERVER, set it to http://127.0.0.1:2586 when ntfy runs on the same
|
||||
# host so publishing never depends on DNS/proxy/tunnel being up.
|
||||
NTFY_SERVER = os.getenv("NTFY_SERVER", "").strip()
|
||||
NTFY_PUBLISH_URL = os.getenv("NTFY_PUBLISH_URL", "").strip() or NTFY_SERVER
|
||||
NTFY_TOPIC_PREFIX = os.getenv("NTFY_TOPIC_PREFIX", "ping").strip()
|
||||
NTFY_TOKEN = os.getenv("NTFY_TOKEN", "").strip() # publish token if ACL enabled
|
||||
PUBLIC_URL = os.getenv("PUBLIC_URL", "https://water.buildfor.life/").strip()
|
||||
REQUEST_TIMEOUT = int(os.getenv("REQUEST_TIMEOUT", "30"))
|
||||
USER_AGENT = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
|
||||
def database_url() -> Optional[str]:
|
||||
"""The PostgreSQL connection string, or None when none is configured."""
|
||||
url = os.getenv("POSTGRES_CONNECTION_STRING", "").strip()
|
||||
if url:
|
||||
return url
|
||||
password = os.getenv("POSTGRES_PASSWORD")
|
||||
if not password:
|
||||
return None
|
||||
user = urllib.parse.quote(os.getenv("POSTGRES_USER", "postgres"), safe="")
|
||||
host = os.getenv("POSTGRES_HOST", "localhost")
|
||||
port = os.getenv("POSTGRES_PORT", "5432")
|
||||
name = os.getenv("POSTGRES_DB", "water_monitoring")
|
||||
return (
|
||||
f"postgresql://{user}:{urllib.parse.quote(password, safe='')}"
|
||||
f"@{host}:{port}/{name}"
|
||||
)
|
||||
|
||||
# Database configuration
|
||||
# When DB_TYPE is not set explicitly, a configured Postgres connection wins over the sqlite default
|
||||
DB_TYPE = os.getenv(
|
||||
"DB_TYPE",
|
||||
"postgresql" if os.getenv("POSTGRES_CONNECTION_STRING") else "sqlite",
|
||||
).lower()
|
||||
|
||||
# VictoriaMetrics settings
|
||||
# Default to localhost; set VM_HOST in the environment for real deployments
|
||||
# (avoids committing infrastructure hostnames to the repo).
|
||||
VM_HOST = os.getenv("VM_HOST", "localhost")
|
||||
VM_PORT = int(os.getenv("VM_PORT", "443"))
|
||||
|
||||
# Support for HTTPS URLs (e.g., behind reverse proxy)
|
||||
VM_URL = os.getenv("VM_URL") # Full URL override (e.g., https://vm.example.com)
|
||||
|
||||
# InfluxDB settings
|
||||
INFLUX_HOST = os.getenv("INFLUX_HOST", "localhost")
|
||||
INFLUX_PORT = int(os.getenv("INFLUX_PORT", "8086"))
|
||||
INFLUX_DATABASE = os.getenv("INFLUX_DATABASE", "water_monitoring")
|
||||
INFLUX_USERNAME = os.getenv("INFLUX_USERNAME")
|
||||
INFLUX_PASSWORD = os.getenv("INFLUX_PASSWORD")
|
||||
|
||||
# PostgreSQL settings
|
||||
POSTGRES_CONNECTION_STRING = os.getenv("POSTGRES_CONNECTION_STRING")
|
||||
POSTGRES_HOST = os.getenv("POSTGRES_HOST", "localhost")
|
||||
POSTGRES_PORT = int(os.getenv("POSTGRES_PORT", "5432"))
|
||||
POSTGRES_DB = os.getenv("POSTGRES_DB", "water_monitoring")
|
||||
POSTGRES_USER = os.getenv("POSTGRES_USER", "postgres")
|
||||
POSTGRES_PASSWORD = os.getenv("POSTGRES_PASSWORD")
|
||||
|
||||
# MySQL settings
|
||||
MYSQL_CONNECTION_STRING = os.getenv("MYSQL_CONNECTION_STRING")
|
||||
|
||||
# HII/ThaiWater open api-v3 collection (rainfall + backup water level)
|
||||
# See docs/DATA_SOURCES.md. Requires a SQL DB_TYPE (sqlite/postgresql/mysql).
|
||||
ENABLE_HII_COLLECTION = os.getenv("ENABLE_HII_COLLECTION", "true").lower() in (
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
)
|
||||
HII_BASIN_CODE = int(os.getenv("HII_BASIN_CODE", "6")) # 6 = Ping Basin
|
||||
|
||||
# RID large-dam daily status (app.rid.go.th/reservoir) — Mae Ngat et al.
|
||||
ENABLE_RESERVOIR_COLLECTION = os.getenv(
|
||||
"ENABLE_RESERVOIR_COLLECTION", "true"
|
||||
).lower() in ("1", "true", "yes")
|
||||
# TTL for the /api/hii/*/latest response cache; source data changes hourly
|
||||
HII_CACHE_TTL_SECONDS = int(os.getenv("HII_CACHE_TTL_SECONDS", "120"))
|
||||
# TTL for the /measurements/latest response cache (hottest endpoint)
|
||||
LATEST_CACHE_TTL_SECONDS = int(os.getenv("LATEST_CACHE_TTL_SECONDS", "45"))
|
||||
|
||||
# TTL for /health check results (includes an external RID-API probe)
|
||||
HEALTH_CACHE_TTL_SECONDS = int(os.getenv("HEALTH_CACHE_TTL_SECONDS", "30"))
|
||||
|
||||
# Thread-pool size for blocking work in the web process (DB queries,
|
||||
# inference, health probes). Waiting threads are cheap; starving the pool
|
||||
# stalls every endpoint that needs a thread.
|
||||
EXECUTOR_THREADS = int(os.getenv("EXECUTOR_THREADS", "48"))
|
||||
|
||||
# Web server worker processes. Above 1, uvicorn forks workers and a
|
||||
# localhost lock port elects a single background-collection leader.
|
||||
WEB_WORKERS = int(os.getenv("WEB_WORKERS", "2"))
|
||||
COLLECTION_LEADER_PORT = int(os.getenv("COLLECTION_LEADER_PORT", "8901"))
|
||||
|
||||
# Umami analytics (self-hosted). The website id is public (it ships in the
|
||||
# dashboard <script> tag); server-side API tracking posts to /api/send.
|
||||
UMAMI_API_URL = os.getenv("UMAMI_API_URL", "https://stats.buildfor.life/api/send")
|
||||
UMAMI_WEBSITE_ID = os.getenv(
|
||||
"UMAMI_WEBSITE_ID", "00b2be73-8f5f-4400-9029-3be852eb08f7"
|
||||
)
|
||||
UMAMI_TRACK_API = os.getenv("UMAMI_TRACK_API", "true").lower() in (
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
)
|
||||
|
||||
# Scheduler settings
|
||||
SCRAPING_INTERVAL_HOURS = int(os.getenv("SCRAPING_INTERVAL_HOURS", "1"))
|
||||
# RID is polled this many minutes past each hour; while the current hour is
|
||||
# not published yet (the evening report often lags 1-2 h) it is retried
|
||||
# every RID_RETRY_MINUTES. Late readings are stored either way.
|
||||
RID_POLL_OFFSET_MINUTES = int(os.getenv("RID_POLL_OFFSET_MINUTES", "5"))
|
||||
RID_RETRY_MINUTES = int(os.getenv("RID_RETRY_MINUTES", "10"))
|
||||
# On startup the collection leader re-fetches RID reports with holes from
|
||||
# the last N days (0 = off). Older history: `run.py --fill-gaps all`.
|
||||
GAP_FILL_DAYS_ON_START = int(os.getenv("GAP_FILL_DAYS_ON_START", "14"))
|
||||
# Leader -> other workers hand-off for /scraping/status (default: temp dir).
|
||||
# With ping-collector, point both services at the same file inside the app
|
||||
# dir (the systemd units use PrivateTmp).
|
||||
SCRAPING_STATUS_FILE = os.getenv("SCRAPING_STATUS_FILE", "")
|
||||
# false: data collection runs in the Rust ping-collector service
|
||||
# (rust/collector); this app then only serves, forecasts and notifies,
|
||||
# re-forecasting when the newest RID reading advances (polled every
|
||||
# FORECAST_POLL_SECONDS) and at least once per clock hour.
|
||||
COLLECTION_ENABLED = os.getenv("COLLECTION_ENABLED", "true").lower() in (
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
)
|
||||
FORECAST_POLL_SECONDS = int(os.getenv("FORECAST_POLL_SECONDS", "60"))
|
||||
# true: ping-collector also forecasts (rust/forecast) and sends the ntfy
|
||||
# notifications; this app serves its stored forecast (forecast_current)
|
||||
# and does neither. Read by both services from the shared .env.
|
||||
FORECAST_IN_COLLECTOR = os.getenv("FORECAST_IN_COLLECTOR", "false").lower() in (
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
)
|
||||
# A stored forecast older than this is not served; /forecast then computes
|
||||
# in-process as before (the collector is down or has no models).
|
||||
FORECAST_MAX_AGE_SECONDS = int(os.getenv("FORECAST_MAX_AGE_SECONDS", "7200"))
|
||||
|
||||
# Logging settings
|
||||
LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO")
|
||||
LOG_FILE = os.getenv("LOG_FILE", "water_monitor.log")
|
||||
LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s"
|
||||
|
||||
# Data retention
|
||||
DATA_RETENTION_DAYS = int(os.getenv("DATA_RETENTION_DAYS", "365"))
|
||||
|
||||
# Retry settings
|
||||
MAX_RETRIES = int(os.getenv("MAX_RETRIES", "3"))
|
||||
RETRY_DELAY_SECONDS = int(os.getenv("RETRY_DELAY_SECONDS", "60"))
|
||||
|
||||
# Station configuration
|
||||
# Runtime-writable JSON file that persists the station mapping across restarts
|
||||
# (station CRUD via the API writes here). If it does not exist, the bundled
|
||||
# defaults in src/data/stations.json are used to seed it.
|
||||
STATION_CONFIG_PATH = os.getenv("STATION_CONFIG_PATH", "stations.json")
|
||||
|
||||
# Web API / CORS settings
|
||||
# Comma-separated list of allowed origins. Defaults to none (same-origin only);
|
||||
# set CORS_ALLOW_ORIGINS to a specific list of front-end origins in production.
|
||||
# Credentials are only enabled when explicit (non-wildcard) origins are set,
|
||||
# because "*" + credentials is rejected by browsers and unsafe.
|
||||
CORS_ALLOW_ORIGINS = [
|
||||
origin.strip()
|
||||
for origin in os.getenv("CORS_ALLOW_ORIGINS", "").split(",")
|
||||
if origin.strip()
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def validate_config(cls) -> bool:
|
||||
"""Validate configuration settings"""
|
||||
errors = []
|
||||
|
||||
# Validate database type
|
||||
try:
|
||||
DatabaseType(cls.DB_TYPE)
|
||||
except ValueError:
|
||||
errors.append(f"Invalid DB_TYPE: {cls.DB_TYPE}")
|
||||
|
||||
# Validate database-specific settings
|
||||
if cls.DB_TYPE == "victoriametrics":
|
||||
if not cls.VM_HOST:
|
||||
errors.append("VM_HOST is required for VictoriaMetrics")
|
||||
if not isinstance(cls.VM_PORT, int) or cls.VM_PORT <= 0:
|
||||
errors.append("VM_PORT must be a positive integer")
|
||||
|
||||
elif cls.DB_TYPE == "influxdb":
|
||||
if not cls.INFLUX_HOST:
|
||||
errors.append("INFLUX_HOST is required for InfluxDB")
|
||||
if not cls.INFLUX_DATABASE:
|
||||
errors.append("INFLUX_DATABASE is required for InfluxDB")
|
||||
|
||||
elif cls.DB_TYPE in ["postgresql", "mysql"]:
|
||||
if cls.DB_TYPE == "postgresql":
|
||||
# Check if either connection string or individual components are provided
|
||||
if not cls.POSTGRES_CONNECTION_STRING:
|
||||
# If no connection string, check individual components
|
||||
if not cls.POSTGRES_HOST:
|
||||
errors.append("POSTGRES_HOST is required for PostgreSQL")
|
||||
if not cls.POSTGRES_USER:
|
||||
errors.append("POSTGRES_USER is required for PostgreSQL")
|
||||
if not cls.POSTGRES_PASSWORD:
|
||||
errors.append("POSTGRES_PASSWORD is required for PostgreSQL")
|
||||
if not cls.POSTGRES_DB:
|
||||
errors.append("POSTGRES_DB is required for PostgreSQL")
|
||||
else: # mysql
|
||||
if not cls.MYSQL_CONNECTION_STRING:
|
||||
errors.append("MYSQL_CONNECTION_STRING is required for MySQL")
|
||||
|
||||
# Validate numeric settings
|
||||
if cls.SCRAPING_INTERVAL_HOURS <= 0:
|
||||
errors.append("SCRAPING_INTERVAL_HOURS must be positive")
|
||||
|
||||
if not 0 <= cls.RID_POLL_OFFSET_MINUTES < 60:
|
||||
errors.append("RID_POLL_OFFSET_MINUTES must be between 0 and 59")
|
||||
|
||||
if cls.RID_RETRY_MINUTES <= 0:
|
||||
errors.append("RID_RETRY_MINUTES must be positive")
|
||||
|
||||
if cls.DATA_RETENTION_DAYS <= 0:
|
||||
errors.append("DATA_RETENTION_DAYS must be positive")
|
||||
|
||||
if errors:
|
||||
raise ConfigurationError(f"Configuration errors: {'; '.join(errors)}")
|
||||
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def get_database_config(cls) -> Dict[str, Any]:
|
||||
"""Returns database configuration based on DB_TYPE"""
|
||||
if cls.DB_TYPE == "victoriametrics":
|
||||
return {"type": "victoriametrics", "host": cls.VM_HOST, "port": cls.VM_PORT}
|
||||
elif cls.DB_TYPE == "influxdb":
|
||||
return {
|
||||
"type": "influxdb",
|
||||
"host": cls.INFLUX_HOST,
|
||||
"port": cls.INFLUX_PORT,
|
||||
"database": cls.INFLUX_DATABASE,
|
||||
"username": cls.INFLUX_USERNAME,
|
||||
"password": cls.INFLUX_PASSWORD,
|
||||
}
|
||||
elif cls.DB_TYPE == "postgresql":
|
||||
# Use individual components if POSTGRES_CONNECTION_STRING is not provided
|
||||
if cls.POSTGRES_CONNECTION_STRING:
|
||||
return {
|
||||
"type": "postgresql",
|
||||
"connection_string": cls.POSTGRES_CONNECTION_STRING,
|
||||
}
|
||||
else:
|
||||
# Build connection string from components (automatically URL-encodes password)
|
||||
import urllib.parse
|
||||
|
||||
if not cls.POSTGRES_PASSWORD:
|
||||
raise ConfigurationError(
|
||||
"POSTGRES_PASSWORD is required for PostgreSQL (no default is provided)"
|
||||
)
|
||||
password = urllib.parse.quote(cls.POSTGRES_PASSWORD, safe="")
|
||||
connection_string = (
|
||||
f"postgresql://{cls.POSTGRES_USER}:{password}"
|
||||
f"@{cls.POSTGRES_HOST}:{cls.POSTGRES_PORT}/{cls.POSTGRES_DB}"
|
||||
)
|
||||
return {"type": "postgresql", "connection_string": connection_string}
|
||||
elif cls.DB_TYPE == "mysql":
|
||||
if not cls.MYSQL_CONNECTION_STRING:
|
||||
raise ConfigurationError(
|
||||
"MYSQL_CONNECTION_STRING is required for MySQL (no default is provided)"
|
||||
)
|
||||
return {"type": "mysql", "connection_string": cls.MYSQL_CONNECTION_STRING}
|
||||
else: # sqlite
|
||||
return {
|
||||
"type": "sqlite",
|
||||
"connection_string": f"sqlite:///{cls.DATABASE_PATH}",
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def get_all_settings(cls) -> Dict[str, Any]:
|
||||
"""Returns all configuration settings"""
|
||||
return {
|
||||
"DB_TYPE": cls.DB_TYPE,
|
||||
"DATABASE_PATH": cls.DATABASE_PATH,
|
||||
"TARGET_URL": cls.TARGET_URL,
|
||||
"API_URL": cls.API_URL,
|
||||
"REQUEST_TIMEOUT": cls.REQUEST_TIMEOUT,
|
||||
"SCRAPING_INTERVAL_HOURS": cls.SCRAPING_INTERVAL_HOURS,
|
||||
"RID_POLL_OFFSET_MINUTES": cls.RID_POLL_OFFSET_MINUTES,
|
||||
"RID_RETRY_MINUTES": cls.RID_RETRY_MINUTES,
|
||||
"GAP_FILL_DAYS_ON_START": cls.GAP_FILL_DAYS_ON_START,
|
||||
"COLLECTION_ENABLED": cls.COLLECTION_ENABLED,
|
||||
"FORECAST_IN_COLLECTOR": cls.FORECAST_IN_COLLECTOR,
|
||||
"LOG_LEVEL": cls.LOG_LEVEL,
|
||||
"LOG_FILE": cls.LOG_FILE,
|
||||
"DATA_RETENTION_DAYS": cls.DATA_RETENTION_DAYS,
|
||||
"MAX_RETRIES": cls.MAX_RETRIES,
|
||||
"RETRY_DELAY_SECONDS": cls.RETRY_DELAY_SECONDS,
|
||||
"VM_HOST": cls.VM_HOST,
|
||||
"VM_PORT": cls.VM_PORT,
|
||||
"INFLUX_HOST": cls.INFLUX_HOST,
|
||||
"INFLUX_PORT": cls.INFLUX_PORT,
|
||||
"INFLUX_DATABASE": cls.INFLUX_DATABASE,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def print_settings(cls):
|
||||
"""Prints all current settings"""
|
||||
print("=== Water Level Monitor Configuration ===")
|
||||
for key, value in cls.get_all_settings().items():
|
||||
# Hide sensitive information
|
||||
if "PASSWORD" in key and value:
|
||||
value = "*" * len(str(value))
|
||||
print(f"{key}: {value}")
|
||||
print("=" * 45)
|
||||
|
||||
print("\nDatabase Configuration:")
|
||||
db_config = cls.get_database_config()
|
||||
for key, value in db_config.items():
|
||||
if "password" in key and value:
|
||||
value = "*" * len(str(value))
|
||||
print(f" {key}: {value}")
|
||||
print("=" * 45)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
Config.print_settings()
|
||||
url = database_url()
|
||||
if url and "@" in url:
|
||||
url = url.split("://", 1)[0] + "://***@" + url.rsplit("@", 1)[1]
|
||||
print(url or "no database configured")
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,352 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Demo script showing different database backend options for water monitoring
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import os
|
||||
import sys
|
||||
|
||||
from water_scraper_v3 import EnhancedWaterMonitorScraper
|
||||
|
||||
|
||||
def demo_sqlite():
|
||||
"""Demo with SQLite (local development)"""
|
||||
print("=" * 60)
|
||||
print("🗄️ SQLite Demo (Local Development)")
|
||||
print("=" * 60)
|
||||
|
||||
config = {"type": "sqlite", "connection_string": "sqlite:///demo_water_sqlite.db"}
|
||||
|
||||
try:
|
||||
scraper = EnhancedWaterMonitorScraper(config)
|
||||
|
||||
# Fetch and save data
|
||||
print("Fetching data from API...")
|
||||
data = scraper.fetch_water_data()
|
||||
|
||||
if data:
|
||||
print(f"✓ Fetched {len(data)} data points")
|
||||
success = scraper.save_to_database(data)
|
||||
|
||||
if success:
|
||||
print("✓ Data saved to SQLite database")
|
||||
|
||||
# Show latest data
|
||||
latest = scraper.get_latest_data(5)
|
||||
print(f"\nLatest 5 measurements:")
|
||||
for measurement in latest:
|
||||
print(
|
||||
f" • {measurement['station_code']} ({measurement['station_name_en']}): "
|
||||
f"{measurement['water_level']:.2f}m, {measurement['discharge']:.1f} cms"
|
||||
)
|
||||
else:
|
||||
print("✗ Failed to save data")
|
||||
else:
|
||||
print("✗ No data fetched")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
|
||||
|
||||
def demo_influxdb():
|
||||
"""Demo with InfluxDB (requires InfluxDB running)"""
|
||||
print("\n" + "=" * 60)
|
||||
print("📊 InfluxDB Demo (Time-Series Database)")
|
||||
print("=" * 60)
|
||||
|
||||
config = {
|
||||
"type": "influxdb",
|
||||
"host": "localhost",
|
||||
"port": 8086,
|
||||
"database": "water_monitoring_demo",
|
||||
"username": None, # Set if authentication is enabled
|
||||
"password": None,
|
||||
}
|
||||
|
||||
try:
|
||||
scraper = EnhancedWaterMonitorScraper(config)
|
||||
|
||||
if scraper.db_adapter and scraper.db_adapter.client:
|
||||
print("✓ Connected to InfluxDB")
|
||||
|
||||
# Fetch and save data
|
||||
print("Fetching data from API...")
|
||||
data = scraper.fetch_water_data()
|
||||
|
||||
if data:
|
||||
print(f"✓ Fetched {len(data)} data points")
|
||||
success = scraper.save_to_database(data)
|
||||
|
||||
if success:
|
||||
print("✓ Data saved to InfluxDB")
|
||||
print("💡 You can now query this data in Grafana or InfluxDB CLI")
|
||||
print(
|
||||
" Example query: SELECT * FROM water_data ORDER BY time DESC LIMIT 10"
|
||||
)
|
||||
else:
|
||||
print("✗ Failed to save data")
|
||||
else:
|
||||
print("✗ No data fetched")
|
||||
else:
|
||||
print("✗ Could not connect to InfluxDB")
|
||||
print(
|
||||
"💡 Make sure InfluxDB is running: docker run -p 8086:8086 influxdb:1.8"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
print("💡 InfluxDB might not be running or accessible")
|
||||
|
||||
|
||||
def demo_postgresql():
|
||||
"""Demo with PostgreSQL (requires PostgreSQL running)"""
|
||||
print("\n" + "=" * 60)
|
||||
print("🐘 PostgreSQL Demo (Relational Database)")
|
||||
print("=" * 60)
|
||||
|
||||
config = {
|
||||
"type": "postgresql",
|
||||
"connection_string": "postgresql://postgres:password@localhost:5432/water_monitoring",
|
||||
}
|
||||
|
||||
try:
|
||||
scraper = EnhancedWaterMonitorScraper(config)
|
||||
|
||||
if scraper.db_adapter and scraper.db_adapter.engine:
|
||||
print("✓ Connected to PostgreSQL")
|
||||
|
||||
# Fetch and save data
|
||||
print("Fetching data from API...")
|
||||
data = scraper.fetch_water_data()
|
||||
|
||||
if data:
|
||||
print(f"✓ Fetched {len(data)} data points")
|
||||
success = scraper.save_to_database(data)
|
||||
|
||||
if success:
|
||||
print("✓ Data saved to PostgreSQL")
|
||||
print("💡 You can now query this data with SQL")
|
||||
print(
|
||||
" Example: SELECT * FROM water_measurements ORDER BY timestamp DESC LIMIT 10;"
|
||||
)
|
||||
else:
|
||||
print("✗ Failed to save data")
|
||||
else:
|
||||
print("✗ No data fetched")
|
||||
else:
|
||||
print("✗ Could not connect to PostgreSQL")
|
||||
print("💡 Make sure PostgreSQL is running with correct credentials")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
print("💡 PostgreSQL might not be running or credentials might be wrong")
|
||||
|
||||
|
||||
def demo_mysql():
|
||||
"""Demo with MySQL (requires MySQL running)"""
|
||||
print("\n" + "=" * 60)
|
||||
print("🐬 MySQL Demo (Relational Database)")
|
||||
print("=" * 60)
|
||||
|
||||
config = {
|
||||
"type": "mysql",
|
||||
"connection_string": "mysql://root:password@localhost:3306/water_monitoring",
|
||||
}
|
||||
|
||||
try:
|
||||
scraper = EnhancedWaterMonitorScraper(config)
|
||||
|
||||
if scraper.db_adapter and scraper.db_adapter.engine:
|
||||
print("✓ Connected to MySQL")
|
||||
|
||||
# Fetch and save data
|
||||
print("Fetching data from API...")
|
||||
data = scraper.fetch_water_data()
|
||||
|
||||
if data:
|
||||
print(f"✓ Fetched {len(data)} data points")
|
||||
success = scraper.save_to_database(data)
|
||||
|
||||
if success:
|
||||
print("✓ Data saved to MySQL")
|
||||
print("💡 You can now query this data with SQL")
|
||||
print(
|
||||
" Example: SELECT * FROM water_measurements ORDER BY timestamp DESC LIMIT 10;"
|
||||
)
|
||||
else:
|
||||
print("✗ Failed to save data")
|
||||
else:
|
||||
print("✗ No data fetched")
|
||||
else:
|
||||
print("✗ Could not connect to MySQL")
|
||||
print("💡 Make sure MySQL is running with correct credentials")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
print("💡 MySQL might not be running or credentials might be wrong")
|
||||
|
||||
|
||||
def demo_victoriametrics():
|
||||
"""Demo with VictoriaMetrics (supports both local and HTTPS configurations)"""
|
||||
print("\n" + "=" * 60)
|
||||
print("⚡ VictoriaMetrics Demo (High-Performance Metrics)")
|
||||
print("=" * 60)
|
||||
|
||||
# Use configuration from environment or config.py
|
||||
from config import Config
|
||||
|
||||
db_config = Config.get_database_config()
|
||||
|
||||
if db_config["type"] != "victoriametrics":
|
||||
# Fallback to default local configuration
|
||||
config = {"type": "victoriametrics", "host": "vm.newedge.house", "port": 443}
|
||||
else:
|
||||
config = db_config
|
||||
|
||||
print(f"Connecting to: {config['host']}:{config['port']}")
|
||||
|
||||
try:
|
||||
scraper = EnhancedWaterMonitorScraper(config)
|
||||
|
||||
if scraper.db_adapter:
|
||||
# Test connection using the adapter's connect method
|
||||
if scraper.db_adapter.connect():
|
||||
print("✓ Connected to VictoriaMetrics")
|
||||
|
||||
# Fetch and save data
|
||||
print("Fetching data from API...")
|
||||
data = scraper.fetch_water_data()
|
||||
|
||||
if data:
|
||||
print(f"✓ Fetched {len(data)} data points")
|
||||
success = scraper.save_to_database(data)
|
||||
|
||||
if success:
|
||||
print("✓ Data saved to VictoriaMetrics")
|
||||
print("💡 You can now query this data via Prometheus API")
|
||||
|
||||
# Show appropriate query URL based on configuration
|
||||
base_url = scraper.db_adapter.base_url
|
||||
print(f" Example: {base_url}/api/v1/query?query=water_level")
|
||||
print(f" Health check: {base_url}/health")
|
||||
else:
|
||||
print("✗ Failed to save data")
|
||||
else:
|
||||
print("✗ No data fetched")
|
||||
else:
|
||||
print("✗ Could not connect to VictoriaMetrics")
|
||||
if config["host"] == "localhost":
|
||||
print("💡 Make sure VictoriaMetrics is running locally:")
|
||||
print(" docker run -p 8428:8428 victoriametrics/victoria-metrics")
|
||||
else:
|
||||
print(
|
||||
f"💡 Check if VictoriaMetrics is accessible at {config['host']}:{config['port']}"
|
||||
)
|
||||
print("💡 Verify HTTPS configuration and network connectivity")
|
||||
else:
|
||||
print("✗ Failed to initialize VictoriaMetrics adapter")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
print("💡 Check your VictoriaMetrics configuration and network connectivity")
|
||||
|
||||
|
||||
def show_recommendations():
|
||||
"""Show database recommendations"""
|
||||
print("\n" + "=" * 60)
|
||||
print("🏆 Database Recommendations")
|
||||
print("=" * 60)
|
||||
|
||||
recommendations = [
|
||||
{
|
||||
"name": "InfluxDB",
|
||||
"best_for": "Time-series data, Grafana dashboards",
|
||||
"pros": [
|
||||
"Purpose-built for time-series",
|
||||
"Great compression",
|
||||
"Built-in retention",
|
||||
],
|
||||
"cons": ["Learning curve", "Less flexible for complex queries"],
|
||||
"use_case": "Recommended for most water monitoring deployments",
|
||||
},
|
||||
{
|
||||
"name": "PostgreSQL + TimescaleDB",
|
||||
"best_for": "Complex queries, existing PostgreSQL infrastructure",
|
||||
"pros": ["Mature ecosystem", "SQL compatibility", "ACID compliance"],
|
||||
"cons": ["More complex setup", "Higher resource usage"],
|
||||
"use_case": "Best for organizations already using PostgreSQL",
|
||||
},
|
||||
{
|
||||
"name": "VictoriaMetrics",
|
||||
"best_for": "High-performance metrics, Prometheus compatibility",
|
||||
"pros": ["Extremely fast", "Low resource usage", "Better compression"],
|
||||
"cons": ["Newer ecosystem", "Less tooling"],
|
||||
"use_case": "Best for high-volume, performance-critical deployments",
|
||||
},
|
||||
{
|
||||
"name": "MySQL",
|
||||
"best_for": "Existing MySQL infrastructure, familiar SQL",
|
||||
"pros": ["Familiar", "Mature", "Wide support"],
|
||||
"cons": ["Not optimized for time-series", "Manual optimization needed"],
|
||||
"use_case": "Good for organizations with existing MySQL expertise",
|
||||
},
|
||||
]
|
||||
|
||||
for rec in recommendations:
|
||||
print(f"\n📊 {rec['name']}")
|
||||
print(f" Best for: {rec['best_for']}")
|
||||
print(f" Pros: {', '.join(rec['pros'])}")
|
||||
print(f" Cons: {', '.join(rec['cons'])}")
|
||||
print(f" 💡 {rec['use_case']}")
|
||||
|
||||
|
||||
def main():
|
||||
"""Main demo function"""
|
||||
print("🌊 Thailand Water Monitor - Database Backend Demo")
|
||||
print("This demo shows how to use different database backends")
|
||||
|
||||
# Always run SQLite demo (no external dependencies)
|
||||
demo_sqlite()
|
||||
|
||||
# Check for command line arguments to run specific demos
|
||||
if len(sys.argv) > 1:
|
||||
db_type = sys.argv[1].lower()
|
||||
|
||||
if db_type == "influxdb":
|
||||
demo_influxdb()
|
||||
elif db_type == "postgresql":
|
||||
demo_postgresql()
|
||||
elif db_type == "mysql":
|
||||
demo_mysql()
|
||||
elif db_type == "victoriametrics":
|
||||
demo_victoriametrics()
|
||||
elif db_type == "all":
|
||||
demo_influxdb()
|
||||
demo_postgresql()
|
||||
demo_mysql()
|
||||
demo_victoriametrics()
|
||||
else:
|
||||
print(f"\nUnknown database type: {db_type}")
|
||||
print(
|
||||
"Available options: influxdb, postgresql, mysql, victoriametrics, all"
|
||||
)
|
||||
else:
|
||||
print("\n💡 To test other databases, run:")
|
||||
print(" python demo_databases.py influxdb")
|
||||
print(" python demo_databases.py postgresql")
|
||||
print(" python demo_databases.py mysql")
|
||||
print(" python demo_databases.py victoriametrics")
|
||||
print(" python demo_databases.py all")
|
||||
|
||||
# Show recommendations
|
||||
show_recommendations()
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("✅ Demo completed!")
|
||||
print("📖 See DATABASE_DEPLOYMENT_GUIDE.md for production setup instructions")
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,46 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Custom exceptions for water monitoring system
|
||||
"""
|
||||
|
||||
|
||||
class WaterMonitorException(Exception):
|
||||
"""Base exception for water monitoring system"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class DatabaseConnectionError(WaterMonitorException):
|
||||
"""Raised when database connection fails"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class APIConnectionError(WaterMonitorException):
|
||||
"""Raised when API connection fails"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class DataValidationError(WaterMonitorException):
|
||||
"""Raised when data validation fails"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class ConfigurationError(WaterMonitorException):
|
||||
"""Raised when configuration is invalid"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class DataParsingError(WaterMonitorException):
|
||||
"""Raised when data parsing fails"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class RetryExhaustedError(WaterMonitorException):
|
||||
"""Raised when all retry attempts are exhausted"""
|
||||
|
||||
pass
|
||||
@@ -1,172 +0,0 @@
|
||||
"""Persistence for issued flood forecasts.
|
||||
|
||||
Every background precompute stores what the deployed model predicted at that
|
||||
moment — predicted 24/12/6 h peak, warning/danger probabilities, model
|
||||
version. Keyed by (as_of, station, horizon), so hourly data yields one row
|
||||
per station-horizon per hour regardless of how often the precompute runs.
|
||||
This is the operational record that lets "predicted vs actual" be graphed
|
||||
later without retraining historical models.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import logging
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ForecastHistoryStore:
|
||||
"""SQL store (sqlite / postgresql / mysql), same pattern as HiiStore."""
|
||||
|
||||
def __init__(self, connection_string: str, db_type: str):
|
||||
self.db_type = db_type.lower()
|
||||
if self.db_type not in ("sqlite", "postgresql", "mysql"):
|
||||
raise ValueError(
|
||||
f"Forecast history requires a SQL database, got '{db_type}'"
|
||||
)
|
||||
self.connection_string = connection_string
|
||||
self.engine = None
|
||||
|
||||
def connect(self) -> bool:
|
||||
try:
|
||||
from sqlalchemy import create_engine, text
|
||||
|
||||
self.engine = create_engine(self.connection_string, pool_pre_ping=True)
|
||||
ddl = """
|
||||
CREATE TABLE IF NOT EXISTS forecast_history (
|
||||
as_of TIMESTAMP NOT NULL,
|
||||
station_code VARCHAR(10) NOT NULL,
|
||||
horizon_hours INTEGER NOT NULL,
|
||||
predicted_max_level NUMERIC(8,3),
|
||||
p_warning NUMERIC(7,5),
|
||||
p_danger NUMERIC(7,5),
|
||||
current_level NUMERIC(8,3),
|
||||
model_version VARCHAR(64),
|
||||
source VARCHAR(16),
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (as_of, station_code, horizon_hours)
|
||||
)
|
||||
"""
|
||||
index = (
|
||||
"CREATE INDEX IF NOT EXISTS idx_forecast_history_station "
|
||||
"ON forecast_history(station_code, as_of)"
|
||||
)
|
||||
with self.engine.begin() as conn:
|
||||
conn.execute(text(ddl))
|
||||
if self.db_type != "mysql": # MySQL lacks IF NOT EXISTS for indexes
|
||||
conn.execute(text(index))
|
||||
return True
|
||||
except Exception as error:
|
||||
logger.error(f"ForecastHistoryStore failed to connect: {error}")
|
||||
self.engine = None
|
||||
return False
|
||||
|
||||
def save_rows(self, rows: List[Dict]) -> int:
|
||||
"""Upsert forecast rows as returned by ml.predict (idempotent)."""
|
||||
if not rows:
|
||||
return 0
|
||||
if not self.engine and not self.connect():
|
||||
return 0
|
||||
from sqlalchemy import text
|
||||
|
||||
cols = (
|
||||
"(as_of, station_code, horizon_hours, predicted_max_level, "
|
||||
"p_warning, p_danger, current_level, model_version, source)"
|
||||
)
|
||||
values = (
|
||||
"(:as_of, :station_code, :horizon_hours, :predicted_max_level, "
|
||||
":p_warning, :p_danger, :current_level, :model_version, :source)"
|
||||
)
|
||||
update_cols = (
|
||||
"predicted_max_level",
|
||||
"p_warning",
|
||||
"p_danger",
|
||||
"current_level",
|
||||
"model_version",
|
||||
"source",
|
||||
)
|
||||
if self.db_type == "mysql":
|
||||
updates = ", ".join(f"{c} = VALUES({c})" for c in update_cols)
|
||||
sql = (
|
||||
f"INSERT INTO forecast_history {cols} VALUES {values} "
|
||||
f"ON DUPLICATE KEY UPDATE {updates}"
|
||||
)
|
||||
else:
|
||||
updates = ", ".join(f"{c} = EXCLUDED.{c}" for c in update_cols)
|
||||
sql = (
|
||||
f"INSERT INTO forecast_history {cols} VALUES {values} "
|
||||
f"ON CONFLICT (as_of, station_code, horizon_hours) "
|
||||
f"DO UPDATE SET {updates}"
|
||||
)
|
||||
|
||||
params = []
|
||||
for row in rows:
|
||||
as_of = row.get("as_of")
|
||||
if isinstance(as_of, str):
|
||||
as_of = datetime.datetime.fromisoformat(as_of)
|
||||
if as_of is None or row.get("station_code") is None:
|
||||
continue
|
||||
params.append(
|
||||
{
|
||||
"as_of": as_of,
|
||||
"station_code": row["station_code"],
|
||||
"horizon_hours": row.get("horizon_hours"),
|
||||
"predicted_max_level": row.get("predicted_max_level"),
|
||||
"p_warning": row.get("p_warning"),
|
||||
"p_danger": row.get("p_danger"),
|
||||
"current_level": row.get("current_level"),
|
||||
"model_version": row.get("model_version"),
|
||||
"source": row.get("source"),
|
||||
}
|
||||
)
|
||||
if not params:
|
||||
return 0
|
||||
try:
|
||||
with self.engine.begin() as conn:
|
||||
conn.execute(text(sql), params)
|
||||
return len(params)
|
||||
except Exception as error:
|
||||
logger.error(f"ForecastHistoryStore save failed: {error}")
|
||||
return 0
|
||||
|
||||
def fetch(
|
||||
self,
|
||||
station_code: str,
|
||||
start: Optional[datetime.datetime] = None,
|
||||
end: Optional[datetime.datetime] = None,
|
||||
horizon_hours: Optional[int] = None,
|
||||
limit: int = 5000,
|
||||
) -> List[Dict]:
|
||||
"""Issued forecasts for one station, ascending by as_of."""
|
||||
if not self.engine and not self.connect():
|
||||
return []
|
||||
from sqlalchemy import text
|
||||
|
||||
clauses = ["station_code = :code"]
|
||||
params: Dict = {"code": station_code, "limit": limit}
|
||||
if start is not None:
|
||||
clauses.append("as_of >= :start")
|
||||
params["start"] = start
|
||||
if end is not None:
|
||||
clauses.append("as_of <= :end")
|
||||
params["end"] = end
|
||||
if horizon_hours is not None:
|
||||
clauses.append("horizon_hours = :horizon")
|
||||
params["horizon"] = horizon_hours
|
||||
sql = (
|
||||
"SELECT as_of, station_code, horizon_hours, predicted_max_level, "
|
||||
"p_warning, p_danger, current_level, model_version, source "
|
||||
f"FROM forecast_history WHERE {' AND '.join(clauses)} "
|
||||
"ORDER BY as_of ASC, horizon_hours ASC LIMIT :limit"
|
||||
)
|
||||
try:
|
||||
with self.engine.connect() as conn:
|
||||
rows = [dict(r._mapping) for r in conn.execute(text(sql), params)]
|
||||
for row in rows:
|
||||
for key, value in row.items():
|
||||
if hasattr(value, "is_finite"): # Decimal -> float
|
||||
row[key] = float(value)
|
||||
return rows
|
||||
except Exception as error:
|
||||
logger.error(f"ForecastHistoryStore fetch failed: {error}")
|
||||
return []
|
||||
@@ -1,290 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Health check system for water monitoring application
|
||||
"""
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta
|
||||
from enum import Enum
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HealthStatus(Enum):
|
||||
HEALTHY = "healthy"
|
||||
DEGRADED = "degraded"
|
||||
UNHEALTHY = "unhealthy"
|
||||
|
||||
|
||||
@dataclass
|
||||
class HealthCheckResult:
|
||||
"""Result of a health check"""
|
||||
|
||||
name: str
|
||||
status: HealthStatus
|
||||
message: str
|
||||
timestamp: datetime
|
||||
response_time_ms: Optional[float] = None
|
||||
details: Optional[Dict[str, Any]] = None
|
||||
|
||||
|
||||
class HealthCheck:
|
||||
"""Base health check class"""
|
||||
|
||||
def __init__(self, name: str, timeout_seconds: int = 30):
|
||||
self.name = name
|
||||
self.timeout_seconds = timeout_seconds
|
||||
|
||||
def check(self) -> HealthCheckResult:
|
||||
"""Perform the health check"""
|
||||
start_time = time.time()
|
||||
|
||||
try:
|
||||
result = self._perform_check()
|
||||
response_time = (time.time() - start_time) * 1000
|
||||
|
||||
return HealthCheckResult(
|
||||
name=self.name,
|
||||
status=result.get("status", HealthStatus.HEALTHY),
|
||||
message=result.get("message", "OK"),
|
||||
timestamp=datetime.now(),
|
||||
response_time_ms=response_time,
|
||||
details=result.get("details"),
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
response_time = (time.time() - start_time) * 1000
|
||||
logger.error(f"Health check {self.name} failed: {e}")
|
||||
|
||||
return HealthCheckResult(
|
||||
name=self.name,
|
||||
status=HealthStatus.UNHEALTHY,
|
||||
message=f"Check failed: {str(e)}",
|
||||
timestamp=datetime.now(),
|
||||
response_time_ms=response_time,
|
||||
)
|
||||
|
||||
def _perform_check(self) -> Dict[str, Any]:
|
||||
"""Override this method to implement the actual check"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class DatabaseHealthCheck(HealthCheck):
|
||||
"""Health check for database connectivity"""
|
||||
|
||||
def __init__(self, db_adapter, name: str = "database"):
|
||||
super().__init__(name)
|
||||
self.db_adapter = db_adapter
|
||||
|
||||
def _perform_check(self) -> Dict[str, Any]:
|
||||
if not self.db_adapter:
|
||||
return {
|
||||
"status": HealthStatus.UNHEALTHY,
|
||||
"message": "Database adapter not initialized",
|
||||
}
|
||||
|
||||
try:
|
||||
# Connect only when there is no live engine yet: connect() re-runs
|
||||
# the CREATE TABLE DDL suite, which is far too heavy per probe.
|
||||
if getattr(self.db_adapter, "engine", None) is None and hasattr(
|
||||
self.db_adapter, "connect"
|
||||
):
|
||||
connected = self.db_adapter.connect()
|
||||
if not connected:
|
||||
return {
|
||||
"status": HealthStatus.UNHEALTHY,
|
||||
"message": "Database connection failed",
|
||||
}
|
||||
|
||||
# Try to get latest data
|
||||
latest_data = self.db_adapter.get_latest_measurements(limit=1)
|
||||
|
||||
if latest_data:
|
||||
latest_timestamp = latest_data[0].get("timestamp")
|
||||
if isinstance(latest_timestamp, str):
|
||||
latest_timestamp = datetime.fromisoformat(
|
||||
latest_timestamp.replace("Z", "+00:00")
|
||||
)
|
||||
|
||||
# Check if data is recent (within last 2 hours)
|
||||
if datetime.now() - latest_timestamp.replace(tzinfo=None) > timedelta(
|
||||
hours=2
|
||||
):
|
||||
return {
|
||||
"status": HealthStatus.DEGRADED,
|
||||
"message": f"Latest data is old: {latest_timestamp}",
|
||||
"details": {"latest_data_timestamp": str(latest_timestamp)},
|
||||
}
|
||||
|
||||
return {
|
||||
"status": HealthStatus.HEALTHY,
|
||||
"message": "Database connection OK",
|
||||
"details": {
|
||||
"latest_data_count": len(latest_data),
|
||||
"latest_timestamp": (
|
||||
str(latest_data[0].get("timestamp")) if latest_data else None
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
return {
|
||||
"status": HealthStatus.UNHEALTHY,
|
||||
"message": f"Database check failed: {str(e)}",
|
||||
}
|
||||
|
||||
|
||||
class APIHealthCheck(HealthCheck):
|
||||
"""Health check for external API connectivity"""
|
||||
|
||||
def __init__(self, api_url: str, session, name: str = "api"):
|
||||
super().__init__(name)
|
||||
# A liveness probe should fail fast: the default 30s timeout meant a
|
||||
# slow upstream pinned executor threads for longer than the /health
|
||||
# cache TTL, so the pool never drained under load.
|
||||
self.timeout_seconds = 5
|
||||
self.api_url = api_url
|
||||
self.session = session
|
||||
|
||||
def _perform_check(self) -> Dict[str, Any]:
|
||||
try:
|
||||
# Simple GET request to check API availability
|
||||
response = self.session.get(self.api_url, timeout=self.timeout_seconds)
|
||||
|
||||
if response.status_code == 200:
|
||||
return {
|
||||
"status": HealthStatus.HEALTHY,
|
||||
"message": "API connection OK",
|
||||
"details": {
|
||||
"status_code": response.status_code,
|
||||
"response_size": len(response.content),
|
||||
},
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"status": HealthStatus.DEGRADED,
|
||||
"message": f"API returned status {response.status_code}",
|
||||
"details": {"status_code": response.status_code},
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
return {
|
||||
"status": HealthStatus.UNHEALTHY,
|
||||
"message": f"API check failed: {str(e)}",
|
||||
}
|
||||
|
||||
|
||||
class MemoryHealthCheck(HealthCheck):
|
||||
"""Health check for memory usage"""
|
||||
|
||||
def __init__(self, max_memory_mb: int = 1000, name: str = "memory"):
|
||||
super().__init__(name)
|
||||
self.max_memory_mb = max_memory_mb
|
||||
|
||||
def _perform_check(self) -> Dict[str, Any]:
|
||||
try:
|
||||
import psutil
|
||||
|
||||
process = psutil.Process()
|
||||
memory_info = process.memory_info()
|
||||
memory_mb = memory_info.rss / 1024 / 1024
|
||||
|
||||
if memory_mb > self.max_memory_mb:
|
||||
return {
|
||||
"status": HealthStatus.DEGRADED,
|
||||
"message": f"High memory usage: {memory_mb:.1f}MB",
|
||||
"details": {
|
||||
"memory_mb": memory_mb,
|
||||
"max_memory_mb": self.max_memory_mb,
|
||||
},
|
||||
}
|
||||
|
||||
return {
|
||||
"status": HealthStatus.HEALTHY,
|
||||
"message": f"Memory usage OK: {memory_mb:.1f}MB",
|
||||
"details": {"memory_mb": memory_mb},
|
||||
}
|
||||
|
||||
except ImportError:
|
||||
return {
|
||||
"status": HealthStatus.HEALTHY,
|
||||
"message": "Memory check skipped (psutil not available)",
|
||||
}
|
||||
except Exception as e:
|
||||
return {
|
||||
"status": HealthStatus.UNHEALTHY,
|
||||
"message": f"Memory check failed: {str(e)}",
|
||||
}
|
||||
|
||||
|
||||
class HealthCheckManager:
|
||||
"""Manages multiple health checks"""
|
||||
|
||||
def __init__(self):
|
||||
self.checks: List[HealthCheck] = []
|
||||
self.last_results: Dict[str, HealthCheckResult] = {}
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def add_check(self, health_check: HealthCheck):
|
||||
"""Add a health check"""
|
||||
with self._lock:
|
||||
self.checks.append(health_check)
|
||||
|
||||
def run_all_checks(self) -> Dict[str, HealthCheckResult]:
|
||||
"""Run all health checks"""
|
||||
results = {}
|
||||
|
||||
for check in self.checks:
|
||||
try:
|
||||
result = check.check()
|
||||
results[check.name] = result
|
||||
|
||||
with self._lock:
|
||||
self.last_results[check.name] = result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error running health check {check.name}: {e}")
|
||||
results[check.name] = HealthCheckResult(
|
||||
name=check.name,
|
||||
status=HealthStatus.UNHEALTHY,
|
||||
message=f"Check execution failed: {str(e)}",
|
||||
timestamp=datetime.now(),
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
def get_overall_status(self) -> HealthStatus:
|
||||
"""Get overall system health status"""
|
||||
if not self.last_results:
|
||||
return HealthStatus.UNHEALTHY
|
||||
|
||||
statuses = [result.status for result in self.last_results.values()]
|
||||
|
||||
if any(status == HealthStatus.UNHEALTHY for status in statuses):
|
||||
return HealthStatus.UNHEALTHY
|
||||
elif any(status == HealthStatus.DEGRADED for status in statuses):
|
||||
return HealthStatus.DEGRADED
|
||||
else:
|
||||
return HealthStatus.HEALTHY
|
||||
|
||||
def get_health_summary(self) -> Dict[str, Any]:
|
||||
"""Get a summary of system health"""
|
||||
overall_status = self.get_overall_status()
|
||||
|
||||
return {
|
||||
"overall_status": overall_status.value,
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"checks": {
|
||||
name: {
|
||||
"status": result.status.value,
|
||||
"message": result.message,
|
||||
"response_time_ms": result.response_time_ms,
|
||||
"timestamp": result.timestamp.isoformat(),
|
||||
}
|
||||
for name, result in self.last_results.items()
|
||||
},
|
||||
}
|
||||
+5
-5
@@ -198,13 +198,13 @@ def main(argv: Optional[List[str]] = None) -> bool:
|
||||
level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
|
||||
)
|
||||
|
||||
from .config import Config
|
||||
from .config import database_url
|
||||
|
||||
db_config = Config.get_database_config()
|
||||
if db_config["type"] not in ("sqlite", "postgresql", "mysql"):
|
||||
logger.error(f"Backfill requires a SQL DB_TYPE, got '{db_config['type']}'")
|
||||
url = database_url()
|
||||
if not url:
|
||||
logger.error("no database: set POSTGRES_CONNECTION_STRING or POSTGRES_*")
|
||||
return False
|
||||
store = HiiStore(db_config["connection_string"], db_config["type"])
|
||||
store = HiiStore(url)
|
||||
if not store.connect():
|
||||
return False
|
||||
|
||||
|
||||
+19
-95
@@ -1,7 +1,10 @@
|
||||
"""Collector for HII/ThaiWater open api-v3 feeds (rainfall + water level).
|
||||
"""HII/ThaiWater open api-v3 client and PostgreSQL store (rainfall + water level).
|
||||
|
||||
Polls the unauthenticated api-v3.thaiwater.net public endpoints, filters to the
|
||||
Ping basin, and persists to dedicated tables alongside the RID data:
|
||||
The hourly collection runs in ping-collector (rust/collector/src/hii.rs, same
|
||||
parsing and tables); this module is used by the history backfill
|
||||
(src/hii_backfill.py, scripts/backfill_hii_waterlevel.py) and the tests.
|
||||
Endpoints: the unauthenticated api-v3.thaiwater.net public API, filtered to the
|
||||
Ping basin. Tables:
|
||||
|
||||
- hii_rain_stations / hii_rainfall (rain_1h / rain_24h gauge telemetry)
|
||||
- hii_wl_stations / hii_waterlevel (independent water-level source, m MSL)
|
||||
@@ -175,16 +178,10 @@ class HiiClient:
|
||||
|
||||
|
||||
class HiiStore:
|
||||
"""SQL persistence for HII feeds (sqlite / postgresql / mysql).
|
||||
"""PostgreSQL persistence for the HII feeds: the hii_* tables next to the
|
||||
RID data in the same database (the DDL matches rust/collector/src/db.rs)."""
|
||||
|
||||
Reuses the app's main relational database (same connection string as
|
||||
the RID tables) but writes to its own hii_* tables.
|
||||
"""
|
||||
|
||||
def __init__(self, connection_string: str, db_type: str):
|
||||
self.db_type = db_type.lower()
|
||||
if self.db_type not in ("sqlite", "postgresql", "mysql"):
|
||||
raise ValueError(f"HII collection requires a SQL database, got '{db_type}'")
|
||||
def __init__(self, connection_string: str):
|
||||
self.connection_string = connection_string
|
||||
self.engine = None
|
||||
|
||||
@@ -203,7 +200,6 @@ class HiiStore:
|
||||
def _create_tables(self):
|
||||
from sqlalchemy import text
|
||||
|
||||
bool_type = "BOOLEAN" if self.db_type != "mysql" else "TINYINT(1)"
|
||||
ddl = [
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS hii_rain_stations (
|
||||
@@ -230,7 +226,7 @@ class HiiStore:
|
||||
PRIMARY KEY (station_id, timestamp)
|
||||
)
|
||||
""",
|
||||
f"""
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS hii_wl_stations (
|
||||
id INTEGER PRIMARY KEY,
|
||||
oldcode VARCHAR(60),
|
||||
@@ -247,7 +243,7 @@ class HiiStore:
|
||||
critical_level_msl NUMERIC(8,3),
|
||||
critical_level_m NUMERIC(8,3),
|
||||
qmax NUMERIC(10,2),
|
||||
is_key_station {bool_type},
|
||||
is_key_station BOOLEAN,
|
||||
updated_at TIMESTAMP
|
||||
)
|
||||
""",
|
||||
@@ -269,10 +265,6 @@ class HiiStore:
|
||||
"CREATE INDEX IF NOT EXISTS idx_hii_rainfall_ts ON hii_rainfall(timestamp)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_hii_waterlevel_ts ON hii_waterlevel(timestamp)",
|
||||
]
|
||||
# MySQL (<8.0.13 semantics) lacks CREATE INDEX IF NOT EXISTS; the unique
|
||||
# constraints already cover the hot (station_id, timestamp) lookups there.
|
||||
if self.db_type == "mysql":
|
||||
ddl = ddl[:4]
|
||||
with self.engine.begin() as conn:
|
||||
for statement in ddl:
|
||||
conn.execute(text(statement))
|
||||
@@ -281,19 +273,11 @@ class HiiStore:
|
||||
cols = key_cols + value_cols
|
||||
col_list = ", ".join(cols)
|
||||
params = ", ".join(f":{c}" for c in cols)
|
||||
if self.db_type == "sqlite":
|
||||
return f"INSERT OR REPLACE INTO {table} ({col_list}) VALUES ({params})"
|
||||
if self.db_type == "postgresql":
|
||||
updates = ", ".join(f"{c} = EXCLUDED.{c}" for c in value_cols)
|
||||
conflict = ", ".join(key_cols)
|
||||
return (
|
||||
f"INSERT INTO {table} ({col_list}) VALUES ({params}) "
|
||||
f"ON CONFLICT ({conflict}) DO UPDATE SET {updates}"
|
||||
)
|
||||
updates = ", ".join(f"{c} = VALUES({c})" for c in value_cols)
|
||||
updates = ", ".join(f"{c} = EXCLUDED.{c}" for c in value_cols)
|
||||
conflict = ", ".join(key_cols)
|
||||
return (
|
||||
f"INSERT INTO {table} ({col_list}) VALUES ({params}) "
|
||||
f"ON DUPLICATE KEY UPDATE {updates}"
|
||||
f"ON CONFLICT ({conflict}) DO UPDATE SET {updates}"
|
||||
)
|
||||
|
||||
def save_rain(self, records: List[Dict]) -> int:
|
||||
@@ -360,18 +344,11 @@ class HiiStore:
|
||||
|
||||
cols = "(station_id, timestamp, wl_msl, discharge)"
|
||||
values = "(:station_id, :timestamp, :wl_msl, :discharge)"
|
||||
if self.db_type == "mysql":
|
||||
sql = (
|
||||
f"INSERT INTO hii_waterlevel {cols} VALUES {values} "
|
||||
"ON DUPLICATE KEY UPDATE wl_msl = VALUES(wl_msl), "
|
||||
"discharge = VALUES(discharge)"
|
||||
)
|
||||
else: # sqlite (>=3.24) and postgresql share upsert syntax
|
||||
sql = (
|
||||
f"INSERT INTO hii_waterlevel {cols} VALUES {values} "
|
||||
"ON CONFLICT (station_id, timestamp) DO UPDATE SET "
|
||||
"wl_msl = EXCLUDED.wl_msl, discharge = EXCLUDED.discharge"
|
||||
)
|
||||
sql = (
|
||||
f"INSERT INTO hii_waterlevel {cols} VALUES {values} "
|
||||
"ON CONFLICT (station_id, timestamp) DO UPDATE SET "
|
||||
"wl_msl = EXCLUDED.wl_msl, discharge = EXCLUDED.discharge"
|
||||
)
|
||||
params = [{**row, "station_id": station_id} for row in rows]
|
||||
try:
|
||||
with self.engine.begin() as conn:
|
||||
@@ -423,56 +400,3 @@ class HiiStore:
|
||||
except Exception as e:
|
||||
logger.error(f"HiiStore save to {measurement_table} failed: {e}")
|
||||
return 0
|
||||
|
||||
|
||||
class HiiCollector:
|
||||
"""Fetch + persist one snapshot of both HII feeds."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
db_config: Dict,
|
||||
basin_code: int = PING_BASIN_CODE,
|
||||
client: Optional[HiiClient] = None,
|
||||
):
|
||||
self.client = client or HiiClient()
|
||||
self.basin_code = basin_code
|
||||
self.store = HiiStore(
|
||||
connection_string=db_config["connection_string"],
|
||||
db_type=db_config["type"],
|
||||
)
|
||||
|
||||
def run_cycle(self) -> Dict[str, int]:
|
||||
"""Collect both feeds; each is independent and failure-isolated."""
|
||||
counts = {"rainfall": 0, "waterlevel": 0}
|
||||
try:
|
||||
counts["rainfall"] = self.store.save_rain(
|
||||
self.client.fetch_rain(self.basin_code)
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"HII rainfall collection failed: {e}")
|
||||
try:
|
||||
counts["waterlevel"] = self.store.save_waterlevel(
|
||||
self.client.fetch_waterlevel(self.basin_code)
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"HII waterlevel collection failed: {e}")
|
||||
logger.info(
|
||||
f"HII collection: {counts['rainfall']} rainfall, "
|
||||
f"{counts['waterlevel']} waterlevel rows saved"
|
||||
)
|
||||
return counts
|
||||
|
||||
|
||||
def create_collector_from_config() -> Optional[HiiCollector]:
|
||||
"""Build a collector from app Config; None when disabled or non-SQL DB."""
|
||||
from .config import Config
|
||||
|
||||
if not Config.ENABLE_HII_COLLECTION:
|
||||
return None
|
||||
db_config = Config.get_database_config()
|
||||
if db_config["type"] not in ("sqlite", "postgresql", "mysql"):
|
||||
logger.warning(
|
||||
f"HII collection skipped: DB_TYPE '{db_config['type']}' is not SQL"
|
||||
)
|
||||
return None
|
||||
return HiiCollector(db_config, basin_code=Config.HII_BASIN_CODE)
|
||||
@@ -1,135 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Centralized logging configuration for water monitoring system
|
||||
"""
|
||||
|
||||
import logging
|
||||
import logging.handlers
|
||||
import os
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class ColoredFormatter(logging.Formatter):
|
||||
"""Colored console formatter"""
|
||||
|
||||
COLORS = {
|
||||
"DEBUG": "\033[36m", # Cyan
|
||||
"INFO": "\033[32m", # Green
|
||||
"WARNING": "\033[33m", # Yellow
|
||||
"ERROR": "\033[31m", # Red
|
||||
"CRITICAL": "\033[35m", # Magenta
|
||||
"RESET": "\033[0m", # Reset
|
||||
}
|
||||
|
||||
def format(self, record):
|
||||
if hasattr(record, "levelname"):
|
||||
color = self.COLORS.get(record.levelname, self.COLORS["RESET"])
|
||||
record.levelname = f"{color}{record.levelname}{self.COLORS['RESET']}"
|
||||
return super().format(record)
|
||||
|
||||
|
||||
def setup_logging(
|
||||
log_level: str = "INFO",
|
||||
log_file: Optional[str] = None,
|
||||
max_file_size: int = 10 * 1024 * 1024, # 10MB
|
||||
backup_count: int = 5,
|
||||
enable_console: bool = True,
|
||||
enable_colors: bool = True,
|
||||
) -> logging.Logger:
|
||||
"""
|
||||
Setup comprehensive logging configuration
|
||||
|
||||
Args:
|
||||
log_level: Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
|
||||
log_file: Path to log file (optional)
|
||||
max_file_size: Maximum size of log file before rotation
|
||||
backup_count: Number of backup files to keep
|
||||
enable_console: Whether to enable console logging
|
||||
enable_colors: Whether to enable colored console output
|
||||
|
||||
Returns:
|
||||
Configured logger instance
|
||||
"""
|
||||
|
||||
# Create logs directory if it doesn't exist
|
||||
if log_file:
|
||||
log_dir = os.path.dirname(log_file)
|
||||
if log_dir and not os.path.exists(log_dir):
|
||||
os.makedirs(log_dir)
|
||||
|
||||
# Configure root logger
|
||||
logger = logging.getLogger()
|
||||
logger.setLevel(getattr(logging, log_level.upper()))
|
||||
|
||||
# Clear existing handlers
|
||||
logger.handlers.clear()
|
||||
|
||||
# Create formatters
|
||||
detailed_formatter = logging.Formatter(
|
||||
"%(asctime)s - %(name)s - %(levelname)s - %(filename)s:%(lineno)d - %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
|
||||
simple_formatter = logging.Formatter(
|
||||
"%(asctime)s - %(levelname)s - %(message)s", datefmt="%H:%M:%S"
|
||||
)
|
||||
|
||||
# Console handler
|
||||
if enable_console:
|
||||
console_handler = logging.StreamHandler()
|
||||
if enable_colors and os.name != "nt": # Don't use colors on Windows
|
||||
console_formatter = ColoredFormatter(
|
||||
"%(asctime)s - %(levelname)s - %(message)s", datefmt="%H:%M:%S"
|
||||
)
|
||||
else:
|
||||
console_formatter = simple_formatter
|
||||
|
||||
console_handler.setFormatter(console_formatter)
|
||||
console_handler.setLevel(getattr(logging, log_level.upper()))
|
||||
logger.addHandler(console_handler)
|
||||
|
||||
# File handler with rotation
|
||||
if log_file:
|
||||
file_handler = logging.handlers.RotatingFileHandler(
|
||||
log_file, maxBytes=max_file_size, backupCount=backup_count, encoding="utf-8"
|
||||
)
|
||||
file_handler.setFormatter(detailed_formatter)
|
||||
file_handler.setLevel(logging.DEBUG) # Always log everything to file
|
||||
logger.addHandler(file_handler)
|
||||
|
||||
# Add performance logger for metrics
|
||||
perf_logger = logging.getLogger("performance")
|
||||
if log_file:
|
||||
perf_file = log_file.replace(".log", "_performance.log")
|
||||
perf_handler = logging.handlers.RotatingFileHandler(
|
||||
perf_file,
|
||||
maxBytes=max_file_size,
|
||||
backupCount=backup_count,
|
||||
encoding="utf-8",
|
||||
)
|
||||
perf_formatter = logging.Formatter(
|
||||
"%(asctime)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S"
|
||||
)
|
||||
perf_handler.setFormatter(perf_formatter)
|
||||
perf_logger.addHandler(perf_handler)
|
||||
perf_logger.setLevel(logging.INFO)
|
||||
perf_logger.propagate = False
|
||||
|
||||
return logger
|
||||
|
||||
|
||||
def log_performance_metric(
|
||||
operation: str, duration: float, additional_info: Optional[str] = None
|
||||
):
|
||||
"""Log performance metrics"""
|
||||
perf_logger = logging.getLogger("performance")
|
||||
message = f"PERF: {operation} took {duration:.3f}s"
|
||||
if additional_info:
|
||||
message += f" - {additional_info}"
|
||||
perf_logger.info(message)
|
||||
|
||||
|
||||
def get_logger(name: str) -> logging.Logger:
|
||||
"""Get a logger with the specified name"""
|
||||
return logging.getLogger(name)
|
||||
-654
@@ -1,654 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Main entry point for the Thailand Water Monitor system
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import signal
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
from .config import Config
|
||||
from .exceptions import ConfigurationError, DatabaseConnectionError
|
||||
from .logging_config import get_logger, setup_logging
|
||||
from .metrics import get_metrics_collector
|
||||
from .water_scraper_v3 import EnhancedWaterMonitorScraper
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
def setup_signal_handlers(scraper: Optional[EnhancedWaterMonitorScraper] = None):
|
||||
"""Setup signal handlers for graceful shutdown"""
|
||||
|
||||
def signal_handler(signum, frame):
|
||||
logger.info(f"Received signal {signum}, shutting down gracefully...")
|
||||
if scraper:
|
||||
logger.info("Stopping scraper...")
|
||||
sys.exit(0)
|
||||
|
||||
signal.signal(signal.SIGINT, signal_handler)
|
||||
signal.signal(signal.SIGTERM, signal_handler)
|
||||
|
||||
|
||||
def run_test_cycle():
|
||||
"""Run a single test cycle"""
|
||||
logger.info("Running test cycle...")
|
||||
|
||||
try:
|
||||
# Validate configuration
|
||||
Config.validate_config()
|
||||
|
||||
# Initialize scraper
|
||||
db_config = Config.get_database_config()
|
||||
scraper = EnhancedWaterMonitorScraper(db_config)
|
||||
|
||||
# Run single scraping cycle
|
||||
result = scraper.run_scraping_cycle()
|
||||
|
||||
if result:
|
||||
logger.info("✅ Test cycle completed successfully")
|
||||
|
||||
# Show some statistics
|
||||
latest_data = scraper.get_latest_data(5)
|
||||
if latest_data:
|
||||
logger.info(f"Latest data points: {len(latest_data)}")
|
||||
for data in latest_data[:3]: # Show first 3
|
||||
logger.info(
|
||||
f" • {data['station_code']}: {data['water_level']:.2f}m, {data['discharge']:.1f} cms"
|
||||
)
|
||||
else:
|
||||
logger.warning("⚠️ Test cycle completed but no new data was found")
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Test cycle failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def run_continuous_monitoring():
|
||||
"""Run continuous monitoring with adaptive scheduling and alerting"""
|
||||
logger.info("Starting continuous monitoring...")
|
||||
|
||||
try:
|
||||
# Validate configuration
|
||||
Config.validate_config()
|
||||
|
||||
# Initialize scraper
|
||||
db_config = Config.get_database_config()
|
||||
scraper = EnhancedWaterMonitorScraper(db_config)
|
||||
|
||||
# Initialize alerting system
|
||||
from .alerting import WaterLevelAlertSystem
|
||||
|
||||
alerting = WaterLevelAlertSystem()
|
||||
|
||||
# Initialize HII/ThaiWater collector (rainfall + backup water level)
|
||||
hii_collector = None
|
||||
try:
|
||||
from .hii_collector import create_collector_from_config
|
||||
|
||||
hii_collector = create_collector_from_config()
|
||||
if hii_collector:
|
||||
logger.info(
|
||||
"HII collection enabled (Ping-basin rainfall + water level)"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"HII collector initialization failed: {e}")
|
||||
|
||||
# Setup signal handlers
|
||||
setup_signal_handlers(scraper)
|
||||
|
||||
logger.info(
|
||||
f"Monitoring started with {Config.SCRAPING_INTERVAL_HOURS}h interval"
|
||||
)
|
||||
logger.info(
|
||||
"Adaptive retry: switches to 1-minute intervals when no data available"
|
||||
)
|
||||
logger.info("Alerts: automatic check after each successful data fetch")
|
||||
logger.info("Press Ctrl+C to stop")
|
||||
|
||||
# Run initial cycle
|
||||
logger.info("Running initial data collection...")
|
||||
initial_success = scraper.run_scraping_cycle()
|
||||
|
||||
# Adaptive scheduling state
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
retry_mode = not initial_success
|
||||
last_successful_fetch = None if not initial_success else datetime.now()
|
||||
|
||||
last_hii_run = None
|
||||
if hii_collector:
|
||||
try:
|
||||
hii_collector.run_cycle()
|
||||
last_hii_run = datetime.now()
|
||||
except Exception as e:
|
||||
logger.error(f"HII collection failed: {e}")
|
||||
|
||||
if retry_mode:
|
||||
logger.warning("No data fetched in initial run - entering retry mode")
|
||||
next_run = datetime.now() + timedelta(minutes=1)
|
||||
else:
|
||||
logger.info("Initial data fetch successful - using hourly schedule")
|
||||
next_run = (datetime.now() + timedelta(hours=1)).replace(
|
||||
minute=0, second=0, microsecond=0
|
||||
)
|
||||
|
||||
logger.info(f"Next run at {next_run.strftime('%H:%M')}")
|
||||
|
||||
while True:
|
||||
current_time = datetime.now()
|
||||
|
||||
if current_time >= next_run:
|
||||
logger.info("Running scheduled data collection...")
|
||||
success = scraper.run_scraping_cycle()
|
||||
|
||||
# HII feeds update hourly; keep collecting on that cadence even
|
||||
# when the RID scraper is in 1-minute retry mode.
|
||||
if hii_collector and (
|
||||
last_hii_run is None
|
||||
or current_time - last_hii_run >= timedelta(minutes=55)
|
||||
):
|
||||
try:
|
||||
hii_collector.run_cycle()
|
||||
last_hii_run = current_time
|
||||
except Exception as e:
|
||||
logger.error(f"HII collection failed: {e}")
|
||||
|
||||
if success:
|
||||
last_successful_fetch = current_time
|
||||
|
||||
# Run alert check after every successful new data fetch
|
||||
logger.info("Running alert check...")
|
||||
try:
|
||||
alert_results = alerting.run_alert_check()
|
||||
if alert_results.get("total_alerts", 0) > 0:
|
||||
logger.info(
|
||||
f"Alerts: {alert_results['total_alerts']} generated, {alert_results['sent']} sent"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Alert check failed: {e}")
|
||||
|
||||
if retry_mode:
|
||||
logger.info(
|
||||
"✅ Data fetch successful - switching back to hourly schedule"
|
||||
)
|
||||
retry_mode = False
|
||||
# Schedule next run at the next full hour
|
||||
next_run = (current_time + timedelta(hours=1)).replace(
|
||||
minute=0, second=0, microsecond=0
|
||||
)
|
||||
else:
|
||||
# Continue hourly schedule
|
||||
next_run = (
|
||||
current_time
|
||||
+ timedelta(hours=Config.SCRAPING_INTERVAL_HOURS)
|
||||
).replace(minute=0, second=0, microsecond=0)
|
||||
|
||||
logger.info(f"Next scheduled run at {next_run.strftime('%H:%M')}")
|
||||
else:
|
||||
if not retry_mode:
|
||||
logger.warning(
|
||||
"⚠️ No data fetched - switching to retry mode (1-minute intervals)"
|
||||
)
|
||||
retry_mode = True
|
||||
|
||||
# Schedule retry in 1 minute
|
||||
next_run = current_time + timedelta(minutes=1)
|
||||
logger.info(f"Retrying in 1 minute at {next_run.strftime('%H:%M')}")
|
||||
|
||||
# Sleep for 10 seconds and check again
|
||||
time.sleep(10)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
logger.info("Monitoring stopped by user")
|
||||
except Exception as e:
|
||||
logger.error(f"Monitoring failed: {e}")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def run_hii_collection():
|
||||
"""Run a single HII/ThaiWater collection cycle (rainfall + water level)"""
|
||||
try:
|
||||
Config.validate_config()
|
||||
from .hii_collector import create_collector_from_config
|
||||
|
||||
collector = create_collector_from_config()
|
||||
if not collector:
|
||||
logger.error(
|
||||
"HII collection unavailable (disabled via ENABLE_HII_COLLECTION "
|
||||
"or DB_TYPE is not a SQL database)"
|
||||
)
|
||||
return False
|
||||
|
||||
counts = collector.run_cycle()
|
||||
return counts["rainfall"] > 0 or counts["waterlevel"] > 0
|
||||
except Exception as e:
|
||||
logger.error(f"HII collection failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def run_gap_filling(days_back: Optional[int]):
|
||||
"""Run gap filling for missing data (days_back=None scans the whole range)"""
|
||||
if days_back is None:
|
||||
logger.info("Checking for data gaps across the whole data range...")
|
||||
else:
|
||||
logger.info(f"Checking for data gaps in the last {days_back} days...")
|
||||
|
||||
try:
|
||||
# Validate configuration
|
||||
Config.validate_config()
|
||||
|
||||
# Initialize scraper
|
||||
db_config = Config.get_database_config()
|
||||
scraper = EnhancedWaterMonitorScraper(db_config)
|
||||
|
||||
# Fill gaps
|
||||
filled_count = scraper.fill_data_gaps(days_back)
|
||||
|
||||
if filled_count > 0:
|
||||
logger.info(f"✅ Filled {filled_count} missing data points")
|
||||
else:
|
||||
logger.info("✅ No data gaps found")
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Gap filling failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def run_data_update(days_back: int):
|
||||
"""Update existing data with latest values"""
|
||||
logger.info(f"Updating existing data for the last {days_back} days...")
|
||||
|
||||
try:
|
||||
# Validate configuration
|
||||
Config.validate_config()
|
||||
|
||||
# Initialize scraper
|
||||
db_config = Config.get_database_config()
|
||||
scraper = EnhancedWaterMonitorScraper(db_config)
|
||||
|
||||
# Update data
|
||||
updated_count = scraper.update_existing_data(days_back)
|
||||
|
||||
if updated_count > 0:
|
||||
logger.info(f"✅ Updated {updated_count} data points")
|
||||
else:
|
||||
logger.info("✅ No data updates needed")
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Data update failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def run_historical_import(
|
||||
start_date_str: str, end_date_str: str, skip_existing: bool = True
|
||||
):
|
||||
"""Import historical data for a date range"""
|
||||
try:
|
||||
# Parse dates
|
||||
start_date = datetime.strptime(start_date_str, "%Y-%m-%d")
|
||||
end_date = datetime.strptime(end_date_str, "%Y-%m-%d")
|
||||
|
||||
if start_date > end_date:
|
||||
logger.error("Start date must be before or equal to end date")
|
||||
return False
|
||||
|
||||
logger.info(
|
||||
f"Importing historical data from {start_date.date()} to {end_date.date()}"
|
||||
)
|
||||
if skip_existing:
|
||||
logger.info("Skipping dates that already have data")
|
||||
|
||||
# Validate configuration
|
||||
Config.validate_config()
|
||||
|
||||
# Initialize scraper
|
||||
db_config = Config.get_database_config()
|
||||
scraper = EnhancedWaterMonitorScraper(db_config)
|
||||
|
||||
# Import historical data
|
||||
imported_count = scraper.import_historical_data(
|
||||
start_date, end_date, skip_existing
|
||||
)
|
||||
|
||||
if imported_count > 0:
|
||||
logger.info(f"✅ Imported {imported_count} historical data points")
|
||||
else:
|
||||
logger.info("✅ No new data imported")
|
||||
|
||||
return True
|
||||
|
||||
except ValueError as e:
|
||||
logger.error(f"❌ Invalid date format. Use YYYY-MM-DD: {e}")
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Historical import failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def run_web_api():
|
||||
"""Run the FastAPI web interface"""
|
||||
logger.info("Starting web API server...")
|
||||
|
||||
try:
|
||||
import uvicorn
|
||||
|
||||
# Validate configuration
|
||||
Config.validate_config()
|
||||
|
||||
workers = max(1, Config.WEB_WORKERS)
|
||||
if workers > 1:
|
||||
# Multi-worker needs the app as an import string; a localhost lock
|
||||
# port keeps background collection in exactly one worker.
|
||||
uvicorn.run(
|
||||
"src.web_api:app",
|
||||
host="0.0.0.0",
|
||||
port=8000,
|
||||
workers=workers,
|
||||
log_config=None,
|
||||
)
|
||||
else:
|
||||
from .web_api import app
|
||||
|
||||
uvicorn.run(app, host="0.0.0.0", port=8000, log_config=None)
|
||||
|
||||
except ImportError:
|
||||
logger.error("FastAPI not installed. Run: pip install fastapi uvicorn")
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"Web API failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def run_alert_check():
|
||||
"""Run water level alert check"""
|
||||
logger.info("Running water level alert check...")
|
||||
|
||||
try:
|
||||
from .alerting import WaterLevelAlertSystem
|
||||
|
||||
# Initialize alerting system
|
||||
alerting = WaterLevelAlertSystem()
|
||||
|
||||
# Run alert check
|
||||
results = alerting.run_alert_check()
|
||||
|
||||
if "error" in results:
|
||||
logger.error("❌ Alert check failed due to database connection")
|
||||
return False
|
||||
|
||||
logger.info(f"✅ Alert check completed:")
|
||||
logger.info(f" • Water level alerts: {results['water_alerts']}")
|
||||
logger.info(f" • Data freshness alerts: {results['data_alerts']}")
|
||||
logger.info(f" • Total alerts generated: {results['total_alerts']}")
|
||||
logger.info(f" • Alerts sent: {results['sent']}")
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Alert check failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def run_alert_test():
|
||||
"""Send test alert message"""
|
||||
logger.info("Sending test alert message...")
|
||||
|
||||
try:
|
||||
from .alerting import WaterLevelAlertSystem
|
||||
|
||||
# Initialize alerting system
|
||||
alerting = WaterLevelAlertSystem()
|
||||
|
||||
if not alerting.matrix_notifier:
|
||||
logger.error("❌ Matrix notifier not configured")
|
||||
logger.info(
|
||||
"Please set MATRIX_ACCESS_TOKEN and MATRIX_ROOM_ID in your .env file"
|
||||
)
|
||||
return False
|
||||
|
||||
# Send test message
|
||||
test_message = "🧪 **Test Alert**\n\nThis is a test message from the Northern Thailand Ping River Monitor.\n\nIf you received this, Matrix notifications are working correctly!"
|
||||
success = alerting.matrix_notifier.send_message(test_message)
|
||||
|
||||
if success:
|
||||
logger.info("✅ Test alert message sent successfully")
|
||||
else:
|
||||
logger.error("❌ Test alert message failed to send")
|
||||
|
||||
return success
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Test alert failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def show_status():
|
||||
"""Show current system status"""
|
||||
logger.info("=== Northern Thailand Ping River Monitor Status ===")
|
||||
|
||||
try:
|
||||
# Show configuration
|
||||
Config.print_settings()
|
||||
|
||||
# Test database connection
|
||||
logger.info("\n=== Database Connection Test ===")
|
||||
db_config = Config.get_database_config()
|
||||
scraper = EnhancedWaterMonitorScraper(db_config)
|
||||
|
||||
if scraper.db_adapter:
|
||||
logger.info("✅ Database connection successful")
|
||||
|
||||
# Show latest data
|
||||
latest_data = scraper.get_latest_data(3)
|
||||
if latest_data:
|
||||
logger.info(f"\n=== Latest Data ({len(latest_data)} points) ===")
|
||||
for data in latest_data:
|
||||
timestamp = data["timestamp"]
|
||||
if isinstance(timestamp, str):
|
||||
timestamp = datetime.fromisoformat(
|
||||
timestamp.replace("Z", "+00:00")
|
||||
)
|
||||
logger.info(
|
||||
f" • {data['station_code']} ({timestamp}): {data['water_level']:.2f}m"
|
||||
)
|
||||
else:
|
||||
logger.info("No data found in database")
|
||||
else:
|
||||
logger.error("❌ Database connection failed")
|
||||
|
||||
# Test alerting system
|
||||
logger.info("\n=== Alerting System Status ===")
|
||||
try:
|
||||
from .alerting import WaterLevelAlertSystem
|
||||
|
||||
alerting = WaterLevelAlertSystem()
|
||||
|
||||
if alerting.matrix_notifier:
|
||||
logger.info("✅ Matrix notifications configured")
|
||||
else:
|
||||
logger.warning("⚠️ Matrix notifications not configured")
|
||||
logger.info("Set MATRIX_ACCESS_TOKEN and MATRIX_ROOM_ID in .env file")
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Alerting system error: {e}")
|
||||
|
||||
# Show metrics if available
|
||||
metrics_collector = get_metrics_collector()
|
||||
metrics = metrics_collector.get_all_metrics()
|
||||
|
||||
if any(metrics.values()):
|
||||
logger.info("\n=== Metrics Summary ===")
|
||||
for metric_type, values in metrics.items():
|
||||
if values:
|
||||
logger.info(f"{metric_type.title()}: {len(values)} metrics")
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Status check failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Northern Thailand Ping River Monitor",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
%(prog)s --test # Run single test cycle
|
||||
%(prog)s # Run continuous monitoring
|
||||
%(prog)s --web-api # Start web API server
|
||||
%(prog)s --fill-gaps 7 # Fill missing data for last 7 days
|
||||
%(prog)s --fill-gaps all # Fill missing data across the whole data range
|
||||
%(prog)s --update-data 2 # Update existing data for last 2 days
|
||||
%(prog)s --import-historical 2024-01-01 2024-01-31 # Import historical data
|
||||
%(prog)s --status # Show system status
|
||||
%(prog)s --alert-check # Check water levels and send alerts
|
||||
%(prog)s --alert-test # Send test Matrix message
|
||||
""",
|
||||
)
|
||||
|
||||
parser.add_argument("--test", action="store_true", help="Run a single test cycle")
|
||||
|
||||
parser.add_argument(
|
||||
"--web-api", action="store_true", help="Start the web API server"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--fill-gaps",
|
||||
metavar="DAYS|all",
|
||||
help=(
|
||||
"Fill missing data gaps for the specified number of days back, "
|
||||
"or 'all' to scan the entire data range in the database"
|
||||
),
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--update-data",
|
||||
type=int,
|
||||
metavar="DAYS",
|
||||
help="Update existing data for the specified number of days back",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--import-historical",
|
||||
nargs=2,
|
||||
metavar=("START_DATE", "END_DATE"),
|
||||
help="Import historical data for date range (YYYY-MM-DD format)",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--force-overwrite",
|
||||
action="store_true",
|
||||
help="Overwrite existing data when importing historical data",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--status", action="store_true", help="Show current system status"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--alert-check", action="store_true", help="Run water level alert check"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--alert-test", action="store_true", help="Send test alert message to Matrix"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--collect-hii",
|
||||
action="store_true",
|
||||
help="Run one HII/ThaiWater collection cycle (rainfall + water level)",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--log-level",
|
||||
choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
|
||||
default=Config.LOG_LEVEL,
|
||||
help="Set logging level",
|
||||
)
|
||||
|
||||
parser.add_argument("--log-file", default=Config.LOG_FILE, help="Log file path")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Setup logging
|
||||
setup_logging(
|
||||
log_level=args.log_level,
|
||||
log_file=args.log_file,
|
||||
enable_console=True,
|
||||
enable_colors=True,
|
||||
)
|
||||
|
||||
logger.info("🏔️ Northern Thailand Ping River Monitor starting...")
|
||||
logger.info(f"Version: 3.1.3")
|
||||
logger.info(f"Log level: {args.log_level}")
|
||||
|
||||
try:
|
||||
success = False
|
||||
|
||||
if args.test:
|
||||
success = run_test_cycle()
|
||||
elif args.web_api:
|
||||
success = run_web_api()
|
||||
elif args.fill_gaps is not None:
|
||||
if args.fill_gaps.lower() == "all":
|
||||
success = run_gap_filling(None)
|
||||
else:
|
||||
try:
|
||||
success = run_gap_filling(int(args.fill_gaps))
|
||||
except ValueError:
|
||||
logger.error(
|
||||
f"Invalid --fill-gaps value '{args.fill_gaps}': "
|
||||
"expected a number of days or 'all'"
|
||||
)
|
||||
sys.exit(1)
|
||||
elif args.update_data is not None:
|
||||
success = run_data_update(args.update_data)
|
||||
elif args.import_historical is not None:
|
||||
start_date, end_date = args.import_historical
|
||||
skip_existing = not args.force_overwrite
|
||||
success = run_historical_import(start_date, end_date, skip_existing)
|
||||
elif args.status:
|
||||
success = show_status()
|
||||
elif args.alert_check:
|
||||
success = run_alert_check()
|
||||
elif args.alert_test:
|
||||
success = run_alert_test()
|
||||
elif args.collect_hii:
|
||||
success = run_hii_collection()
|
||||
else:
|
||||
success = run_continuous_monitoring()
|
||||
|
||||
if success:
|
||||
logger.info("✅ Operation completed successfully")
|
||||
sys.exit(0)
|
||||
else:
|
||||
logger.error("❌ Operation failed")
|
||||
sys.exit(1)
|
||||
|
||||
except ConfigurationError as e:
|
||||
logger.error(f"Configuration error: {e}")
|
||||
sys.exit(1)
|
||||
except KeyboardInterrupt:
|
||||
logger.info("Operation cancelled by user")
|
||||
sys.exit(0)
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
-198
@@ -1,198 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Metrics collection and monitoring for water monitoring system
|
||||
"""
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from collections import defaultdict, deque
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MetricPoint:
|
||||
"""Single metric data point"""
|
||||
|
||||
timestamp: datetime
|
||||
value: float
|
||||
labels: Dict[str, str] = field(default_factory=dict)
|
||||
|
||||
|
||||
class MetricsCollector:
|
||||
"""Collects and manages application metrics"""
|
||||
|
||||
def __init__(self, retention_hours: int = 24):
|
||||
self.retention_hours = retention_hours
|
||||
self.metrics: Dict[str, deque] = defaultdict(lambda: deque(maxlen=1000))
|
||||
self.counters: Dict[str, float] = defaultdict(float)
|
||||
self.gauges: Dict[str, float] = defaultdict(float)
|
||||
self.histograms: Dict[str, List[float]] = defaultdict(list)
|
||||
self._lock = threading.Lock()
|
||||
|
||||
# Start cleanup thread
|
||||
self._cleanup_thread = threading.Thread(
|
||||
target=self._cleanup_old_metrics, daemon=True
|
||||
)
|
||||
self._cleanup_thread.start()
|
||||
|
||||
def increment_counter(
|
||||
self, name: str, value: float = 1.0, labels: Optional[Dict[str, str]] = None
|
||||
):
|
||||
"""Increment a counter metric"""
|
||||
with self._lock:
|
||||
key = self._make_key(name, labels)
|
||||
self.counters[key] += value
|
||||
self.metrics[key].append(
|
||||
MetricPoint(datetime.now(), self.counters[key], labels or {})
|
||||
)
|
||||
|
||||
def set_gauge(
|
||||
self, name: str, value: float, labels: Optional[Dict[str, str]] = None
|
||||
):
|
||||
"""Set a gauge metric"""
|
||||
with self._lock:
|
||||
key = self._make_key(name, labels)
|
||||
self.gauges[key] = value
|
||||
self.metrics[key].append(MetricPoint(datetime.now(), value, labels or {}))
|
||||
|
||||
def record_histogram(
|
||||
self, name: str, value: float, labels: Optional[Dict[str, str]] = None
|
||||
):
|
||||
"""Record a histogram value"""
|
||||
with self._lock:
|
||||
key = self._make_key(name, labels)
|
||||
self.histograms[key].append(value)
|
||||
# Keep only recent values
|
||||
if len(self.histograms[key]) > 1000:
|
||||
self.histograms[key] = self.histograms[key][-1000:]
|
||||
|
||||
self.metrics[key].append(MetricPoint(datetime.now(), value, labels or {}))
|
||||
|
||||
def get_counter(self, name: str, labels: Optional[Dict[str, str]] = None) -> float:
|
||||
"""Get current counter value"""
|
||||
key = self._make_key(name, labels)
|
||||
return self.counters.get(key, 0.0)
|
||||
|
||||
def get_gauge(self, name: str, labels: Optional[Dict[str, str]] = None) -> float:
|
||||
"""Get current gauge value"""
|
||||
key = self._make_key(name, labels)
|
||||
return self.gauges.get(key, 0.0)
|
||||
|
||||
def get_histogram_stats(
|
||||
self, name: str, labels: Optional[Dict[str, str]] = None
|
||||
) -> Dict[str, float]:
|
||||
"""Get histogram statistics"""
|
||||
key = self._make_key(name, labels)
|
||||
values = self.histograms.get(key, [])
|
||||
|
||||
if not values:
|
||||
return {"count": 0, "sum": 0, "avg": 0, "min": 0, "max": 0}
|
||||
|
||||
return {
|
||||
"count": len(values),
|
||||
"sum": sum(values),
|
||||
"avg": sum(values) / len(values),
|
||||
"min": min(values),
|
||||
"max": max(values),
|
||||
}
|
||||
|
||||
def get_all_metrics(self) -> Dict[str, Any]:
|
||||
"""Get all current metrics"""
|
||||
with self._lock:
|
||||
return {
|
||||
"counters": dict(self.counters),
|
||||
"gauges": dict(self.gauges),
|
||||
"histograms": {k: self.get_histogram_stats(k) for k in self.histograms},
|
||||
}
|
||||
|
||||
def _make_key(self, name: str, labels: Optional[Dict[str, str]]) -> str:
|
||||
"""Create a unique key for metric with labels"""
|
||||
if not labels:
|
||||
return name
|
||||
|
||||
label_str = ",".join(f"{k}={v}" for k, v in sorted(labels.items()))
|
||||
return f"{name}{{{label_str}}}"
|
||||
|
||||
def _cleanup_old_metrics(self):
|
||||
"""Clean up old metric data points"""
|
||||
while True:
|
||||
try:
|
||||
cutoff_time = datetime.now() - timedelta(hours=self.retention_hours)
|
||||
|
||||
with self._lock:
|
||||
for metric_name, points in self.metrics.items():
|
||||
# Remove old points
|
||||
while points and points[0].timestamp < cutoff_time:
|
||||
points.popleft()
|
||||
|
||||
time.sleep(3600) # Run cleanup every hour
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in metrics cleanup: {e}")
|
||||
time.sleep(60) # Wait a minute before retrying
|
||||
|
||||
|
||||
# Global metrics collector instance
|
||||
_metrics_collector = None
|
||||
|
||||
|
||||
def get_metrics_collector() -> MetricsCollector:
|
||||
"""Get the global metrics collector instance"""
|
||||
global _metrics_collector
|
||||
if _metrics_collector is None:
|
||||
_metrics_collector = MetricsCollector()
|
||||
return _metrics_collector
|
||||
|
||||
|
||||
# Convenience functions
|
||||
def increment_counter(
|
||||
name: str, value: float = 1.0, labels: Optional[Dict[str, str]] = None
|
||||
):
|
||||
"""Increment a counter metric"""
|
||||
get_metrics_collector().increment_counter(name, value, labels)
|
||||
|
||||
|
||||
def set_gauge(name: str, value: float, labels: Optional[Dict[str, str]] = None):
|
||||
"""Set a gauge metric"""
|
||||
get_metrics_collector().set_gauge(name, value, labels)
|
||||
|
||||
|
||||
def record_histogram(name: str, value: float, labels: Optional[Dict[str, str]] = None):
|
||||
"""Record a histogram value"""
|
||||
get_metrics_collector().record_histogram(name, value, labels)
|
||||
|
||||
|
||||
class Timer:
|
||||
"""Context manager for timing operations"""
|
||||
|
||||
def __init__(self, metric_name: str, labels: Optional[Dict[str, str]] = None):
|
||||
self.metric_name = metric_name
|
||||
self.labels = labels
|
||||
self.start_time = None
|
||||
|
||||
def __enter__(self):
|
||||
self.start_time = time.time()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
if self.start_time:
|
||||
duration = time.time() - self.start_time
|
||||
record_histogram(self.metric_name, duration, self.labels)
|
||||
|
||||
|
||||
def timer(metric_name: str, labels: Optional[Dict[str, str]] = None):
|
||||
"""Decorator for timing function execution"""
|
||||
|
||||
def decorator(func):
|
||||
def wrapper(*args, **kwargs):
|
||||
with Timer(metric_name, labels):
|
||||
return func(*args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
return decorator
|
||||
+3
-11
@@ -18,7 +18,7 @@ from typing import Dict, List, Optional
|
||||
import pandas as pd
|
||||
from sqlalchemy import create_engine, text
|
||||
|
||||
from ..config import Config
|
||||
from ..config import database_url
|
||||
from .features import UPSTREAM_LEADS
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -34,7 +34,7 @@ _MEASUREMENT_COLUMNS = ["timestamp", "station_code", "water_level", "discharge"]
|
||||
|
||||
def resolve_db_url(db_url: Optional[str] = None) -> Optional[str]:
|
||||
"""Resolve a Postgres connection string: explicit param > FLOOD_ML_DB_URL env >
|
||||
Config's postgresql connection string > None (caller should fall back to HTTP)."""
|
||||
POSTGRES_* settings (src/config.py) > None (caller should fall back to HTTP)."""
|
||||
if db_url:
|
||||
return db_url
|
||||
|
||||
@@ -42,15 +42,7 @@ def resolve_db_url(db_url: Optional[str] = None) -> Optional[str]:
|
||||
if env_url:
|
||||
return env_url
|
||||
|
||||
try:
|
||||
db_config = Config.get_database_config()
|
||||
except Exception as error:
|
||||
logger.warning(f"Could not resolve database config: {error}")
|
||||
return None
|
||||
|
||||
if db_config.get("type") == "postgresql":
|
||||
return db_config.get("connection_string")
|
||||
return None
|
||||
return database_url()
|
||||
|
||||
|
||||
def _default_stations() -> List[str]:
|
||||
|
||||
+54
-12
@@ -8,8 +8,8 @@ joblib/pickle. It reads:
|
||||
table). ``train_all()`` writes one next to each ``.joblib``;
|
||||
``python -m src.ml.export`` (re)exports the bundles already in ``models/``.
|
||||
* ``rust/forecast/flood_config.json`` -- thresholds, flood stages, upstream
|
||||
leads and the other constants features.py / predict.py / notify.py use,
|
||||
compiled into the Rust binary. ``python -m src.ml.export --config``
|
||||
leads and the other constants features.py / predict.py use, plus the
|
||||
notifier settings below, compiled into the Rust binary. ``python -m src.ml.export --config``
|
||||
regenerates it; tests/test_model_export.py fails while it is out of date,
|
||||
so a threshold change cannot reach Python without reaching Rust.
|
||||
|
||||
@@ -31,6 +31,49 @@ import numpy as np
|
||||
from . import data as data_mod
|
||||
from . import features, predict
|
||||
|
||||
# --- Notifier settings (rust/collector/src/notify.rs; docs/NOTIFICATIONS.md).
|
||||
# Only the Rust notifier uses these; they live here so the thresholds, stages
|
||||
# and these values reach it through one generated file.
|
||||
|
||||
# Hysteresis: an all-clear needs the level this far BELOW the threshold, so a
|
||||
# river bobbing around 3.70 m does not toggle warning/clear every hour.
|
||||
CLEAR_MARGIN_M = 0.10
|
||||
# Capacity guard. The level thresholds in features.THRESHOLDS were calibrated
|
||||
# from RID's discharge_percent (% of channel capacity); if RID re-rates a
|
||||
# gauge or moves its datum, the level crosses while capacity says the channel
|
||||
# is nearly empty (P.77, 2026-09: 3.0 m "warning" at 22 %). A crossing is
|
||||
# only announced when the reported capacity agrees that the river is high.
|
||||
# P.1 is exempt: its stages come from the municipal inundation map, not from
|
||||
# capacity. Readings without a capacity figure fall back to level only.
|
||||
CAPACITY_GUARD_MIN_PCT = 60.0
|
||||
CAPACITY_GUARD_EXEMPT = {"P.1"}
|
||||
# The P.1 outlook fires when p_warning at OUTLOOK_HORIZON hours rises through
|
||||
# OUTLOOK_ON and clears below OUTLOOK_OFF.
|
||||
OUTLOOK_ON = 0.50
|
||||
OUTLOOK_OFF = 0.25
|
||||
OUTLOOK_HORIZON = 24
|
||||
# No reading this many hours old or newer: the feed is stale.
|
||||
FEED_STALE_AFTER_H = 3.0
|
||||
|
||||
STATION_NAMES = {
|
||||
"P.1": "Nawarat Bridge, Chiang Mai city",
|
||||
"P.103": "Ring Road Bridge 3, Chiang Mai",
|
||||
"P.67": "Ban Tae (Mae Taeng)",
|
||||
"P.21": "Ban Rim Tai (Mae Rim)",
|
||||
"P.75": "Ban Chai Lat",
|
||||
"P.92": "Ban Muang Aut",
|
||||
"P.20": "Ban Chiang Dao",
|
||||
"P.4A": "Ban Mae Taeng",
|
||||
"P.5": "Tha Nang Bridge (downstream)",
|
||||
"P.81": "Ban Pong (downstream)",
|
||||
"P.82": "Ban Sob Win",
|
||||
"P.84": "Ban Panton",
|
||||
"P.87": "Ban Pa Sang",
|
||||
"P.77": "Ban Sop Mae Sapuat",
|
||||
"P.85": "Ban Lai Kaew",
|
||||
"P.76": "Ban Mae I Hai",
|
||||
}
|
||||
|
||||
FORMAT_VERSION = 1
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
CONFIG_PATH = REPO_ROOT / "rust" / "forecast" / "flood_config.json"
|
||||
@@ -123,8 +166,7 @@ def export_models_dir(models_dir: Path, force: bool = False) -> int:
|
||||
|
||||
|
||||
def flood_config() -> dict:
|
||||
"""Every constant the Rust forecaster and notifier share with Python."""
|
||||
from .. import notify
|
||||
"""Every constant the Rust forecaster and notifier take from Python."""
|
||||
from . import dam as dam_mod
|
||||
|
||||
return {
|
||||
@@ -166,14 +208,14 @@ def flood_config() -> dict:
|
||||
"serving_days": dam_mod.SERVING_DAYS,
|
||||
},
|
||||
"notify": {
|
||||
"clear_margin_m": float(notify.CLEAR_MARGIN_M),
|
||||
"capacity_guard_min_pct": float(notify.CAPACITY_GUARD_MIN_PCT),
|
||||
"capacity_guard_exempt": sorted(notify.CAPACITY_GUARD_EXEMPT),
|
||||
"outlook_on": float(notify.OUTLOOK_ON),
|
||||
"outlook_off": float(notify.OUTLOOK_OFF),
|
||||
"outlook_horizon": int(notify.OUTLOOK_HORIZON),
|
||||
"feed_stale_after_h": float(notify.FEED_STALE_AFTER_H),
|
||||
"station_names": dict(notify.STATION_NAMES),
|
||||
"clear_margin_m": float(CLEAR_MARGIN_M),
|
||||
"capacity_guard_min_pct": float(CAPACITY_GUARD_MIN_PCT),
|
||||
"capacity_guard_exempt": sorted(CAPACITY_GUARD_EXEMPT),
|
||||
"outlook_on": float(OUTLOOK_ON),
|
||||
"outlook_off": float(OUTLOOK_OFF),
|
||||
"outlook_horizon": int(OUTLOOK_HORIZON),
|
||||
"feed_stale_after_h": float(FEED_STALE_AFTER_H),
|
||||
"station_names": dict(STATION_NAMES),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
+4
-3
@@ -385,10 +385,11 @@ def get_latest_forecasts(
|
||||
now: Optional[datetime.datetime] = None,
|
||||
rain: Optional[pd.Series] = None,
|
||||
) -> List[dict]:
|
||||
"""Convenience wrapper for web_api: load the latest window from the DB/API and forecast.
|
||||
"""Load the latest window from the DB/API and forecast: the Python reference
|
||||
for the Rust forecaster (scripts/forecast_parity.py,
|
||||
rust/forecast/testdata/gen_forecast_fixtures.py).
|
||||
|
||||
Raises FileNotFoundError when no trained model bundle exists at all, so the
|
||||
API can 503 instead of serving purely heuristic output as if it were a forecast.
|
||||
Raises FileNotFoundError when no trained model bundle exists at all.
|
||||
``now`` and ``rain`` are for reproducible comparisons
|
||||
(scripts/forecast_parity.py); by default the clock and a live fetch.
|
||||
"""
|
||||
|
||||
+12
-21
@@ -158,7 +158,7 @@ def serving_series() -> Optional[pd.Series]:
|
||||
return None
|
||||
|
||||
|
||||
def backfill_db(engine, db_type: str, chunk_rows: int = 5000) -> int:
|
||||
def backfill_db(engine, chunk_rows: int = 5000) -> int:
|
||||
"""Push the full Open-Meteo history (2021+) into openmeteo_rain.
|
||||
|
||||
Loads (or fetches) the archive cache and upserts in chunks; idempotent,
|
||||
@@ -171,19 +171,19 @@ def backfill_db(engine, db_type: str, chunk_rows: int = 5000) -> int:
|
||||
total = 0
|
||||
for start in range(0, len(history), chunk_rows):
|
||||
part = history.iloc[start : start + chunk_rows]
|
||||
total += save_to_db(part, engine, db_type)
|
||||
total += save_to_db(part, engine)
|
||||
logger.info(f"openmeteo_rain backfill: {total}/{len(history)} rows")
|
||||
return total
|
||||
|
||||
|
||||
def save_to_db(df: pd.DataFrame, engine, db_type: str) -> int:
|
||||
def save_to_db(df: pd.DataFrame, engine) -> int:
|
||||
"""Upsert per-point + catchment-mean hourly rain into openmeteo_rain.
|
||||
|
||||
Called by the leader worker's hourly precompute with the live forecast
|
||||
frame, so the DB accumulates both what fell (past rows are the model
|
||||
analysis) and what was forecast (future rows, overwritten as they become
|
||||
past). The ML training path reads Open-Meteo's own archive, not this
|
||||
table — this is for dashboards, SQL analysis, and source independence.
|
||||
The live writer is ping-collector (rust/collector/src/rain.rs, hourly,
|
||||
same table and values): past rows are the model analysis, future rows
|
||||
the forecast, overwritten as they become past. This Python path is the
|
||||
archive backfill (scripts/backfill_rain_db.py). The ML training path
|
||||
reads Open-Meteo's own archive, not this table.
|
||||
"""
|
||||
if df is None or df.empty:
|
||||
return 0
|
||||
@@ -199,20 +199,11 @@ def save_to_db(df: pd.DataFrame, engine, db_type: str) -> int:
|
||||
)
|
||||
cols = ["timestamp"] + point_cols + ["catchment_mean"]
|
||||
placeholders = ", ".join(f":{c}" for c in cols)
|
||||
updates = ", ".join(
|
||||
f"{c} = " + (f"VALUES({c})" if db_type == "mysql" else f"EXCLUDED.{c}")
|
||||
for c in cols[1:]
|
||||
updates = ", ".join(f"{c} = EXCLUDED.{c}" for c in cols[1:])
|
||||
sql = (
|
||||
f"INSERT INTO openmeteo_rain ({', '.join(cols)}) VALUES ({placeholders}) "
|
||||
f"ON CONFLICT (timestamp) DO UPDATE SET {updates}"
|
||||
)
|
||||
if db_type == "mysql":
|
||||
sql = (
|
||||
f"INSERT INTO openmeteo_rain ({', '.join(cols)}) VALUES ({placeholders}) "
|
||||
f"ON DUPLICATE KEY UPDATE {updates}"
|
||||
)
|
||||
else:
|
||||
sql = (
|
||||
f"INSERT INTO openmeteo_rain ({', '.join(cols)}) VALUES ({placeholders}) "
|
||||
f"ON CONFLICT (timestamp) DO UPDATE SET {updates}"
|
||||
)
|
||||
mean = df.mean(axis=1)
|
||||
params = [
|
||||
{
|
||||
|
||||
-170
@@ -1,170 +0,0 @@
|
||||
"""Live forecast skill: what the deployed model said versus what the river did.
|
||||
|
||||
Every hour the precompute stores the issued 24 h forecast (forecast_history);
|
||||
water_measurements holds what actually happened. Joining the two gives a
|
||||
verification that needs no retraining and answers the question the dashboard
|
||||
is asked most: "is the model getting better?" — per model version, on the
|
||||
hours that version was actually serving.
|
||||
|
||||
Metrics per version and horizon:
|
||||
n verified forecasts (issued, and the horizon has since elapsed)
|
||||
mae |predicted_max - observed_max| over the horizon window, metres
|
||||
bias mean(predicted - observed): >0 over-predicts the peak
|
||||
persistence MAE of the trivial "peak = current level" forecast on the
|
||||
same rows; a model is only useful if it beats this
|
||||
skill 1 - mae/persistence (0 = no better than persistence, 1 = perfect)
|
||||
above_2m same MAE restricted to rows where the observed peak >= 2 m,
|
||||
i.e. the flood-relevant regime
|
||||
|
||||
Only the P.1 gauge is verified by default: it is the one the city threshold
|
||||
is keyed to, and one station keeps the query cheap enough to run on request.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import logging
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DEFAULT_STATION = "P.1"
|
||||
DEFAULT_HORIZON = 24
|
||||
MIN_VERIFIED = 24 # fewer than a day of verified hours is not a number
|
||||
|
||||
|
||||
def _sql_for(db_type: str) -> str:
|
||||
"""Join each issued forecast to the observed max over (as_of, as_of + h]."""
|
||||
if db_type == "postgresql":
|
||||
window_end = "f.as_of + (f.horizon_hours || ' hours')::interval"
|
||||
elif db_type == "mysql":
|
||||
window_end = "DATE_ADD(f.as_of, INTERVAL f.horizon_hours HOUR)"
|
||||
else: # sqlite
|
||||
window_end = "datetime(f.as_of, '+' || f.horizon_hours || ' hours')"
|
||||
return f"""
|
||||
SELECT f.as_of, f.model_version, f.predicted_max_level, f.current_level,
|
||||
(SELECT MAX(m.water_level) FROM water_measurements m
|
||||
JOIN stations s ON s.id = m.station_id
|
||||
WHERE s.station_code = f.station_code
|
||||
AND m.timestamp > f.as_of AND m.timestamp <= {window_end}) AS observed_max,
|
||||
(SELECT COUNT(m.water_level) FROM water_measurements m
|
||||
JOIN stations s ON s.id = m.station_id
|
||||
WHERE s.station_code = f.station_code
|
||||
AND m.timestamp > f.as_of AND m.timestamp <= {window_end}) AS observed_n
|
||||
FROM forecast_history f
|
||||
WHERE f.station_code = :code AND f.horizon_hours = :horizon
|
||||
AND f.source = 'model' AND f.predicted_max_level IS NOT NULL
|
||||
AND f.as_of <= :verifiable_before
|
||||
ORDER BY f.as_of
|
||||
"""
|
||||
|
||||
|
||||
def compute_skill(
|
||||
engine,
|
||||
db_type: str,
|
||||
station_code: str = DEFAULT_STATION,
|
||||
horizon_hours: int = DEFAULT_HORIZON,
|
||||
now: Optional[datetime.datetime] = None,
|
||||
) -> Dict:
|
||||
"""Per-model-version verification of issued forecasts against observations."""
|
||||
from sqlalchemy import text
|
||||
|
||||
now = now or datetime.datetime.now()
|
||||
verifiable_before = now - datetime.timedelta(hours=horizon_hours)
|
||||
with engine.connect() as conn:
|
||||
rows = [
|
||||
dict(r._mapping)
|
||||
for r in conn.execute(
|
||||
text(_sql_for(db_type)),
|
||||
{
|
||||
"code": station_code,
|
||||
"horizon": horizon_hours,
|
||||
"verifiable_before": verifiable_before,
|
||||
},
|
||||
)
|
||||
]
|
||||
|
||||
def _ts(value):
|
||||
# sqlite hands back strings; postgres/mysql give datetimes
|
||||
if isinstance(value, datetime.datetime):
|
||||
return value
|
||||
return datetime.datetime.fromisoformat(str(value).replace(" ", "T"))
|
||||
|
||||
by_version: Dict[str, List[dict]] = {}
|
||||
for r in rows:
|
||||
r["as_of"] = _ts(r["as_of"])
|
||||
# need most of the window observed, or the "max" is not the peak
|
||||
if r["observed_max"] is None or (r["observed_n"] or 0) < horizon_hours * 0.75:
|
||||
continue
|
||||
by_version.setdefault(r["model_version"] or "unknown", []).append(r)
|
||||
|
||||
versions = []
|
||||
for version, vrows in by_version.items():
|
||||
pred = [float(r["predicted_max_level"]) for r in vrows]
|
||||
obs = [float(r["observed_max"]) for r in vrows]
|
||||
cur = [
|
||||
float(r["current_level"]) if r["current_level"] is not None else None
|
||||
for r in vrows
|
||||
]
|
||||
err = [p - o for p, o in zip(pred, obs)]
|
||||
mae = sum(abs(e) for e in err) / len(err)
|
||||
bias = sum(err) / len(err)
|
||||
pers_rows = [(c, o) for c, o in zip(cur, obs) if c is not None]
|
||||
persistence = (
|
||||
sum(abs(c - o) for c, o in pers_rows) / len(pers_rows)
|
||||
if pers_rows
|
||||
else None
|
||||
)
|
||||
high = [(p, o) for p, o in zip(pred, obs) if o >= 2.0]
|
||||
versions.append(
|
||||
{
|
||||
"model_version": version,
|
||||
"first_issued": min(r["as_of"] for r in vrows).isoformat(),
|
||||
"last_issued": max(r["as_of"] for r in vrows).isoformat(),
|
||||
"n": len(vrows),
|
||||
"mae_m": round(mae, 3),
|
||||
"bias_m": round(bias, 3),
|
||||
"persistence_mae_m": (
|
||||
None if persistence is None else round(persistence, 3)
|
||||
),
|
||||
"skill": (
|
||||
None if not persistence else round(1.0 - mae / persistence, 3)
|
||||
),
|
||||
"above_2m_n": len(high),
|
||||
"above_2m_mae_m": (
|
||||
round(sum(abs(p - o) for p, o in high) / len(high), 3)
|
||||
if high
|
||||
else None
|
||||
),
|
||||
"enough_data": len(vrows) >= MIN_VERIFIED,
|
||||
}
|
||||
)
|
||||
versions.sort(key=lambda v: v["first_issued"])
|
||||
|
||||
# Headline: current version vs the previous one that had enough data
|
||||
current = versions[-1] if versions else None
|
||||
previous = (
|
||||
next((v for v in reversed(versions[:-1]) if v["enough_data"]), None)
|
||||
if versions
|
||||
else None
|
||||
)
|
||||
trend = None
|
||||
if current and previous and current["enough_data"]:
|
||||
trend = {
|
||||
"previous_version": previous["model_version"],
|
||||
"mae_delta_m": round(current["mae_m"] - previous["mae_m"], 3),
|
||||
"skill_delta": (
|
||||
None
|
||||
if current["skill"] is None or previous["skill"] is None
|
||||
else round(current["skill"] - previous["skill"], 3)
|
||||
),
|
||||
"better": current["mae_m"] < previous["mae_m"],
|
||||
}
|
||||
|
||||
return {
|
||||
"station_code": station_code,
|
||||
"horizon_hours": horizon_hours,
|
||||
"verified_until": verifiable_before.isoformat(),
|
||||
"min_verified": MIN_VERIFIED,
|
||||
"versions": versions,
|
||||
"current": current,
|
||||
"trend": trend,
|
||||
}
|
||||
-121
@@ -1,121 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Data models for water monitoring system
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
|
||||
class DatabaseType(Enum):
|
||||
SQLITE = "sqlite"
|
||||
MYSQL = "mysql"
|
||||
POSTGRESQL = "postgresql"
|
||||
INFLUXDB = "influxdb"
|
||||
VICTORIAMETRICS = "victoriametrics"
|
||||
|
||||
|
||||
class StationStatus(Enum):
|
||||
ACTIVE = "active"
|
||||
INACTIVE = "inactive"
|
||||
MAINTENANCE = "maintenance"
|
||||
ERROR = "error"
|
||||
|
||||
|
||||
@dataclass
|
||||
class StationInfo:
|
||||
"""Station information model"""
|
||||
|
||||
station_id: int
|
||||
station_code: str
|
||||
thai_name: str
|
||||
english_name: str
|
||||
latitude: Optional[float] = None
|
||||
longitude: Optional[float] = None
|
||||
geohash: Optional[str] = None
|
||||
status: StationStatus = StationStatus.ACTIVE
|
||||
|
||||
|
||||
@dataclass
|
||||
class WaterMeasurement:
|
||||
"""Water measurement data model"""
|
||||
|
||||
timestamp: datetime
|
||||
station_info: StationInfo
|
||||
water_level: float
|
||||
discharge: float
|
||||
water_level_unit: str = "m"
|
||||
discharge_unit: str = "cms"
|
||||
discharge_percent: Optional[float] = None
|
||||
status: StationStatus = StationStatus.ACTIVE
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert to dictionary for database storage"""
|
||||
return {
|
||||
"timestamp": self.timestamp,
|
||||
"station_id": self.station_info.station_id,
|
||||
"station_code": self.station_info.station_code,
|
||||
"station_name_en": self.station_info.english_name,
|
||||
"station_name_th": self.station_info.thai_name,
|
||||
"latitude": self.station_info.latitude,
|
||||
"longitude": self.station_info.longitude,
|
||||
"geohash": self.station_info.geohash,
|
||||
"water_level": self.water_level,
|
||||
"water_level_unit": self.water_level_unit,
|
||||
"discharge": self.discharge,
|
||||
"discharge_unit": self.discharge_unit,
|
||||
"discharge_percent": self.discharge_percent,
|
||||
"status": self.status.value,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class DatabaseConfig:
|
||||
"""Database configuration model"""
|
||||
|
||||
db_type: DatabaseType
|
||||
connection_string: Optional[str] = None
|
||||
host: Optional[str] = None
|
||||
port: Optional[int] = None
|
||||
database: Optional[str] = None
|
||||
username: Optional[str] = None
|
||||
password: Optional[str] = None
|
||||
additional_params: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScrapingResult:
|
||||
"""Result of a scraping operation"""
|
||||
|
||||
success: bool
|
||||
measurements_count: int
|
||||
error_message: Optional[str] = None
|
||||
timestamp: datetime = field(default_factory=datetime.now)
|
||||
processing_time_seconds: Optional[float] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class StationCreateRequest:
|
||||
"""Request model for creating a new station"""
|
||||
|
||||
station_code: str
|
||||
thai_name: str
|
||||
english_name: str
|
||||
latitude: Optional[float] = None
|
||||
longitude: Optional[float] = None
|
||||
geohash: Optional[str] = None
|
||||
status: StationStatus = StationStatus.ACTIVE
|
||||
|
||||
|
||||
@dataclass
|
||||
class StationUpdateRequest:
|
||||
"""Request model for updating an existing station"""
|
||||
|
||||
thai_name: Optional[str] = None
|
||||
english_name: Optional[str] = None
|
||||
latitude: Optional[float] = None
|
||||
longitude: Optional[float] = None
|
||||
geohash: Optional[str] = None
|
||||
status: Optional[StationStatus] = None
|
||||
-456
@@ -1,456 +0,0 @@
|
||||
"""Public flood notifications over ntfy.
|
||||
|
||||
Runs once per collection cycle inside the API process (leader only), right
|
||||
after the forecast precompute, so it sees the same readings and forecasts the
|
||||
dashboard shows. Publishes to a self-hosted ntfy server; anyone subscribes to
|
||||
a topic from the free app or a browser, no account needed.
|
||||
|
||||
Topics (all under one configurable prefix, default "ping"):
|
||||
|
||||
{prefix}-{station}-warning observed level crossed the station's warning threshold
|
||||
{prefix}-{station}-danger observed level crossed the danger threshold
|
||||
{prefix}-warning any station crossed warning (basin-wide digest)
|
||||
{prefix}-danger any station crossed danger
|
||||
{prefix}-p1-outlook model early warning for Chiang Mai city: P.1's 24 h
|
||||
warning probability crossed the alert level (opt-in;
|
||||
the forecast is experimental and says so)
|
||||
{prefix}-status feed/monitor health: data stale, recovered
|
||||
|
||||
Each notification is a TRANSITION, not a state: crossing UP into a level sends
|
||||
one message; dropping back below (with hysteresis) sends an all-clear. While
|
||||
the river sits above a threshold nothing is repeated, so a subscriber in a
|
||||
flood gets a handful of messages, not one an hour. The per-topic state is
|
||||
persisted (notification_state table) so a restart never re-sends.
|
||||
|
||||
Everything is fail-safe: ntfy unreachable, table missing, malformed
|
||||
reading -> a logged warning, never an exception into the collection loop.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from typing import Dict, Iterable, List, Optional
|
||||
|
||||
import requests
|
||||
|
||||
from .ml import features
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Hysteresis: an all-clear needs the level this far BELOW the threshold, so a
|
||||
# river bobbing around 3.70 m does not toggle warning/clear every hour.
|
||||
CLEAR_MARGIN_M = 0.10
|
||||
# Capacity guard. The level thresholds in features.THRESHOLDS were calibrated
|
||||
# from RID's discharge_percent (% of channel capacity); if RID re-rates a
|
||||
# gauge or moves its datum, the level crosses while capacity says the channel
|
||||
# is nearly empty (P.77, 2026-09: 3.0 m "warning" at 22 %). A crossing is
|
||||
# only announced when the reported capacity agrees that the river is high.
|
||||
# P.1 is exempt: its stages come from the municipal inundation map, not from
|
||||
# capacity. Readings without a capacity figure fall back to level only.
|
||||
CAPACITY_GUARD_MIN_PCT = 60.0
|
||||
CAPACITY_GUARD_EXEMPT = {"P.1"}
|
||||
# Outlook alert fires when p_warning(24h) rises through ON, clears below OFF.
|
||||
OUTLOOK_ON = 0.50
|
||||
OUTLOOK_OFF = 0.25
|
||||
# Below this the outlook is not announced at all (avoid "5 % chance" noise).
|
||||
OUTLOOK_HORIZON = 24
|
||||
# No reading this many hours old or newer: the feed is stale.
|
||||
FEED_STALE_AFTER_H = 3.0
|
||||
|
||||
STATION_NAMES: Dict[str, str] = {
|
||||
"P.1": "Nawarat Bridge, Chiang Mai city",
|
||||
"P.103": "Ring Road Bridge 3, Chiang Mai",
|
||||
"P.67": "Ban Tae (Mae Taeng)",
|
||||
"P.21": "Ban Rim Tai (Mae Rim)",
|
||||
"P.75": "Ban Chai Lat",
|
||||
"P.92": "Ban Muang Aut",
|
||||
"P.20": "Ban Chiang Dao",
|
||||
"P.4A": "Ban Mae Taeng",
|
||||
"P.5": "Tha Nang Bridge (downstream)",
|
||||
"P.81": "Ban Pong (downstream)",
|
||||
"P.82": "Ban Sob Win",
|
||||
"P.84": "Ban Panton",
|
||||
"P.87": "Ban Pa Sang",
|
||||
"P.77": "Ban Sop Mae Sapuat",
|
||||
"P.85": "Ban Lai Kaew",
|
||||
"P.76": "Ban Mae I Hai",
|
||||
}
|
||||
|
||||
|
||||
def _slug(code: str) -> str:
|
||||
return code.lower().replace(".", "")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Notification:
|
||||
topic: str
|
||||
title: str
|
||||
message: str
|
||||
priority: int = 3 # ntfy: 1 min .. 5 max
|
||||
tags: Optional[List[str]] = None
|
||||
click: Optional[str] = None
|
||||
|
||||
|
||||
class NtfyPublisher:
|
||||
def __init__(
|
||||
self,
|
||||
server: str,
|
||||
prefix: str = "ping",
|
||||
token: Optional[str] = None,
|
||||
dashboard_url: str = "https://water.buildfor.life/",
|
||||
timeout: int = 10,
|
||||
):
|
||||
self.server = server.rstrip("/")
|
||||
self.prefix = prefix
|
||||
self.token = token
|
||||
self.dashboard_url = dashboard_url
|
||||
self.timeout = timeout
|
||||
|
||||
def topic(self, *parts: str) -> str:
|
||||
return "-".join([self.prefix, *parts])
|
||||
|
||||
def publish(self, n: Notification) -> bool:
|
||||
headers = {
|
||||
"Title": n.title,
|
||||
"Priority": str(n.priority),
|
||||
"Click": n.click or self.dashboard_url,
|
||||
"Actions": f"view, Open dashboard, {n.click or self.dashboard_url}",
|
||||
}
|
||||
if n.tags:
|
||||
headers["Tags"] = ",".join(n.tags)
|
||||
if self.token:
|
||||
headers["Authorization"] = f"Bearer {self.token}"
|
||||
try:
|
||||
r = requests.post(
|
||||
f"{self.server}/{n.topic}",
|
||||
data=n.message.encode("utf-8"),
|
||||
headers=headers,
|
||||
timeout=self.timeout,
|
||||
)
|
||||
if r.status_code >= 300:
|
||||
logger.warning(f"ntfy {n.topic}: HTTP {r.status_code} {r.text[:120]}")
|
||||
return False
|
||||
return True
|
||||
except Exception as error:
|
||||
logger.warning(f"ntfy {n.topic}: {error}")
|
||||
return False
|
||||
|
||||
|
||||
class NotificationState:
|
||||
"""Per-key last-sent state, in the monitor's own SQL database."""
|
||||
|
||||
def __init__(self, engine, db_type: str):
|
||||
self.engine = engine
|
||||
self.db_type = db_type
|
||||
self._ensure()
|
||||
|
||||
def _ensure(self) -> None:
|
||||
from sqlalchemy import text
|
||||
|
||||
ddl = (
|
||||
"CREATE TABLE IF NOT EXISTS notification_state ("
|
||||
"key VARCHAR(64) PRIMARY KEY, state VARCHAR(16) NOT NULL, "
|
||||
"value NUMERIC(8,3), updated_at TIMESTAMP NOT NULL)"
|
||||
)
|
||||
try:
|
||||
with self.engine.begin() as conn:
|
||||
conn.execute(text(ddl))
|
||||
except Exception as error:
|
||||
# Postgres: two sessions racing CREATE TABLE IF NOT EXISTS can
|
||||
# both pass the existence check; the loser fails with a unique
|
||||
# violation on pg_type. The table exists either way; verify.
|
||||
with self.engine.connect() as conn:
|
||||
conn.execute(text("SELECT 1 FROM notification_state WHERE 1=0"))
|
||||
logger.debug(f"notification_state DDL raced, table present: {error}")
|
||||
|
||||
def get(self, key: str) -> Optional[str]:
|
||||
from sqlalchemy import text
|
||||
|
||||
with self.engine.connect() as conn:
|
||||
row = conn.execute(
|
||||
text("SELECT state FROM notification_state WHERE key = :k"), {"k": key}
|
||||
).fetchone()
|
||||
return row[0] if row else None
|
||||
|
||||
def set(self, key: str, state: str, value: Optional[float] = None) -> None:
|
||||
from sqlalchemy import text
|
||||
|
||||
now = datetime.datetime.now()
|
||||
with self.engine.begin() as conn:
|
||||
if self.db_type == "mysql":
|
||||
sql = (
|
||||
"INSERT INTO notification_state (key, state, value, updated_at) "
|
||||
"VALUES (:k, :s, :v, :t) ON DUPLICATE KEY UPDATE "
|
||||
"state = VALUES(state), value = VALUES(value), updated_at = VALUES(updated_at)"
|
||||
)
|
||||
else:
|
||||
sql = (
|
||||
"INSERT INTO notification_state (key, state, value, updated_at) "
|
||||
"VALUES (:k, :s, :v, :t) ON CONFLICT (key) DO UPDATE SET "
|
||||
"state = EXCLUDED.state, value = EXCLUDED.value, updated_at = EXCLUDED.updated_at"
|
||||
)
|
||||
conn.execute(text(sql), {"k": key, "s": state, "v": value, "t": now})
|
||||
|
||||
|
||||
class InMemoryState(NotificationState):
|
||||
"""For tests and when no SQL engine is available (loses state on restart)."""
|
||||
|
||||
def __init__(self): # noqa: D107 - intentionally skips the SQL parent
|
||||
self._d: Dict[str, str] = {}
|
||||
|
||||
def get(self, key: str) -> Optional[str]:
|
||||
return self._d.get(key)
|
||||
|
||||
def set(self, key: str, state: str, value: Optional[float] = None) -> None:
|
||||
self._d[key] = state
|
||||
|
||||
|
||||
def _level_state(level: float, warn: float, danger: float, prev: Optional[str]) -> str:
|
||||
"""'clear' | 'warning' | 'danger', with hysteresis on the way down."""
|
||||
if level >= danger:
|
||||
return "danger"
|
||||
if level >= warn:
|
||||
# from danger: stay 'danger' until below danger - margin
|
||||
if prev == "danger" and level >= danger - CLEAR_MARGIN_M:
|
||||
return "danger"
|
||||
return "warning"
|
||||
if prev in ("warning", "danger") and level >= warn - CLEAR_MARGIN_M:
|
||||
return "warning"
|
||||
return "clear"
|
||||
|
||||
|
||||
def evaluate(
|
||||
readings: Iterable[dict],
|
||||
forecasts: Iterable[dict],
|
||||
state: NotificationState,
|
||||
publisher: NtfyPublisher,
|
||||
stale_after_h: float = FEED_STALE_AFTER_H,
|
||||
now: Optional[datetime.datetime] = None,
|
||||
) -> List[Notification]:
|
||||
"""Compare current readings/forecasts with last-sent state; publish transitions.
|
||||
|
||||
readings: rows with station_code, water_level, timestamp (latest per station)
|
||||
forecasts: /forecast rows (station_code, horizon_hours, p_warning, predicted_max_level)
|
||||
Returns the notifications that were published (for logs/tests).
|
||||
"""
|
||||
now = now or datetime.datetime.now()
|
||||
sent: List[Notification] = []
|
||||
|
||||
def emit(n: Notification) -> bool:
|
||||
ok = publisher.publish(n)
|
||||
if ok:
|
||||
sent.append(n)
|
||||
return ok
|
||||
|
||||
# ---- observed levels, per station, plus basin-wide fan-out
|
||||
basin_changes: Dict[str, List[str]] = {"warning": [], "danger": [], "clear": []}
|
||||
latest_ts: Optional[datetime.datetime] = None
|
||||
for r in readings:
|
||||
code = r.get("station_code")
|
||||
level = r.get("water_level")
|
||||
if not code or level is None:
|
||||
continue
|
||||
try:
|
||||
level = float(level)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
ts = r.get("timestamp")
|
||||
if isinstance(ts, str):
|
||||
try:
|
||||
ts = datetime.datetime.fromisoformat(ts)
|
||||
except ValueError:
|
||||
ts = None
|
||||
if isinstance(ts, datetime.datetime) and (latest_ts is None or ts > latest_ts):
|
||||
latest_ts = ts
|
||||
warn, danger = features.get_thresholds(code)
|
||||
key = f"level:{code}"
|
||||
prev = state.get(key) or "clear"
|
||||
cur = _level_state(level, warn, danger, prev)
|
||||
pct = r.get("discharge_percent")
|
||||
if (
|
||||
cur != "clear"
|
||||
and prev == "clear"
|
||||
and code not in CAPACITY_GUARD_EXEMPT
|
||||
and pct is not None
|
||||
):
|
||||
try:
|
||||
if float(pct) < CAPACITY_GUARD_MIN_PCT:
|
||||
logger.info(
|
||||
f"{code}: level {level:.2f} m >= {warn:.2f} but only "
|
||||
f"{float(pct):.0f}% capacity; threshold looks stale, not alerting"
|
||||
)
|
||||
continue
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
if cur == prev:
|
||||
continue
|
||||
name = STATION_NAMES.get(code, code)
|
||||
slug = _slug(code)
|
||||
when = (
|
||||
ts.strftime("%d %b %H:%M") if isinstance(ts, datetime.datetime) else "now"
|
||||
)
|
||||
if cur == "danger":
|
||||
ok = emit(
|
||||
Notification(
|
||||
publisher.topic(slug, "danger"),
|
||||
f"DANGER level at {code}",
|
||||
f"{name}: {level:.2f} m at {when}, above the danger level of {danger:.2f} m.",
|
||||
priority=5,
|
||||
tags=["rotating_light", code],
|
||||
)
|
||||
)
|
||||
basin_changes["danger"].append(f"{code} {level:.2f} m")
|
||||
elif cur == "warning":
|
||||
if prev == "danger":
|
||||
ok = emit(
|
||||
Notification(
|
||||
publisher.topic(slug, "danger"),
|
||||
f"{code} back below danger level",
|
||||
f"{name}: {level:.2f} m at {when}; still above the warning level of {warn:.2f} m.",
|
||||
priority=3,
|
||||
tags=["arrow_down", code],
|
||||
)
|
||||
)
|
||||
basin_changes["clear"].append(f"{code} below danger ({level:.2f} m)")
|
||||
else:
|
||||
ok = emit(
|
||||
Notification(
|
||||
publisher.topic(slug, "warning"),
|
||||
f"Warning level at {code}",
|
||||
f"{name}: {level:.2f} m at {when}, above the warning level of {warn:.2f} m.",
|
||||
priority=4,
|
||||
tags=["warning", code],
|
||||
)
|
||||
)
|
||||
basin_changes["warning"].append(f"{code} {level:.2f} m")
|
||||
else: # clear
|
||||
ok = emit(
|
||||
Notification(
|
||||
publisher.topic(slug, "warning"),
|
||||
f"{code} back to normal",
|
||||
f"{name}: {level:.2f} m at {when}, below the warning level of {warn:.2f} m.",
|
||||
priority=2,
|
||||
tags=["white_check_mark", code],
|
||||
)
|
||||
)
|
||||
basin_changes["clear"].append(f"{code} normal ({level:.2f} m)")
|
||||
# Only remember the transition once it was actually delivered: if ntfy
|
||||
# was down, the next cycle retries instead of silently swallowing a
|
||||
# flood crossing.
|
||||
if ok:
|
||||
state.set(key, cur, level)
|
||||
|
||||
if basin_changes["danger"]:
|
||||
emit(
|
||||
Notification(
|
||||
publisher.topic("danger"),
|
||||
"Ping River: danger level reached",
|
||||
"; ".join(basin_changes["danger"]),
|
||||
priority=5,
|
||||
tags=["rotating_light"],
|
||||
)
|
||||
)
|
||||
if basin_changes["warning"]:
|
||||
emit(
|
||||
Notification(
|
||||
publisher.topic("warning"),
|
||||
"Ping River: warning level reached",
|
||||
"; ".join(basin_changes["warning"]),
|
||||
priority=4,
|
||||
tags=["warning"],
|
||||
)
|
||||
)
|
||||
if basin_changes["clear"]:
|
||||
emit(
|
||||
Notification(
|
||||
publisher.topic("warning"),
|
||||
"Ping River: levels falling",
|
||||
"; ".join(basin_changes["clear"]),
|
||||
priority=2,
|
||||
tags=["white_check_mark"],
|
||||
)
|
||||
)
|
||||
|
||||
# ---- model outlook for the city gauge (opt-in topic, experimental)
|
||||
p1 = next(
|
||||
(
|
||||
f
|
||||
for f in forecasts
|
||||
if f.get("station_code") == "P.1"
|
||||
and f.get("horizon_hours") == OUTLOOK_HORIZON
|
||||
and f.get("source") == "model"
|
||||
),
|
||||
None,
|
||||
)
|
||||
if p1 and p1.get("p_warning") is not None:
|
||||
p = float(p1["p_warning"])
|
||||
key = "outlook:P.1"
|
||||
prev = state.get(key) or "off"
|
||||
cur = (
|
||||
"on" if (p >= OUTLOOK_ON or (prev == "on" and p >= OUTLOOK_OFF)) else "off"
|
||||
)
|
||||
if cur != prev:
|
||||
peak = p1.get("predicted_max_level")
|
||||
warn, _ = features.get_thresholds("P.1")
|
||||
if cur == "on":
|
||||
ok = emit(
|
||||
Notification(
|
||||
publisher.topic("p1-outlook"),
|
||||
"Early warning: Chiang Mai flood risk rising",
|
||||
f"The forecast model gives a {p * 100:.0f}% chance that Nawarat Bridge (P.1) "
|
||||
f"reaches {warn:.2f} m within 24 h"
|
||||
+ (
|
||||
f" (expected peak {float(peak):.2f} m)"
|
||||
if peak is not None
|
||||
else ""
|
||||
)
|
||||
+ ". Experimental model output, not an official warning; "
|
||||
"follow ThaiWater/TMD for official alerts.",
|
||||
priority=4,
|
||||
tags=["crystal_ball"],
|
||||
)
|
||||
)
|
||||
else:
|
||||
ok = emit(
|
||||
Notification(
|
||||
publisher.topic("p1-outlook"),
|
||||
"Chiang Mai flood risk easing",
|
||||
f"The model's 24 h probability of reaching {warn:.2f} m at P.1 has dropped to {p * 100:.0f}%.",
|
||||
priority=2,
|
||||
tags=["crystal_ball"],
|
||||
)
|
||||
)
|
||||
if ok:
|
||||
state.set(key, cur, p)
|
||||
|
||||
# ---- feed health
|
||||
if latest_ts is not None:
|
||||
age_h = (now - latest_ts).total_seconds() / 3600.0
|
||||
key = "feed"
|
||||
prev = state.get(key) or "ok"
|
||||
cur = "stale" if age_h >= stale_after_h else "ok"
|
||||
if cur != prev:
|
||||
if cur == "stale":
|
||||
ok = emit(
|
||||
Notification(
|
||||
publisher.topic("status"),
|
||||
"Ping River monitor: gauge feed stale",
|
||||
f"No new readings for {age_h:.0f} h (last {latest_ts:%d %b %H:%M}). "
|
||||
"Levels and forecasts on the dashboard are not current.",
|
||||
priority=3,
|
||||
tags=["hourglass"],
|
||||
)
|
||||
)
|
||||
else:
|
||||
ok = emit(
|
||||
Notification(
|
||||
publisher.topic("status"),
|
||||
"Ping River monitor: feed recovered",
|
||||
f"Readings are current again (latest {latest_ts:%d %b %H:%M}).",
|
||||
priority=2,
|
||||
tags=["white_check_mark"],
|
||||
)
|
||||
)
|
||||
if ok:
|
||||
state.set(key, cur, age_h)
|
||||
return sent
|
||||
@@ -1,97 +0,0 @@
|
||||
"""Read historical station measurements from PostgreSQL."""
|
||||
|
||||
import datetime
|
||||
import os
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
from sqlalchemy import create_engine, text
|
||||
|
||||
# Stage-discharge rating curves: Q = a * (H - b)^c
|
||||
# Key: station_code, Value: (a, b, c)
|
||||
# Use linear fallback Q = slope * H if a curve is not defined.
|
||||
_RATING_CURVES: Dict[str, Tuple[float, float, float]] = {}
|
||||
_DEFAULT_LINEAR_SLOPE = 20.0 # m^3/s per meter
|
||||
|
||||
|
||||
def _calculate_discharge(
|
||||
water_level: Optional[float], station_code: str = None
|
||||
) -> Optional[float]:
|
||||
"""Estimate discharge from water level using a rating curve or linear fallback."""
|
||||
if water_level is None:
|
||||
return None
|
||||
|
||||
curve = _RATING_CURVES.get(station_code)
|
||||
if curve:
|
||||
a, b, c = curve
|
||||
h_excess = water_level - b
|
||||
if h_excess <= 0:
|
||||
return 0.0
|
||||
return round(a * (h_excess**c), 2)
|
||||
|
||||
# Linear fallback: Q = slope * H
|
||||
return round(_DEFAULT_LINEAR_SLOPE * water_level, 2)
|
||||
|
||||
|
||||
class PostgresHistory:
|
||||
def __init__(self, connection_string: Optional[str] = None, engine=None):
|
||||
connection_string = connection_string or os.getenv("POSTGRES_CONNECTION_STRING")
|
||||
if engine is None and not connection_string:
|
||||
raise RuntimeError("POSTGRES_CONNECTION_STRING is not configured")
|
||||
self.engine = engine or create_engine(connection_string, pool_pre_ping=True)
|
||||
|
||||
def station_history(
|
||||
self,
|
||||
station_code: str,
|
||||
start: datetime.datetime,
|
||||
end: datetime.datetime,
|
||||
limit: int = 2000,
|
||||
) -> List[Dict]:
|
||||
if not 1 <= limit <= 100000:
|
||||
raise ValueError("limit must be between 1 and 100000")
|
||||
if start >= end:
|
||||
raise ValueError("start must be before end")
|
||||
|
||||
query = text("""
|
||||
SELECT m.timestamp, s.station_code, m.water_level,
|
||||
m.discharge, m.discharge_percent
|
||||
FROM water_measurements m
|
||||
JOIN stations s ON m.station_id = s.id
|
||||
WHERE s.station_code = :station_code
|
||||
AND m.timestamp >= :start_time
|
||||
AND m.timestamp <= :end_time
|
||||
ORDER BY m.timestamp ASC
|
||||
LIMIT :limit
|
||||
""")
|
||||
with self.engine.connect() as connection:
|
||||
rows = connection.execute(
|
||||
query,
|
||||
{
|
||||
"station_code": station_code,
|
||||
"start_time": start,
|
||||
"end_time": end,
|
||||
"limit": limit,
|
||||
},
|
||||
)
|
||||
result = []
|
||||
for row in rows:
|
||||
timestamp = row[0]
|
||||
if isinstance(timestamp, str):
|
||||
timestamp = datetime.datetime.fromisoformat(timestamp)
|
||||
station_code = row[1]
|
||||
water_level = float(row[2]) if row[2] is not None else None
|
||||
discharge = float(row[3]) if row[3] is not None else None
|
||||
# Estimate discharge from water level if DB value is missing
|
||||
if discharge is None and water_level is not None:
|
||||
discharge = _calculate_discharge(water_level, station_code)
|
||||
result.append(
|
||||
{
|
||||
"timestamp": timestamp,
|
||||
"station_code": station_code,
|
||||
"water_level": water_level,
|
||||
"discharge": discharge,
|
||||
"discharge_percent": (
|
||||
float(row[4]) if row[4] is not None else None
|
||||
),
|
||||
}
|
||||
)
|
||||
return result
|
||||
@@ -1,182 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Rate limiting utilities for API requests
|
||||
"""
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from collections import deque
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RateLimiter:
|
||||
"""Token bucket rate limiter"""
|
||||
|
||||
def __init__(self, max_requests: int, time_window_seconds: int):
|
||||
"""
|
||||
Initialize rate limiter
|
||||
|
||||
Args:
|
||||
max_requests: Maximum number of requests allowed
|
||||
time_window_seconds: Time window in seconds
|
||||
"""
|
||||
self.max_requests = max_requests
|
||||
self.time_window = time_window_seconds
|
||||
self.requests = deque()
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def is_allowed(self) -> bool:
|
||||
"""Check if a request is allowed"""
|
||||
with self._lock:
|
||||
now = time.time()
|
||||
|
||||
# Remove old requests outside the time window
|
||||
while self.requests and self.requests[0] <= now - self.time_window:
|
||||
self.requests.popleft()
|
||||
|
||||
# Check if we can make a new request
|
||||
if len(self.requests) < self.max_requests:
|
||||
self.requests.append(now)
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def wait_time(self) -> float:
|
||||
"""Get the time to wait before next request is allowed"""
|
||||
with self._lock:
|
||||
if len(self.requests) < self.max_requests:
|
||||
return 0.0
|
||||
|
||||
# Time until the oldest request expires
|
||||
oldest_request = self.requests[0]
|
||||
return max(0.0, (oldest_request + self.time_window) - time.time())
|
||||
|
||||
def wait_if_needed(self):
|
||||
"""Block until a request is allowed"""
|
||||
wait_time = self.wait_time()
|
||||
if wait_time > 0:
|
||||
logger.info(f"Rate limit reached, waiting {wait_time:.2f} seconds")
|
||||
time.sleep(wait_time)
|
||||
|
||||
|
||||
class AdaptiveRateLimiter:
|
||||
"""Adaptive rate limiter that adjusts based on response times"""
|
||||
|
||||
def __init__(
|
||||
self, initial_rate: float = 1.0, min_rate: float = 0.1, max_rate: float = 10.0
|
||||
):
|
||||
"""
|
||||
Initialize adaptive rate limiter
|
||||
|
||||
Args:
|
||||
initial_rate: Initial requests per second
|
||||
min_rate: Minimum requests per second
|
||||
max_rate: Maximum requests per second
|
||||
"""
|
||||
self.current_rate = initial_rate
|
||||
self.min_rate = min_rate
|
||||
self.max_rate = max_rate
|
||||
self.last_request_time = 0.0
|
||||
self.response_times = deque(maxlen=10)
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def wait_and_record(self, response_time: Optional[float] = None):
|
||||
"""Wait for rate limit and record response time"""
|
||||
with self._lock:
|
||||
now = time.time()
|
||||
|
||||
# Calculate wait time based on current rate
|
||||
time_since_last = now - self.last_request_time
|
||||
min_interval = 1.0 / self.current_rate
|
||||
|
||||
if time_since_last < min_interval:
|
||||
wait_time = min_interval - time_since_last
|
||||
time.sleep(wait_time)
|
||||
now = time.time()
|
||||
|
||||
self.last_request_time = now
|
||||
|
||||
# Record response time and adjust rate
|
||||
if response_time is not None:
|
||||
self.response_times.append(response_time)
|
||||
self._adjust_rate()
|
||||
|
||||
def _adjust_rate(self):
|
||||
"""Adjust rate based on recent response times"""
|
||||
if len(self.response_times) < 3:
|
||||
return
|
||||
|
||||
avg_response_time = sum(self.response_times) / len(self.response_times)
|
||||
|
||||
# Decrease rate if responses are slow
|
||||
if avg_response_time > 5.0: # 5 seconds
|
||||
self.current_rate = max(self.min_rate, self.current_rate * 0.8)
|
||||
logger.info(
|
||||
f"Decreased rate to {self.current_rate:.2f} req/s due to slow responses"
|
||||
)
|
||||
|
||||
# Increase rate if responses are fast
|
||||
elif avg_response_time < 1.0: # 1 second
|
||||
self.current_rate = min(self.max_rate, self.current_rate * 1.1)
|
||||
logger.debug(f"Increased rate to {self.current_rate:.2f} req/s")
|
||||
|
||||
|
||||
class RequestTracker:
|
||||
"""Track API request statistics"""
|
||||
|
||||
def __init__(self):
|
||||
self.total_requests = 0
|
||||
self.successful_requests = 0
|
||||
self.failed_requests = 0
|
||||
self.total_response_time = 0.0
|
||||
self.last_request_time = None
|
||||
self.error_count_by_type = {}
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def record_request(
|
||||
self, success: bool, response_time: float, error_type: Optional[str] = None
|
||||
):
|
||||
"""Record a request"""
|
||||
with self._lock:
|
||||
self.total_requests += 1
|
||||
self.total_response_time += response_time
|
||||
self.last_request_time = datetime.now()
|
||||
|
||||
if success:
|
||||
self.successful_requests += 1
|
||||
else:
|
||||
self.failed_requests += 1
|
||||
if error_type:
|
||||
self.error_count_by_type[error_type] = (
|
||||
self.error_count_by_type.get(error_type, 0) + 1
|
||||
)
|
||||
|
||||
def get_stats(self) -> Dict[str, any]:
|
||||
"""Get request statistics"""
|
||||
with self._lock:
|
||||
if self.total_requests == 0:
|
||||
return {
|
||||
"total_requests": 0,
|
||||
"success_rate": 0.0,
|
||||
"average_response_time": 0.0,
|
||||
"last_request_time": None,
|
||||
"error_breakdown": {},
|
||||
}
|
||||
|
||||
return {
|
||||
"total_requests": self.total_requests,
|
||||
"successful_requests": self.successful_requests,
|
||||
"failed_requests": self.failed_requests,
|
||||
"success_rate": self.successful_requests / self.total_requests,
|
||||
"average_response_time": self.total_response_time / self.total_requests,
|
||||
"last_request_time": (
|
||||
self.last_request_time.isoformat()
|
||||
if self.last_request_time
|
||||
else None
|
||||
),
|
||||
"error_breakdown": dict(self.error_count_by_type),
|
||||
}
|
||||
+17
-107
@@ -1,4 +1,10 @@
|
||||
"""Collector for RID large-dam daily status (app.rid.go.th/reservoir).
|
||||
"""RID large-dam daily status (app.rid.go.th/reservoir): client, PostgreSQL
|
||||
store and archive backfill.
|
||||
|
||||
The daily collection (today + yesterday's late revisions) runs in
|
||||
ping-collector (rust/collector/src/dams.rs, same parsing and tables); this
|
||||
module is used by scripts/backfill_rid_reservoir.py, the dam features
|
||||
(src/ml/dam.py) and the tests.
|
||||
|
||||
The Royal Irrigation Department reservoir app exposes an unauthenticated
|
||||
JSON API: ``POST https://app.rid.go.th/reservoir/api/dams`` with form field
|
||||
@@ -215,19 +221,14 @@ class RidReservoirClient:
|
||||
|
||||
|
||||
class RidReservoirStore:
|
||||
"""SQL persistence for dam metadata + daily measurements.
|
||||
"""PostgreSQL persistence for dam metadata + daily measurements.
|
||||
|
||||
Shares the app's relational database; writes rid_dams (metadata) and
|
||||
rid_reservoir_daily keyed (dam_id, date) — the composite natural PK keeps
|
||||
the table TimescaleDB-hypertable compatible.
|
||||
Writes rid_dams (metadata) and rid_reservoir_daily keyed (dam_id, date) —
|
||||
the composite natural PK keeps the table TimescaleDB-hypertable
|
||||
compatible. The DDL matches rust/collector/src/db.rs.
|
||||
"""
|
||||
|
||||
def __init__(self, connection_string: str, db_type: str):
|
||||
self.db_type = db_type.lower()
|
||||
if self.db_type not in ("sqlite", "postgresql", "mysql"):
|
||||
raise ValueError(
|
||||
f"Reservoir collection requires a SQL database, got '{db_type}'"
|
||||
)
|
||||
def __init__(self, connection_string: str):
|
||||
self.connection_string = connection_string
|
||||
self.engine = None
|
||||
|
||||
@@ -272,35 +273,12 @@ class RidReservoirStore:
|
||||
PRIMARY KEY (dam_id, date)
|
||||
)
|
||||
""",
|
||||
"CREATE INDEX IF NOT EXISTS idx_rid_reservoir_date "
|
||||
"ON rid_reservoir_daily(date)",
|
||||
]
|
||||
if self.db_type != "mysql":
|
||||
ddl.append(
|
||||
"CREATE INDEX IF NOT EXISTS idx_rid_reservoir_date "
|
||||
"ON rid_reservoir_daily(date)"
|
||||
)
|
||||
with self.engine.begin() as conn:
|
||||
for statement in ddl:
|
||||
conn.execute(text(statement))
|
||||
# Widen storage_pct on tables created before 2026-08-13: the source
|
||||
# publishes junk percents (dam 100602 reports 87798%) that overflowed
|
||||
# NUMERIC(6,2) and discarded whole daily batches.
|
||||
if self.db_type == "postgresql":
|
||||
migrations = (
|
||||
"ALTER TABLE rid_reservoir_daily "
|
||||
"ALTER COLUMN storage_pct TYPE NUMERIC(8,2)",
|
||||
)
|
||||
elif self.db_type == "mysql":
|
||||
migrations = (
|
||||
"ALTER TABLE rid_reservoir_daily MODIFY storage_pct NUMERIC(8,2)",
|
||||
)
|
||||
else: # sqlite: NUMERIC is affinity only, nothing to widen
|
||||
migrations = ()
|
||||
for statement in migrations:
|
||||
try:
|
||||
with self.engine.begin() as conn:
|
||||
conn.execute(text(statement))
|
||||
except Exception as e:
|
||||
logger.warning(f"rid_reservoir_daily migration skipped: {e}")
|
||||
|
||||
def _upsert(
|
||||
self,
|
||||
@@ -319,45 +297,17 @@ class RidReservoirStore:
|
||||
col_list = ", ".join(cols)
|
||||
params = ", ".join(f":{c}" for c in cols)
|
||||
conflict = ", ".join(key_cols)
|
||||
if self.db_type == "sqlite":
|
||||
if not preserve_cols:
|
||||
return f"INSERT OR REPLACE INTO {table} ({col_list}) VALUES ({params})"
|
||||
updates = ", ".join(
|
||||
(
|
||||
f"{c} = COALESCE(excluded.{c}, {table}.{c})"
|
||||
if c in preserve_cols
|
||||
else f"{c} = excluded.{c}"
|
||||
)
|
||||
for c in value_cols
|
||||
)
|
||||
return (
|
||||
f"INSERT INTO {table} ({col_list}) VALUES ({params}) "
|
||||
f"ON CONFLICT ({conflict}) DO UPDATE SET {updates}"
|
||||
)
|
||||
if self.db_type == "postgresql":
|
||||
updates = ", ".join(
|
||||
(
|
||||
f"{c} = COALESCE(EXCLUDED.{c}, {table}.{c})"
|
||||
if c in preserve_cols
|
||||
else f"{c} = EXCLUDED.{c}"
|
||||
)
|
||||
for c in value_cols
|
||||
)
|
||||
return (
|
||||
f"INSERT INTO {table} ({col_list}) VALUES ({params}) "
|
||||
f"ON CONFLICT ({conflict}) DO UPDATE SET {updates}"
|
||||
)
|
||||
updates = ", ".join(
|
||||
(
|
||||
f"{c} = COALESCE(VALUES({c}), {c})"
|
||||
f"{c} = COALESCE(EXCLUDED.{c}, {table}.{c})"
|
||||
if c in preserve_cols
|
||||
else f"{c} = VALUES({c})"
|
||||
else f"{c} = EXCLUDED.{c}"
|
||||
)
|
||||
for c in value_cols
|
||||
)
|
||||
return (
|
||||
f"INSERT INTO {table} ({col_list}) VALUES ({params}) "
|
||||
f"ON DUPLICATE KEY UPDATE {updates}"
|
||||
f"ON CONFLICT ({conflict}) DO UPDATE SET {updates}"
|
||||
)
|
||||
|
||||
def save(self, records: List[Dict]) -> int:
|
||||
@@ -447,37 +397,12 @@ class RidReservoirStore:
|
||||
values = conn.execute(text(sql), params).fetchall()
|
||||
dates = set()
|
||||
for (value,) in values:
|
||||
if isinstance(value, str): # sqlite returns ISO strings
|
||||
value = datetime.date.fromisoformat(value[:10])
|
||||
if isinstance(value, datetime.datetime):
|
||||
value = value.date()
|
||||
dates.add(value)
|
||||
return dates
|
||||
|
||||
|
||||
class RidReservoirCollector:
|
||||
"""Fetch + persist the daily dam snapshot (today and yesterday)."""
|
||||
|
||||
def __init__(self, db_config: Dict, client: Optional[RidReservoirClient] = None):
|
||||
self.client = client or RidReservoirClient()
|
||||
self.store = RidReservoirStore(
|
||||
connection_string=db_config["connection_string"],
|
||||
db_type=db_config["type"],
|
||||
)
|
||||
|
||||
def run_cycle(self) -> int:
|
||||
"""Collect today's snapshot plus yesterday's (late daily revisions)."""
|
||||
saved = 0
|
||||
today = datetime.date.today()
|
||||
for date in (None, today - datetime.timedelta(days=1)):
|
||||
try:
|
||||
saved += self.store.save(self.client.fetch_day(date))
|
||||
except Exception as e:
|
||||
logger.error(f"RID reservoir collection failed for {date}: {e}")
|
||||
logger.info(f"RID reservoir collection: {saved} dam-day rows saved")
|
||||
return saved
|
||||
|
||||
|
||||
def backfill(
|
||||
store: RidReservoirStore,
|
||||
start: datetime.date,
|
||||
@@ -616,18 +541,3 @@ def backfill_dam(
|
||||
if chunk_start <= end:
|
||||
time.sleep(throttle_seconds)
|
||||
return total
|
||||
|
||||
|
||||
def create_collector_from_config() -> Optional[RidReservoirCollector]:
|
||||
"""Build a collector from app Config; None when disabled or non-SQL DB."""
|
||||
from .config import Config
|
||||
|
||||
if not Config.ENABLE_RESERVOIR_COLLECTION:
|
||||
return None
|
||||
db_config = Config.get_database_config()
|
||||
if db_config["type"] not in ("sqlite", "postgresql", "mysql"):
|
||||
logger.warning(
|
||||
f"Reservoir collection skipped: DB_TYPE '{db_config['type']}' is not SQL"
|
||||
)
|
||||
return None
|
||||
return RidReservoirCollector(db_config)
|
||||
@@ -1,80 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Pydantic request/response schemas for the water monitoring web API."""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class StationResponse(BaseModel):
|
||||
station_id: int
|
||||
station_code: str
|
||||
thai_name: str
|
||||
english_name: str
|
||||
latitude: Optional[float] = None
|
||||
longitude: Optional[float] = None
|
||||
geohash: Optional[str] = None
|
||||
status: str = "active"
|
||||
|
||||
|
||||
class StationCreateModel(BaseModel):
|
||||
station_code: str = Field(..., description="Station code (e.g., P.1, P.20)")
|
||||
thai_name: str = Field(..., description="Thai name of the station")
|
||||
english_name: str = Field(..., description="English name of the station")
|
||||
latitude: Optional[float] = Field(
|
||||
None, ge=-90, le=90, description="Latitude coordinate"
|
||||
)
|
||||
longitude: Optional[float] = Field(
|
||||
None, ge=-180, le=180, description="Longitude coordinate"
|
||||
)
|
||||
geohash: Optional[str] = Field(None, description="Geohash for the location")
|
||||
status: str = Field("active", description="Station status")
|
||||
|
||||
|
||||
class StationUpdateModel(BaseModel):
|
||||
thai_name: Optional[str] = Field(None, description="Thai name of the station")
|
||||
english_name: Optional[str] = Field(None, description="English name of the station")
|
||||
latitude: Optional[float] = Field(
|
||||
None, ge=-90, le=90, description="Latitude coordinate"
|
||||
)
|
||||
longitude: Optional[float] = Field(
|
||||
None, ge=-180, le=180, description="Longitude coordinate"
|
||||
)
|
||||
geohash: Optional[str] = Field(None, description="Geohash for the location")
|
||||
status: Optional[str] = Field(None, description="Station status")
|
||||
|
||||
|
||||
class MeasurementResponse(BaseModel):
|
||||
timestamp: datetime
|
||||
station_code: str
|
||||
station_name_en: str
|
||||
station_name_th: str
|
||||
water_level: float
|
||||
discharge: Optional[float] = None
|
||||
discharge_percent: Optional[float] = None
|
||||
status: str = "active"
|
||||
|
||||
|
||||
class HealthResponse(BaseModel):
|
||||
overall_status: str
|
||||
timestamp: str
|
||||
checks: Dict[str, Dict[str, Any]]
|
||||
|
||||
|
||||
class MetricsResponse(BaseModel):
|
||||
counters: Dict[str, float]
|
||||
gauges: Dict[str, float]
|
||||
histograms: Dict[str, Dict[str, float]]
|
||||
|
||||
|
||||
class ScrapingStatusResponse(BaseModel):
|
||||
is_running: bool
|
||||
last_run: Optional[datetime] = None
|
||||
next_run: Optional[datetime] = None
|
||||
total_runs: int = 0
|
||||
successful_runs: int = 0 # cycles that found the current hour published
|
||||
failed_runs: int = 0 # cycles behind (RID late) or failed
|
||||
feed_current: Optional[bool] = None # last cycle had the current hour
|
||||
latest_reading: Optional[datetime] = None # newest RID reading stored
|
||||
readings_saved: int = 0 # readings upserted by the last cycle
|
||||
@@ -1,61 +0,0 @@
|
||||
"""Client for ThaiWater's public water-level sensor feed."""
|
||||
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
class ThaiWaterClient:
|
||||
API_URL = "https://twa-api-public.thaiwater.net/v2/waterlevel"
|
||||
|
||||
def __init__(self, session=None, api_key: Optional[str] = None, timeout: int = 30):
|
||||
self.session = session or requests.Session()
|
||||
self.api_key = api_key
|
||||
self.timeout = timeout
|
||||
|
||||
def fetch_ping_sensors(self) -> List[Dict]:
|
||||
if not self.api_key:
|
||||
raise RuntimeError("THAIWATER_API_KEY is not configured")
|
||||
|
||||
response = self.session.get(
|
||||
self.API_URL,
|
||||
headers={"Accept-Language": "en", "x-api-key": self.api_key},
|
||||
timeout=self.timeout,
|
||||
)
|
||||
response.raise_for_status()
|
||||
return self._parse_ping_features(response.json())
|
||||
|
||||
@staticmethod
|
||||
def _parse_ping_features(payload: Dict) -> List[Dict]:
|
||||
sensors = []
|
||||
for collection in payload.get("data", {}).values():
|
||||
for feature in collection.get("features", []):
|
||||
properties = feature.get("properties") or {}
|
||||
basin = properties.get("basin") or {}
|
||||
if basin.get("basin") != "Ping":
|
||||
continue
|
||||
|
||||
geometry = feature.get("geometry") or {}
|
||||
coordinates = geometry.get("coordinates") or []
|
||||
if len(coordinates) < 2:
|
||||
continue
|
||||
|
||||
station = properties.get("station") or {}
|
||||
station_code = station.get("stationCode", "")
|
||||
sensors.append(
|
||||
{
|
||||
"id": f"thaiwater:{properties.get('id')}",
|
||||
"station_code": station_code.split("-", 1)[-1],
|
||||
"station_name": station.get("station"),
|
||||
"latitude": coordinates[1],
|
||||
"longitude": coordinates[0],
|
||||
"timestamp": properties.get("waterlevelDatetime"),
|
||||
"water_level_msl": properties.get("waterlevelMsl"),
|
||||
"bank_percent": properties.get("storagePercent"),
|
||||
"distance_to_bank": properties.get("diffWlBank"),
|
||||
"river_name": properties.get("riverName"),
|
||||
"agency": (properties.get("agency") or {}).get("agencyShort"),
|
||||
"source": "ThaiWater",
|
||||
}
|
||||
)
|
||||
return sensors
|
||||
@@ -1,144 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Data validation utilities for water monitoring system
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .exceptions import DataValidationError
|
||||
from .models import StationInfo, WaterMeasurement
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DataValidator:
|
||||
"""Validates water measurement data"""
|
||||
|
||||
# Reasonable ranges for water measurements
|
||||
WATER_LEVEL_MIN = -10.0 # meters
|
||||
WATER_LEVEL_MAX = 50.0 # meters
|
||||
DISCHARGE_MIN = 0.0 # cms
|
||||
DISCHARGE_MAX = 10000.0 # cms
|
||||
DISCHARGE_PERCENT_MIN = 0.0
|
||||
# % of channel capacity. Major floods genuinely exceed 200% (Oct 2024 peaked
|
||||
# at 226%); values beyond 500% are treated as data errors.
|
||||
DISCHARGE_PERCENT_MAX = 500.0
|
||||
|
||||
@classmethod
|
||||
def validate_measurement(cls, measurement: Dict[str, Any]) -> bool:
|
||||
"""Validate a single measurement"""
|
||||
try:
|
||||
# Check required fields (discharge is now optional)
|
||||
required_fields = ["timestamp", "station_id", "water_level"]
|
||||
for field in required_fields:
|
||||
if field not in measurement:
|
||||
logger.warning(f"Missing required field: {field}")
|
||||
return False
|
||||
|
||||
# Validate timestamp
|
||||
if not isinstance(measurement["timestamp"], datetime):
|
||||
logger.warning(
|
||||
f"Invalid timestamp type: {type(measurement['timestamp'])}"
|
||||
)
|
||||
return False
|
||||
|
||||
# Validate water level (required)
|
||||
if measurement["water_level"] is None:
|
||||
logger.warning("Water level cannot be None")
|
||||
return False
|
||||
water_level = float(measurement["water_level"])
|
||||
if not (cls.WATER_LEVEL_MIN <= water_level <= cls.WATER_LEVEL_MAX):
|
||||
logger.warning(f"Water level out of range: {water_level}")
|
||||
return False
|
||||
|
||||
# Validate discharge (optional - can be None)
|
||||
discharge_value = measurement.get("discharge")
|
||||
if discharge_value is not None:
|
||||
discharge = float(discharge_value)
|
||||
if not (cls.DISCHARGE_MIN <= discharge <= cls.DISCHARGE_MAX):
|
||||
logger.warning(f"Discharge out of range: {discharge}")
|
||||
return False
|
||||
|
||||
# Validate discharge percent if present. This is an auxiliary field:
|
||||
# an implausible value must not cost us the water level and discharge
|
||||
# (rejecting rows here silently deleted the Oct 2024 flood peaks), so
|
||||
# out-of-range percents are nulled and the measurement is kept.
|
||||
if measurement.get("discharge_percent") is not None:
|
||||
discharge_percent = float(measurement["discharge_percent"])
|
||||
if not (
|
||||
cls.DISCHARGE_PERCENT_MIN
|
||||
<= discharge_percent
|
||||
<= cls.DISCHARGE_PERCENT_MAX
|
||||
):
|
||||
logger.warning(
|
||||
f"Discharge percent out of range ({discharge_percent}); "
|
||||
"keeping measurement with discharge_percent=None"
|
||||
)
|
||||
measurement["discharge_percent"] = None
|
||||
|
||||
# Validate station ID
|
||||
station_id = measurement["station_id"]
|
||||
if not isinstance(station_id, int) or station_id < 1 or station_id > 16:
|
||||
logger.warning(f"Invalid station ID: {station_id}")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
except (ValueError, TypeError) as e:
|
||||
logger.warning(f"Data validation error: {e}")
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def validate_measurements(
|
||||
cls, measurements: List[Dict[str, Any]]
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Validate and filter a list of measurements"""
|
||||
valid_measurements = []
|
||||
invalid_count = 0
|
||||
|
||||
for measurement in measurements:
|
||||
if cls.validate_measurement(measurement):
|
||||
valid_measurements.append(measurement)
|
||||
else:
|
||||
invalid_count += 1
|
||||
|
||||
if invalid_count > 0:
|
||||
logger.warning(f"Filtered out {invalid_count} invalid measurements")
|
||||
|
||||
return valid_measurements
|
||||
|
||||
@classmethod
|
||||
def validate_station_info(cls, station_info: Dict[str, Any]) -> bool:
|
||||
"""Validate station information"""
|
||||
try:
|
||||
required_fields = [
|
||||
"station_id",
|
||||
"station_code",
|
||||
"thai_name",
|
||||
"english_name",
|
||||
]
|
||||
for field in required_fields:
|
||||
if field not in station_info or not station_info[field]:
|
||||
logger.warning(f"Missing or empty station field: {field}")
|
||||
return False
|
||||
|
||||
# Validate coordinates if present
|
||||
if station_info.get("latitude") is not None:
|
||||
lat = float(station_info["latitude"])
|
||||
if not (-90 <= lat <= 90):
|
||||
logger.warning(f"Invalid latitude: {lat}")
|
||||
return False
|
||||
|
||||
if station_info.get("longitude") is not None:
|
||||
lon = float(station_info["longitude"])
|
||||
if not (-180 <= lon <= 180):
|
||||
logger.warning(f"Invalid longitude: {lon}")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
except (ValueError, TypeError) as e:
|
||||
logger.warning(f"Station validation error: {e}")
|
||||
return False
|
||||
@@ -1,883 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Enhanced Water Monitor Scraper with multiple database backend support
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
import requests
|
||||
import schedule
|
||||
|
||||
try:
|
||||
from .config import Config
|
||||
from .database_adapters import create_database_adapter
|
||||
from .logging_config import get_logger
|
||||
from .metrics import Timer, increment_counter, record_histogram, set_gauge
|
||||
from .rate_limiter import RateLimiter, RequestTracker
|
||||
from .validators import DataValidator
|
||||
except ImportError:
|
||||
# Handle case when running as standalone script
|
||||
from config import Config
|
||||
from database_adapters import create_database_adapter
|
||||
|
||||
def get_logger(name):
|
||||
return logging.getLogger(name)
|
||||
|
||||
def increment_counter(*args, **kwargs):
|
||||
pass
|
||||
|
||||
def set_gauge(*args, **kwargs):
|
||||
pass
|
||||
|
||||
def record_histogram(*args, **kwargs):
|
||||
pass
|
||||
|
||||
class Timer:
|
||||
def __init__(self, *args, **kwargs):
|
||||
pass
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
pass
|
||||
|
||||
class RateLimiter:
|
||||
def __init__(self, *args, **kwargs):
|
||||
pass
|
||||
|
||||
def wait_if_needed(self):
|
||||
pass
|
||||
|
||||
class RequestTracker:
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def record_request(self, *args, **kwargs):
|
||||
pass
|
||||
|
||||
class DataValidator:
|
||||
@staticmethod
|
||||
def validate_measurements(measurements):
|
||||
return measurements
|
||||
|
||||
|
||||
# Get logger instance
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class EnhancedWaterMonitorScraper:
|
||||
def __init__(self, db_config: Dict):
|
||||
"""
|
||||
Initialize scraper with database configuration
|
||||
|
||||
Args:
|
||||
db_config: Database configuration dictionary
|
||||
"""
|
||||
self.api_url = "https://hyd-app-db.rid.go.th/webservice/getGroupHourlyWaterLevelReportAllHL.ashx"
|
||||
self.db_config = db_config.copy() # Make a copy to avoid modifying original
|
||||
self.db_adapter = None
|
||||
|
||||
# Scheduler state tracking
|
||||
self.last_successful_update = None
|
||||
self.retry_mode = False
|
||||
self.next_hourly_check = None
|
||||
self.last_cycle: Optional[Dict] = None # see run_scraping_cycle()
|
||||
|
||||
# Rate limiting and request tracking
|
||||
self.rate_limiter = RateLimiter(max_requests=10, time_window_seconds=60)
|
||||
self.request_tracker = RequestTracker()
|
||||
|
||||
# HTTP session for API requests
|
||||
self.session = requests.Session()
|
||||
self.session.headers.update(
|
||||
{
|
||||
"User-Agent": Config.USER_AGENT,
|
||||
"Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
|
||||
"Accept": "application/json, text/javascript, */*; q=0.01",
|
||||
"X-Requested-With": "XMLHttpRequest",
|
||||
}
|
||||
)
|
||||
|
||||
# Station mapping is persisted to a JSON file so that station CRUD via the
|
||||
# API survives restarts; on first run it is seeded from the bundled
|
||||
# defaults in data/stations.json.
|
||||
self.station_config_path = Config.STATION_CONFIG_PATH
|
||||
self.station_mapping = self._load_station_mapping()
|
||||
|
||||
self.init_database()
|
||||
|
||||
@staticmethod
|
||||
def _default_station_mapping_path() -> str:
|
||||
"""Path to the bundled default station mapping shipped with the package."""
|
||||
return os.path.join(
|
||||
os.path.dirname(os.path.abspath(__file__)), "data", "stations.json"
|
||||
)
|
||||
|
||||
def _load_station_mapping(self) -> Dict:
|
||||
"""Load the station mapping, preferring the runtime-writable config file.
|
||||
|
||||
Order of precedence:
|
||||
1. The runtime config file (STATION_CONFIG_PATH) if it exists — this holds
|
||||
any changes made through the station CRUD API.
|
||||
2. The bundled defaults in data/stations.json.
|
||||
"""
|
||||
for source in (self.station_config_path, self._default_station_mapping_path()):
|
||||
if source and os.path.exists(source):
|
||||
try:
|
||||
with open(source, encoding="utf-8") as f:
|
||||
mapping = json.load(f)
|
||||
logger.info(f"Loaded {len(mapping)} stations from {source}")
|
||||
return mapping
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load station mapping from {source}: {e}")
|
||||
|
||||
logger.error(
|
||||
"No station mapping could be loaded; starting with an empty mapping"
|
||||
)
|
||||
return {}
|
||||
|
||||
def save_stations(self) -> bool:
|
||||
"""Persist the current station mapping to the runtime config file.
|
||||
|
||||
Written atomically (temp file + replace) so a crash mid-write cannot
|
||||
corrupt the existing configuration.
|
||||
"""
|
||||
path = self.station_config_path
|
||||
if not path:
|
||||
logger.warning(
|
||||
"STATION_CONFIG_PATH not set; station changes will not persist"
|
||||
)
|
||||
return False
|
||||
try:
|
||||
tmp_path = f"{path}.tmp"
|
||||
with open(tmp_path, "w", encoding="utf-8") as f:
|
||||
json.dump(self.station_mapping, f, ensure_ascii=False, indent=2)
|
||||
f.write("\n")
|
||||
os.replace(tmp_path, path)
|
||||
logger.info(f"Persisted {len(self.station_mapping)} stations to {path}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to persist station mapping to {path}: {e}")
|
||||
return False
|
||||
|
||||
def init_database(self):
|
||||
"""Initialize database connection"""
|
||||
try:
|
||||
# Extract db_type and pass remaining config as kwargs
|
||||
db_config_copy = self.db_config.copy()
|
||||
db_type = db_config_copy.pop("type")
|
||||
self.db_adapter = create_database_adapter(db_type, **db_config_copy)
|
||||
success = self.db_adapter.connect()
|
||||
|
||||
if success:
|
||||
logger.info(f"Successfully connected to {db_type.upper()} database")
|
||||
set_gauge("database_connected", 1)
|
||||
increment_counter("database_connections_successful")
|
||||
else:
|
||||
logger.error(f"Failed to connect to {db_type.upper()} database")
|
||||
set_gauge("database_connected", 0)
|
||||
increment_counter("database_connections_failed")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error initializing database: {e}")
|
||||
set_gauge("database_connected", 0)
|
||||
increment_counter("database_connections_failed")
|
||||
self.db_adapter = None
|
||||
|
||||
def fetch_water_data_for_date(
|
||||
self, target_date: datetime.datetime
|
||||
) -> Optional[List[Dict]]:
|
||||
"""Fetch water levels and discharge data from API for a specific date"""
|
||||
with Timer("api_request_duration"):
|
||||
try:
|
||||
logger.info(
|
||||
f"Starting data fetch from API for date: {target_date.strftime('%Y-%m-%d')}"
|
||||
)
|
||||
|
||||
# Rate limiting
|
||||
self.rate_limiter.wait_if_needed()
|
||||
|
||||
# Create Thai format date (Buddhist calendar)
|
||||
thai_year = target_date.year + 543
|
||||
thai_date = f"{target_date.day:02d}/{target_date.month:02d}/{thai_year}"
|
||||
|
||||
# API parameters
|
||||
payload = {
|
||||
"DW[UtokID]": "1",
|
||||
"DW[BasinID]": "6",
|
||||
"DW[TimeCurrent]": thai_date,
|
||||
"_search": "false",
|
||||
"nd": str(int(time.time() * 1000)),
|
||||
"rows": "100",
|
||||
"page": "1",
|
||||
"sidx": "indexhourly",
|
||||
"sord": "asc",
|
||||
}
|
||||
|
||||
logger.debug(f"API parameters: {payload}")
|
||||
|
||||
# POST request to API
|
||||
start_time = time.time()
|
||||
response = self.session.post(self.api_url, data=payload, timeout=30)
|
||||
response_time = time.time() - start_time
|
||||
|
||||
response.raise_for_status()
|
||||
|
||||
# Record successful request
|
||||
self.request_tracker.record_request(True, response_time)
|
||||
increment_counter("api_requests_successful")
|
||||
record_histogram("api_response_time", response_time)
|
||||
|
||||
# Parse JSON response
|
||||
try:
|
||||
json_data = response.json()
|
||||
logger.debug(
|
||||
f"API response received: {len(str(json_data))} characters"
|
||||
)
|
||||
except ValueError as e:
|
||||
logger.error(f"Error parsing JSON response: {e}")
|
||||
self.request_tracker.record_request(
|
||||
False, response_time, "json_parse_error"
|
||||
)
|
||||
increment_counter("api_requests_failed")
|
||||
return None
|
||||
|
||||
water_data = []
|
||||
|
||||
# Parse JSON data
|
||||
if json_data and isinstance(json_data, dict) and "rows" in json_data:
|
||||
for row in json_data["rows"]:
|
||||
try:
|
||||
# Parse timestamp
|
||||
time_str = row.get("hourlytime", "")
|
||||
if not time_str:
|
||||
continue
|
||||
|
||||
try:
|
||||
# Format: "1.00", "2.00", ..., "24.00"
|
||||
api_hour = int(float(time_str))
|
||||
if api_hour < 1 or api_hour > 24:
|
||||
continue
|
||||
|
||||
if api_hour == 24:
|
||||
# Hour 24 = midnight (00:00) of the next day
|
||||
data_time = target_date.replace(
|
||||
hour=0, minute=0, second=0, microsecond=0
|
||||
)
|
||||
data_time = data_time + datetime.timedelta(days=1)
|
||||
else:
|
||||
# Hours 1-23 = 01:00-23:00 of the same day
|
||||
data_time = target_date.replace(
|
||||
hour=api_hour, minute=0, second=0, microsecond=0
|
||||
)
|
||||
|
||||
except (ValueError, IndexError):
|
||||
logger.warning(f"Could not parse timestamp: {time_str}")
|
||||
continue
|
||||
|
||||
# Parse all water levels and discharge values
|
||||
station_count = 0
|
||||
for station_num in range(1, 17): # Stations 1-16
|
||||
wl_key = f"wlvalues{station_num}"
|
||||
q_key = f"qvalues{station_num}"
|
||||
qp_key = f"QPercent{station_num}"
|
||||
|
||||
# Check if water level data exists (required)
|
||||
if wl_key in row:
|
||||
try:
|
||||
water_level = row[wl_key]
|
||||
|
||||
# Skip if water level is None or invalid
|
||||
if water_level is None:
|
||||
continue
|
||||
|
||||
# Convert water level to float (required)
|
||||
water_level = float(water_level)
|
||||
|
||||
# Try to parse discharge data (optional)
|
||||
discharge = None
|
||||
discharge_percent = None
|
||||
|
||||
if q_key in row:
|
||||
try:
|
||||
discharge_raw = row[q_key]
|
||||
if (
|
||||
discharge_raw is not None
|
||||
and discharge_raw != "***"
|
||||
):
|
||||
discharge = float(discharge_raw)
|
||||
|
||||
# Only parse discharge percent if discharge is valid
|
||||
discharge_percent_raw = row.get(
|
||||
qp_key
|
||||
)
|
||||
if (
|
||||
discharge_percent_raw
|
||||
is not None
|
||||
):
|
||||
try:
|
||||
discharge_percent = float(
|
||||
discharge_percent_raw
|
||||
)
|
||||
except (ValueError, TypeError):
|
||||
discharge_percent = None
|
||||
else:
|
||||
logger.debug(
|
||||
"Skipping malformed discharge data for "
|
||||
f"station {station_num}: {discharge_raw}"
|
||||
)
|
||||
except (ValueError, TypeError) as e:
|
||||
logger.debug(
|
||||
f"Could not parse discharge for station {station_num}: {e}"
|
||||
)
|
||||
|
||||
station_info = self.station_mapping.get(
|
||||
str(station_num),
|
||||
{
|
||||
"code": f"P.{19+station_num}",
|
||||
"thai_name": f"Station {station_num}",
|
||||
"english_name": f"Station {station_num}",
|
||||
},
|
||||
)
|
||||
|
||||
water_data.append(
|
||||
{
|
||||
"timestamp": data_time,
|
||||
"station_id": station_num,
|
||||
"station_code": station_info["code"],
|
||||
"station_name_en": station_info[
|
||||
"english_name"
|
||||
],
|
||||
"station_name_th": station_info[
|
||||
"thai_name"
|
||||
],
|
||||
"latitude": station_info.get(
|
||||
"latitude"
|
||||
),
|
||||
"longitude": station_info.get(
|
||||
"longitude"
|
||||
),
|
||||
"geohash": station_info.get("geohash"),
|
||||
"water_level": water_level,
|
||||
"water_level_unit": "m",
|
||||
"discharge": discharge,
|
||||
"discharge_unit": "cms",
|
||||
"discharge_percent": discharge_percent,
|
||||
"status": "active",
|
||||
}
|
||||
)
|
||||
|
||||
station_count += 1
|
||||
|
||||
except (ValueError, TypeError) as e:
|
||||
logger.warning(
|
||||
f"Could not parse water level for station {station_num}: {e}"
|
||||
)
|
||||
continue
|
||||
|
||||
logger.debug(
|
||||
f"Processed {station_count} stations for time {time_str}"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Error processing data row: {e}")
|
||||
continue
|
||||
|
||||
# Validate data
|
||||
water_data = DataValidator.validate_measurements(water_data)
|
||||
|
||||
logger.info(
|
||||
f"Successfully fetched {len(water_data)} data points from API "
|
||||
f"for {target_date.strftime('%Y-%m-%d')}"
|
||||
)
|
||||
return water_data
|
||||
|
||||
except requests.RequestException as e:
|
||||
logger.error(f"Network error fetching API data: {e}")
|
||||
self.request_tracker.record_request(False, 0, "network_error")
|
||||
increment_counter("api_requests_failed")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error fetching API data: {e}")
|
||||
self.request_tracker.record_request(False, 0, "unexpected_error")
|
||||
increment_counter("api_requests_failed")
|
||||
return None
|
||||
|
||||
def fetch_water_data(self) -> Optional[List[Dict]]:
|
||||
"""Fetch water levels and discharge data from API with smart date selection"""
|
||||
current_time = datetime.datetime.now()
|
||||
|
||||
# If it's past 01:00, try today's data first, then yesterday as fallback
|
||||
if current_time.hour >= 1:
|
||||
logger.info(
|
||||
"After 01:00 - trying today's data first, will fallback to yesterday if needed"
|
||||
)
|
||||
|
||||
# Try today's data first
|
||||
today_data = self.fetch_water_data_for_date(current_time)
|
||||
if today_data and len(today_data) > 0:
|
||||
logger.info(
|
||||
f"Successfully fetched {len(today_data)} data points for today"
|
||||
)
|
||||
return today_data
|
||||
|
||||
# Fallback to yesterday's data
|
||||
logger.info("No data available for today, trying yesterday's data")
|
||||
yesterday = current_time - datetime.timedelta(days=1)
|
||||
yesterday_data = self.fetch_water_data_for_date(yesterday)
|
||||
if yesterday_data and len(yesterday_data) > 0:
|
||||
logger.info(
|
||||
f"Successfully fetched {len(yesterday_data)} data points for yesterday"
|
||||
)
|
||||
return yesterday_data
|
||||
|
||||
logger.warning("No data available for today or yesterday")
|
||||
return None
|
||||
else:
|
||||
# Before 01:00 - only try yesterday's data (API likely hasn't updated yet)
|
||||
logger.info("Before 01:00 - fetching yesterday's data only")
|
||||
yesterday = current_time - datetime.timedelta(days=1)
|
||||
return self.fetch_water_data_for_date(yesterday)
|
||||
|
||||
def save_to_database(self, water_data: List[Dict], max_retries: int = 3) -> bool:
|
||||
"""Save water measurements to database with retry logic"""
|
||||
if not self.db_adapter:
|
||||
logger.error("Database adapter not initialized")
|
||||
return False
|
||||
|
||||
if not water_data:
|
||||
logger.warning("No data to save")
|
||||
return False
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
success = self.db_adapter.save_measurements(water_data)
|
||||
if success:
|
||||
logger.info(
|
||||
f"Successfully saved {len(water_data)} measurements to database"
|
||||
)
|
||||
increment_counter("database_saves_successful")
|
||||
set_gauge("last_save_timestamp", time.time())
|
||||
return True
|
||||
else:
|
||||
logger.warning(f"Save attempt {attempt + 1} failed, retrying...")
|
||||
|
||||
except Exception as e:
|
||||
if "database is locked" in str(e).lower() and attempt < max_retries - 1:
|
||||
logger.warning(
|
||||
f"Database locked on attempt {attempt + 1}, retrying in {2 ** attempt} seconds..."
|
||||
)
|
||||
time.sleep(2**attempt) # Exponential backoff
|
||||
continue
|
||||
else:
|
||||
logger.error(
|
||||
f"Error saving to database (attempt {attempt + 1}): {e}"
|
||||
)
|
||||
if attempt == max_retries - 1:
|
||||
increment_counter("database_saves_failed")
|
||||
return False
|
||||
|
||||
return False
|
||||
|
||||
def get_latest_data(self, limit: int = 100) -> List[Dict]:
|
||||
"""Get latest data from database"""
|
||||
if not self.db_adapter:
|
||||
return []
|
||||
|
||||
try:
|
||||
return self.db_adapter.get_latest_measurements(limit=limit)
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting latest data: {e}")
|
||||
return []
|
||||
|
||||
def _cycle_source_dates(self, now: datetime.datetime) -> List[datetime.date]:
|
||||
"""Report dates (the API's ``DW[TimeCurrent]``) to fetch in one cycle.
|
||||
|
||||
Always the report holding the current hour: today's, or before 01:00
|
||||
yesterday's (its "hour 24" is today's 00:00). Plus yesterday's report
|
||||
while the store still has holes in it. RID often publishes the evening
|
||||
hours an hour or two late, and hours 22-24 of a day are only reachable
|
||||
through that day's report, so they must be re-fetched after midnight.
|
||||
"""
|
||||
today = now.date()
|
||||
yesterday = today - datetime.timedelta(days=1)
|
||||
dates = {today if now.hour >= 1 else yesterday}
|
||||
try:
|
||||
dates.update(self._find_gap_fetch_dates(yesterday, today, now))
|
||||
except Exception as e:
|
||||
logger.warning(f"Gap detection failed, fetching the current report: {e}")
|
||||
return sorted(dates)
|
||||
|
||||
def run_scraping_cycle(self, now: Optional[datetime.datetime] = None) -> bool:
|
||||
"""Fetch the RID hourly report(s) and store every reading in them.
|
||||
|
||||
Returns True when the current hour's reading is in (the feed is
|
||||
current) and False when the caller should retry soon. Readings are
|
||||
saved either way: RID's report regularly lags an hour or two in the
|
||||
evening, and an all-or-nothing freshness check used to discard every
|
||||
reading of a late report, losing the evening hours and leaving the
|
||||
feed stale until the next morning.
|
||||
|
||||
Details of the last cycle are kept in ``self.last_cycle``.
|
||||
"""
|
||||
now = now or datetime.datetime.now()
|
||||
current_hour = now.replace(minute=0, second=0, microsecond=0)
|
||||
status = {
|
||||
"time": now,
|
||||
"dates": [],
|
||||
"fetched": 0,
|
||||
"saved": 0,
|
||||
"latest_reading": None,
|
||||
"current": False,
|
||||
"error": None,
|
||||
}
|
||||
self.last_cycle = status
|
||||
logger.info("Starting scraping cycle...")
|
||||
|
||||
try:
|
||||
dates = self._cycle_source_dates(now)
|
||||
status["dates"] = [d.isoformat() for d in dates]
|
||||
|
||||
water_data: List[Dict] = []
|
||||
for source_date in dates:
|
||||
rows = self.fetch_water_data_for_date(
|
||||
datetime.datetime.combine(source_date, datetime.time.min)
|
||||
)
|
||||
if rows:
|
||||
water_data.extend(rows)
|
||||
status["fetched"] = len(water_data)
|
||||
|
||||
if not water_data:
|
||||
status["error"] = "no data fetched"
|
||||
logger.warning("No data fetched")
|
||||
increment_counter("scraping_cycles_failed")
|
||||
return False
|
||||
|
||||
if not self.save_to_database(water_data):
|
||||
status["error"] = "save failed"
|
||||
logger.error("Failed to save data")
|
||||
increment_counter("scraping_cycles_failed")
|
||||
return False
|
||||
status["saved"] = len(water_data)
|
||||
|
||||
latest = max(m["timestamp"] for m in water_data)
|
||||
status["latest_reading"] = latest
|
||||
status["current"] = latest >= current_hour
|
||||
|
||||
if status["current"]:
|
||||
logger.info(
|
||||
f"Scraping cycle complete: {len(water_data)} readings "
|
||||
f"saved, latest {latest:%Y-%m-%d %H:%M}"
|
||||
)
|
||||
increment_counter("scraping_cycles_successful")
|
||||
else:
|
||||
behind_h = (current_hour - latest).total_seconds() / 3600
|
||||
logger.warning(
|
||||
f"Saved {len(water_data)} readings up to "
|
||||
f"{latest:%Y-%m-%d %H:%M}; RID has not published "
|
||||
f"{current_hour:%H:%M} yet ({behind_h:.0f} h behind), "
|
||||
"will retry"
|
||||
)
|
||||
increment_counter("scraping_cycles_behind")
|
||||
return status["current"]
|
||||
|
||||
except Exception as e:
|
||||
status["error"] = str(e)
|
||||
logger.error(f"Scraping cycle failed: {e}")
|
||||
increment_counter("scraping_cycles_failed")
|
||||
return False
|
||||
|
||||
def fill_data_gaps(self, days_back: Optional[int] = None) -> int:
|
||||
"""Fill gaps in stored data by re-fetching incomplete days.
|
||||
|
||||
Args:
|
||||
days_back: How many days back to scan. None scans the whole data
|
||||
range, from the earliest measurement in the database to now.
|
||||
|
||||
Detection is hour-granular where the backend supports it: any calendar
|
||||
day missing one or more hourly slots gets its source date(s) re-fetched.
|
||||
Because the API reports hour 24 as midnight of the next day, a missing
|
||||
00:00 slot on day D is repaired by re-fetching day D-1.
|
||||
"""
|
||||
logger = get_logger(__name__)
|
||||
filled_count = 0
|
||||
|
||||
try:
|
||||
now = datetime.datetime.now()
|
||||
end_date = now.date()
|
||||
|
||||
if days_back is not None:
|
||||
start_date = end_date - datetime.timedelta(days=days_back)
|
||||
else:
|
||||
if not self.db_adapter:
|
||||
logger.error("Database adapter not initialized")
|
||||
return 0
|
||||
date_range = self.db_adapter.get_measurement_date_range()
|
||||
if not date_range:
|
||||
logger.error(
|
||||
"Database is empty or does not support range queries; "
|
||||
"use --import-historical to seed data first"
|
||||
)
|
||||
return 0
|
||||
start_date = date_range[0].date()
|
||||
|
||||
logger.info(f"Checking for gaps from {start_date} to {end_date}")
|
||||
|
||||
fetch_dates = self._find_gap_fetch_dates(start_date, end_date, now)
|
||||
|
||||
if not fetch_dates:
|
||||
logger.info("No gaps found")
|
||||
return 0
|
||||
|
||||
logger.info(f"Found {len(fetch_dates)} day(s) needing a re-fetch")
|
||||
|
||||
for fetch_date in sorted(fetch_dates):
|
||||
fetch_dt = datetime.datetime.combine(fetch_date, datetime.time.min)
|
||||
logger.info(f"Filling gap for date: {fetch_date}")
|
||||
|
||||
data = self.fetch_water_data_for_date(fetch_dt)
|
||||
|
||||
if data:
|
||||
if self.save_to_database(data):
|
||||
filled_count += len(data)
|
||||
logger.info(f"Filled {len(data)} measurements for {fetch_date}")
|
||||
else:
|
||||
logger.warning(f"Failed to save data for {fetch_date}")
|
||||
else:
|
||||
logger.warning(f"No data available for {fetch_date}")
|
||||
|
||||
# Be respectful to the API
|
||||
time.sleep(1)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Gap filling error: {e}")
|
||||
|
||||
return filled_count
|
||||
|
||||
def _find_gap_fetch_dates(
|
||||
self,
|
||||
start_date: datetime.date,
|
||||
end_date: datetime.date,
|
||||
now: datetime.datetime,
|
||||
) -> List[datetime.date]:
|
||||
"""Determine which source dates need re-fetching to fill gaps.
|
||||
|
||||
Uses hour-granular detection when the adapter supports it, otherwise
|
||||
falls back to re-fetching only days with no data at all.
|
||||
"""
|
||||
hours_by_day = None
|
||||
if self.db_adapter:
|
||||
hours_by_day = self.db_adapter.get_recorded_hours_by_day(
|
||||
start_date, end_date
|
||||
)
|
||||
|
||||
fetch_dates = set()
|
||||
|
||||
if hours_by_day is None:
|
||||
logger.info(
|
||||
"Backend does not support hour-granular gap detection; "
|
||||
"checking for fully-missing days only"
|
||||
)
|
||||
current = start_date
|
||||
while current <= end_date:
|
||||
current_dt = datetime.datetime.combine(current, datetime.time.min)
|
||||
if not self._check_data_exists_for_date(current_dt):
|
||||
fetch_dates.add(current)
|
||||
current += datetime.timedelta(days=1)
|
||||
return sorted(fetch_dates)
|
||||
|
||||
current = start_date
|
||||
while current <= end_date:
|
||||
if current == now.date():
|
||||
# Today: only expect hours that have already passed
|
||||
expected_hours = set(range(0, now.hour))
|
||||
else:
|
||||
expected_hours = set(range(24))
|
||||
|
||||
missing = expected_hours - hours_by_day.get(current, set())
|
||||
|
||||
if missing:
|
||||
# Hours 1-23 of day D come from fetching D; hour 0 comes from
|
||||
# the previous day's fetch (the API's "hour 24")
|
||||
if any(h >= 1 for h in missing):
|
||||
fetch_dates.add(current)
|
||||
if 0 in missing and current > start_date:
|
||||
fetch_dates.add(current - datetime.timedelta(days=1))
|
||||
|
||||
current += datetime.timedelta(days=1)
|
||||
|
||||
return sorted(fetch_dates)
|
||||
|
||||
def update_existing_data(self, days_back: int) -> int:
|
||||
"""Update existing data with latest values for the specified number of days back"""
|
||||
logger = get_logger(__name__)
|
||||
updated_count = 0
|
||||
|
||||
try:
|
||||
# Calculate date range
|
||||
end_date = datetime.datetime.now()
|
||||
start_date = end_date - datetime.timedelta(days=days_back)
|
||||
|
||||
logger.info(f"Updating data from {start_date.date()} to {end_date.date()}")
|
||||
|
||||
# Iterate through each date in the range
|
||||
current_date = start_date
|
||||
while current_date <= end_date:
|
||||
logger.info(f"Updating data for date: {current_date.date()}")
|
||||
|
||||
# Fetch fresh data for this date
|
||||
data = self.fetch_water_data_for_date(current_date)
|
||||
|
||||
if data:
|
||||
# Save the data (this will update existing records)
|
||||
if self.save_to_database(data):
|
||||
updated_count += len(data)
|
||||
logger.info(
|
||||
f"Updated {len(data)} measurements for {current_date.date()}"
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
f"Failed to update data for {current_date.date()}"
|
||||
)
|
||||
else:
|
||||
logger.warning(f"No data available for {current_date.date()}")
|
||||
|
||||
current_date += datetime.timedelta(days=1)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Data update error: {e}")
|
||||
|
||||
return updated_count
|
||||
|
||||
def _check_data_exists_for_date(self, target_date: datetime.datetime) -> bool:
|
||||
"""Check if data exists for a specific date"""
|
||||
try:
|
||||
if not self.db_adapter:
|
||||
return False
|
||||
|
||||
# Get data for the specific date
|
||||
measurements = self.db_adapter.get_measurements_for_date(target_date)
|
||||
return len(measurements) > 0
|
||||
|
||||
except Exception as e:
|
||||
logger = get_logger(__name__)
|
||||
logger.debug(f"Error checking data existence: {e}")
|
||||
return False
|
||||
|
||||
def import_historical_data(
|
||||
self,
|
||||
start_date: datetime.datetime,
|
||||
end_date: datetime.datetime,
|
||||
skip_existing: bool = True,
|
||||
) -> int:
|
||||
"""
|
||||
Import historical data for a date range
|
||||
|
||||
Args:
|
||||
start_date: Start date for historical import
|
||||
end_date: End date for historical import
|
||||
skip_existing: Skip dates that already have data (default: True)
|
||||
|
||||
Returns:
|
||||
Number of data points imported
|
||||
"""
|
||||
logger.info(
|
||||
f"Starting historical data import from {start_date.date()} to {end_date.date()}"
|
||||
)
|
||||
|
||||
total_imported = 0
|
||||
current_date = start_date
|
||||
|
||||
while current_date <= end_date:
|
||||
try:
|
||||
# Check if data already exists for this date
|
||||
if skip_existing and self._check_data_exists_for_date(current_date):
|
||||
logger.info(
|
||||
f"Data already exists for {current_date.date()}, skipping..."
|
||||
)
|
||||
current_date += datetime.timedelta(days=1)
|
||||
continue
|
||||
|
||||
logger.info(f"Importing data for {current_date.date()}...")
|
||||
|
||||
# Fetch data for this date
|
||||
data = self.fetch_water_data_for_date(current_date)
|
||||
|
||||
if data:
|
||||
# Save to database
|
||||
if self.save_to_database(data):
|
||||
total_imported += len(data)
|
||||
logger.info(
|
||||
f"Successfully imported {len(data)} data points for {current_date.date()}"
|
||||
)
|
||||
else:
|
||||
logger.warning(f"Failed to save data for {current_date.date()}")
|
||||
else:
|
||||
logger.warning(f"No data available for {current_date.date()}")
|
||||
|
||||
# Add small delay to be respectful to the API
|
||||
time.sleep(1)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error importing data for {current_date.date()}: {e}")
|
||||
|
||||
current_date += datetime.timedelta(days=1)
|
||||
|
||||
logger.info(
|
||||
f"Historical import completed. Total data points imported: {total_imported}"
|
||||
)
|
||||
return total_imported
|
||||
|
||||
|
||||
# Main execution for standalone usage
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s - %(levelname)s - %(message)s",
|
||||
handlers=[logging.FileHandler("water_monitor.log"), logging.StreamHandler()],
|
||||
)
|
||||
|
||||
parser = argparse.ArgumentParser(description="Thailand Water Monitor")
|
||||
parser.add_argument("--test", action="store_true", help="Run single test cycle")
|
||||
args = parser.parse_args()
|
||||
|
||||
# Default SQLite configuration
|
||||
db_config = {"type": "sqlite", "connection_string": "sqlite:///water_levels.db"}
|
||||
|
||||
try:
|
||||
scraper = EnhancedWaterMonitorScraper(db_config)
|
||||
|
||||
if args.test:
|
||||
logger.info("Running test cycle...")
|
||||
result = scraper.run_scraping_cycle()
|
||||
if result:
|
||||
logger.info("✅ Test completed successfully")
|
||||
sys.exit(0)
|
||||
else:
|
||||
logger.error("❌ Test failed")
|
||||
sys.exit(1)
|
||||
else:
|
||||
logger.info("Starting continuous monitoring...")
|
||||
schedule.every(1).hours.do(scraper.run_scraping_cycle)
|
||||
|
||||
# Run initial cycle
|
||||
scraper.run_scraping_cycle()
|
||||
|
||||
while True:
|
||||
schedule.run_pending()
|
||||
time.sleep(60)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
logger.info("Monitoring stopped by user")
|
||||
except Exception as e:
|
||||
logger.error(f"Error: {e}")
|
||||
sys.exit(1)
|
||||
-1869
File diff suppressed because it is too large.
Load diff
Reference in new issue
Block a user