grabowski/Northern-Thailand-Ping-River-Monitor
5
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases 15 Wiki Activity

Fix Matrix message formatting and harden security

Browse Source 
Matrix alerts:
- Send HTML formatted_body (org.matrix.custom.html) so **bold** and URLs
  render instead of showing literal Markdown; add plain-text body fallback.
  Add dependency-free markdown_to_matrix_html/strip_markdown helpers with
  HTML escaping of station/message data.

Security:
- InfluxDB: bind untrusted station_codes as query params and cast limit to
  int (was f-string interpolation / injection risk).
- VictoriaMetrics: escape Prometheus label values and coerce metric values
  to float, preventing exposition-format injection and None crashes.
- web_api: run blocking scrape cycle via run_in_executor so it no longer
  freezes the event loop; make CORS origins configurable and only allow
  credentials with explicit origins ("*" + credentials is invalid/unsafe).
- config: remove hardcoded root/postgres password fallbacks (raise instead)
  and stop defaulting VM_HOST to a real infrastructure hostname.

Also remove unused imports and wrap long lines to satisfy flake8.
This commit is contained in:
grabowski
2026-07-22 12:07:05 +07:00
parent d3ec5a77e6
commit f4c63cabef
4 changed files with 688 additions and 525 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/alerting.py
+56 -6
View File
@@ -4,7 +4,9 @@ Water Level Alerting System with Matrix Integration
""" """
import datetime import datetime
import html
import os import os
import re
from dataclasses import dataclass from dataclasses import dataclass
from enum import Enum from enum import Enum
from typing import Dict, List, Optional from typing import Dict, List, Optional
@@ -28,6 +30,31 @@ except ImportError:
logger = get_logger(__name__) logger = get_logger(__name__)
_BOLD_RE = re.compile(r"\*\*(.+?)\*\*")
_URL_RE = re.compile(r"(https?://[^\s<]+)")
def markdown_to_matrix_html(text: str) -> str:
"""Convert the small Markdown subset we emit into Matrix-compatible HTML.
Matrix clients do NOT render Markdown in the plain ``body`` field; formatting
only shows when an HTML ``formatted_body`` is sent alongside it. We only use
``**bold**``, bare URLs and newlines, so a minimal converter is sufficient and
avoids adding a Markdown dependency.
"""
# Escape HTML special chars first so station/message data can't inject markup.
result = html.escape(text, quote=False)
result = _BOLD_RE.sub(r"<strong>\1</strong>", result)
result = _URL_RE.sub(r'<a href="\1">\1</a>', result)
result = result.replace("\n", "<br/>")
return result
def strip_markdown(text: str) -> str:
"""Produce a clean plain-text fallback for the Matrix ``body`` field."""
return _BOLD_RE.sub(r"\1", text)
class AlertLevel(Enum): class AlertLevel(Enum):
INFO = "info" INFO = "info"
WARNING = "warning" WARNING = "warning"
@@ -55,16 +82,32 @@ class MatrixNotifier:
self.room_id = room_id self.room_id = room_id
self.session = requests.Session() self.session = requests.Session()
def send_message(self, message: str, msgtype: str = "m.text") -> bool: def send_message(self, message: str, msgtype: str = "m.text", markdown: bool = True) -> bool:
"""Send message to Matrix room""" """Send a message to the Matrix room.
When ``markdown`` is True (default) the ``message`` is treated as Markdown:
a rendered HTML ``formatted_body`` is sent so clients show real formatting,
with a plain-text ``body`` fallback for clients that ignore HTML.
"""
try: try:
# Add transaction ID to prevent duplicates # Add transaction ID to prevent duplicates
txn_id = datetime.datetime.now().strftime("%Y%m%d_%H%M%S_%f") txn_id = datetime.datetime.now().strftime("%Y%m%d_%H%M%S_%f")
url = f"{self.homeserver}/_matrix/client/v3/rooms/{self.room_id}/send/m.room.message/{txn_id}" url = f"{self.homeserver}/_matrix/client/v3/rooms/{self.room_id}/send/m.room.message/{txn_id}"
headers = {"Authorization": f"Bearer {self.access_token}", "Content-Type": "application/json"} headers = {
"Authorization": f"Bearer {self.access_token}",
"Content-Type": "application/json",
}
data = {"msgtype": msgtype, "body": message} if markdown:
data = {
"msgtype": msgtype,
"body": strip_markdown(message),
"format": "org.matrix.custom.html",
"formatted_body": markdown_to_matrix_html(message),
}
else:
data = {"msgtype": msgtype, "body": message}
# Matrix API requires PUT when transaction ID is in the URL path # Matrix API requires PUT when transaction ID is in the URL path
response = self.session.put(url, headers=headers, json=data, timeout=10) response = self.session.put(url, headers=headers, json=data, timeout=10)
@@ -239,7 +282,12 @@ class WaterLevelAlertSystem:
("zone_1", 3.7, AlertLevel.INFO, "Zone 1 - Info"), ("zone_1", 3.7, AlertLevel.INFO, "Zone 1 - Info"),
] ]
for zone_name, zone_threshold, zone_alert_level, zone_description in zones: for (
zone_name,
zone_threshold,
zone_alert_level,
zone_description,
) in zones:
if water_level >= zone_threshold: if water_level >= zone_threshold:
alert_level = zone_alert_level alert_level = zone_alert_level
threshold_value = zone_threshold threshold_value = zone_threshold
@@ -348,7 +396,9 @@ class WaterLevelAlertSystem:
# Get measurements for this station in the time window # Get measurements for this station in the time window
current_time = datetime.datetime.now() current_time = datetime.datetime.now()
measurements = self.db_adapter.get_measurements_by_timerange( measurements = self.db_adapter.get_measurements_by_timerange(
start_time=cutoff_time, end_time=current_time, station_codes=[station_code] start_time=cutoff_time,
end_time=current_time,
station_codes=[station_code],
) )
if len(measurements) < 2: if len(measurements) < 2:
src/config.py
+94 -82
View File
@@ -1,9 +1,10 @@
import os import os
from typing import Dict, Any, Optional from typing import Any, Dict
# Load environment variables from .env file # Load environment variables from .env file
try: try:
from dotenv import load_dotenv from dotenv import load_dotenv
load_dotenv() load_dotenv()
except ImportError: except ImportError:
# python-dotenv not installed, continue without it # python-dotenv not installed, continue without it
@@ -11,7 +12,7 @@ except ImportError:
try: try:
from .exceptions import ConfigurationError from .exceptions import ConfigurationError
from .models import DatabaseType, DatabaseConfig from .models import DatabaseType
except ImportError: except ImportError:
# Handle case when running as standalone script # Handle case when running as standalone script
class ConfigurationError(Exception): class ConfigurationError(Exception):
@@ -26,60 +27,73 @@ except ImportError:
INFLUXDB = "influxdb" INFLUXDB = "influxdb"
VICTORIAMETRICS = "victoriametrics" VICTORIAMETRICS = "victoriametrics"
class Config: class Config:
"""Configuration class for the Water Level Monitor""" """Configuration class for the Water Level Monitor"""
# Database settings # Database settings
DATABASE_PATH = os.getenv('WATER_DB_PATH', 'water_levels.db') DATABASE_PATH = os.getenv("WATER_DB_PATH", "water_levels.db")
# Website settings # Website settings
TARGET_URL = "https://hyd-app-db.rid.go.th/hydro1h.html" TARGET_URL = "https://hyd-app-db.rid.go.th/hydro1h.html"
API_URL = "https://hyd-app-db.rid.go.th/webservice/getGroupHourlyWaterLevelReportAllHL.ashx" API_URL = "https://hyd-app-db.rid.go.th/webservice/getGroupHourlyWaterLevelReportAllHL.ashx"
REQUEST_TIMEOUT = int(os.getenv('REQUEST_TIMEOUT', '30')) REQUEST_TIMEOUT = int(os.getenv("REQUEST_TIMEOUT", "30"))
USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36" USER_AGENT = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
)
# Database configuration # Database configuration
DB_TYPE = os.getenv('DB_TYPE', 'sqlite').lower() DB_TYPE = os.getenv("DB_TYPE", "sqlite").lower()
# VictoriaMetrics settings # VictoriaMetrics settings
VM_HOST = os.getenv('VM_HOST', 'vm.newedge.house') # Default to localhost; set VM_HOST in the environment for real deployments
VM_PORT = int(os.getenv('VM_PORT', '443')) # (avoids committing infrastructure hostnames to the repo).
VM_HOST = os.getenv("VM_HOST", "localhost")
VM_PORT = int(os.getenv("VM_PORT", "443"))
# Support for HTTPS URLs (e.g., behind reverse proxy) # Support for HTTPS URLs (e.g., behind reverse proxy)
VM_URL = os.getenv('VM_URL') # Full URL override (e.g., https://vm.example.com) VM_URL = os.getenv("VM_URL") # Full URL override (e.g., https://vm.example.com)
# InfluxDB settings # InfluxDB settings
INFLUX_HOST = os.getenv('INFLUX_HOST', 'localhost') INFLUX_HOST = os.getenv("INFLUX_HOST", "localhost")
INFLUX_PORT = int(os.getenv('INFLUX_PORT', '8086')) INFLUX_PORT = int(os.getenv("INFLUX_PORT", "8086"))
INFLUX_DATABASE = os.getenv('INFLUX_DATABASE', 'water_monitoring') INFLUX_DATABASE = os.getenv("INFLUX_DATABASE", "water_monitoring")
INFLUX_USERNAME = os.getenv('INFLUX_USERNAME') INFLUX_USERNAME = os.getenv("INFLUX_USERNAME")
INFLUX_PASSWORD = os.getenv('INFLUX_PASSWORD') INFLUX_PASSWORD = os.getenv("INFLUX_PASSWORD")
# PostgreSQL settings # PostgreSQL settings
POSTGRES_CONNECTION_STRING = os.getenv('POSTGRES_CONNECTION_STRING') POSTGRES_CONNECTION_STRING = os.getenv("POSTGRES_CONNECTION_STRING")
POSTGRES_HOST = os.getenv('POSTGRES_HOST', 'localhost') POSTGRES_HOST = os.getenv("POSTGRES_HOST", "localhost")
POSTGRES_PORT = int(os.getenv('POSTGRES_PORT', '5432')) POSTGRES_PORT = int(os.getenv("POSTGRES_PORT", "5432"))
POSTGRES_DB = os.getenv('POSTGRES_DB', 'water_monitoring') POSTGRES_DB = os.getenv("POSTGRES_DB", "water_monitoring")
POSTGRES_USER = os.getenv('POSTGRES_USER', 'postgres') POSTGRES_USER = os.getenv("POSTGRES_USER", "postgres")
POSTGRES_PASSWORD = os.getenv('POSTGRES_PASSWORD') POSTGRES_PASSWORD = os.getenv("POSTGRES_PASSWORD")
# MySQL settings # MySQL settings
MYSQL_CONNECTION_STRING = os.getenv('MYSQL_CONNECTION_STRING') MYSQL_CONNECTION_STRING = os.getenv("MYSQL_CONNECTION_STRING")
# Scheduler settings # Scheduler settings
SCRAPING_INTERVAL_HOURS = int(os.getenv('SCRAPING_INTERVAL_HOURS', '1')) SCRAPING_INTERVAL_HOURS = int(os.getenv("SCRAPING_INTERVAL_HOURS", "1"))
# Logging settings # Logging settings
LOG_LEVEL = os.getenv('LOG_LEVEL', 'INFO') LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO")
LOG_FILE = os.getenv('LOG_FILE', 'water_monitor.log') LOG_FILE = os.getenv("LOG_FILE", "water_monitor.log")
LOG_FORMAT = '%(asctime)s - %(levelname)s - %(message)s' LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s"
# Data retention # Data retention
DATA_RETENTION_DAYS = int(os.getenv('DATA_RETENTION_DAYS', '365')) DATA_RETENTION_DAYS = int(os.getenv("DATA_RETENTION_DAYS", "365"))
# Retry settings # Retry settings
MAX_RETRIES = int(os.getenv('MAX_RETRIES', '3')) MAX_RETRIES = int(os.getenv("MAX_RETRIES", "3"))
RETRY_DELAY_SECONDS = int(os.getenv('RETRY_DELAY_SECONDS', '60')) RETRY_DELAY_SECONDS = int(os.getenv("RETRY_DELAY_SECONDS", "60"))
# Web API / CORS settings
# Comma-separated list of allowed origins. Defaults to none (same-origin only);
# set CORS_ALLOW_ORIGINS to a specific list of front-end origins in production.
# Credentials are only enabled when explicit (non-wildcard) origins are set,
# because "*" + credentials is rejected by browsers and unsafe.
CORS_ALLOW_ORIGINS = [origin.strip() for origin in os.getenv("CORS_ALLOW_ORIGINS", "").split(",") if origin.strip()]
@classmethod @classmethod
def validate_config(cls) -> bool: def validate_config(cls) -> bool:
@@ -93,20 +107,20 @@ class Config:
errors.append(f"Invalid DB_TYPE: {cls.DB_TYPE}") errors.append(f"Invalid DB_TYPE: {cls.DB_TYPE}")
# Validate database-specific settings # Validate database-specific settings
if cls.DB_TYPE == 'victoriametrics': if cls.DB_TYPE == "victoriametrics":
if not cls.VM_HOST: if not cls.VM_HOST:
errors.append("VM_HOST is required for VictoriaMetrics") errors.append("VM_HOST is required for VictoriaMetrics")
if not isinstance(cls.VM_PORT, int) or cls.VM_PORT <= 0: if not isinstance(cls.VM_PORT, int) or cls.VM_PORT <= 0:
errors.append("VM_PORT must be a positive integer") errors.append("VM_PORT must be a positive integer")
elif cls.DB_TYPE == 'influxdb': elif cls.DB_TYPE == "influxdb":
if not cls.INFLUX_HOST: if not cls.INFLUX_HOST:
errors.append("INFLUX_HOST is required for InfluxDB") errors.append("INFLUX_HOST is required for InfluxDB")
if not cls.INFLUX_DATABASE: if not cls.INFLUX_DATABASE:
errors.append("INFLUX_DATABASE is required for InfluxDB") errors.append("INFLUX_DATABASE is required for InfluxDB")
elif cls.DB_TYPE in ['postgresql', 'mysql']: elif cls.DB_TYPE in ["postgresql", "mysql"]:
if cls.DB_TYPE == 'postgresql': if cls.DB_TYPE == "postgresql":
# Check if either connection string or individual components are provided # Check if either connection string or individual components are provided
if not cls.POSTGRES_CONNECTION_STRING: if not cls.POSTGRES_CONNECTION_STRING:
# If no connection string, check individual components # If no connection string, check individual components
@@ -137,69 +151,66 @@ class Config:
@classmethod @classmethod
def get_database_config(cls) -> Dict[str, Any]: def get_database_config(cls) -> Dict[str, Any]:
"""Returns database configuration based on DB_TYPE""" """Returns database configuration based on DB_TYPE"""
if cls.DB_TYPE == 'victoriametrics': if cls.DB_TYPE == "victoriametrics":
return {"type": "victoriametrics", "host": cls.VM_HOST, "port": cls.VM_PORT}
elif cls.DB_TYPE == "influxdb":
return { return {
'type': 'victoriametrics', "type": "influxdb",
'host': cls.VM_HOST, "host": cls.INFLUX_HOST,
'port': cls.VM_PORT "port": cls.INFLUX_PORT,
"database": cls.INFLUX_DATABASE,
"username": cls.INFLUX_USERNAME,
"password": cls.INFLUX_PASSWORD,
} }
elif cls.DB_TYPE == 'influxdb': elif cls.DB_TYPE == "postgresql":
return {
'type': 'influxdb',
'host': cls.INFLUX_HOST,
'port': cls.INFLUX_PORT,
'database': cls.INFLUX_DATABASE,
'username': cls.INFLUX_USERNAME,
'password': cls.INFLUX_PASSWORD
}
elif cls.DB_TYPE == 'postgresql':
# Use individual components if POSTGRES_CONNECTION_STRING is not provided # Use individual components if POSTGRES_CONNECTION_STRING is not provided
if cls.POSTGRES_CONNECTION_STRING: if cls.POSTGRES_CONNECTION_STRING:
return { return {
'type': 'postgresql', "type": "postgresql",
'connection_string': cls.POSTGRES_CONNECTION_STRING "connection_string": cls.POSTGRES_CONNECTION_STRING,
} }
else: else:
# Build connection string from components (automatically URL-encodes password) # Build connection string from components (automatically URL-encodes password)
import urllib.parse import urllib.parse
password = urllib.parse.quote(cls.POSTGRES_PASSWORD or 'password', safe='')
connection_string = f'postgresql://{cls.POSTGRES_USER}:{password}@{cls.POSTGRES_HOST}:{cls.POSTGRES_PORT}/{cls.POSTGRES_DB}' if not cls.POSTGRES_PASSWORD:
return { raise ConfigurationError("POSTGRES_PASSWORD is required for PostgreSQL (no default is provided)")
'type': 'postgresql', password = urllib.parse.quote(cls.POSTGRES_PASSWORD, safe="")
'connection_string': connection_string connection_string = (
} f"postgresql://{cls.POSTGRES_USER}:{password}"
elif cls.DB_TYPE == 'mysql': f"@{cls.POSTGRES_HOST}:{cls.POSTGRES_PORT}/{cls.POSTGRES_DB}"
return { )
'type': 'mysql', return {"type": "postgresql", "connection_string": connection_string}
'connection_string': cls.MYSQL_CONNECTION_STRING or elif cls.DB_TYPE == "mysql":
'mysql://root:password@localhost:3306/water_monitoring' if not cls.MYSQL_CONNECTION_STRING:
} raise ConfigurationError("MYSQL_CONNECTION_STRING is required for MySQL (no default is provided)")
return {"type": "mysql", "connection_string": cls.MYSQL_CONNECTION_STRING}
else: # sqlite else: # sqlite
return { return {
'type': 'sqlite', "type": "sqlite",
'connection_string': f'sqlite:///{cls.DATABASE_PATH}' "connection_string": f"sqlite:///{cls.DATABASE_PATH}",
} }
@classmethod @classmethod
def get_all_settings(cls) -> Dict[str, Any]: def get_all_settings(cls) -> Dict[str, Any]:
"""Returns all configuration settings""" """Returns all configuration settings"""
return { return {
'DB_TYPE': cls.DB_TYPE, "DB_TYPE": cls.DB_TYPE,
'DATABASE_PATH': cls.DATABASE_PATH, "DATABASE_PATH": cls.DATABASE_PATH,
'TARGET_URL': cls.TARGET_URL, "TARGET_URL": cls.TARGET_URL,
'API_URL': cls.API_URL, "API_URL": cls.API_URL,
'REQUEST_TIMEOUT': cls.REQUEST_TIMEOUT, "REQUEST_TIMEOUT": cls.REQUEST_TIMEOUT,
'SCRAPING_INTERVAL_HOURS': cls.SCRAPING_INTERVAL_HOURS, "SCRAPING_INTERVAL_HOURS": cls.SCRAPING_INTERVAL_HOURS,
'LOG_LEVEL': cls.LOG_LEVEL, "LOG_LEVEL": cls.LOG_LEVEL,
'LOG_FILE': cls.LOG_FILE, "LOG_FILE": cls.LOG_FILE,
'DATA_RETENTION_DAYS': cls.DATA_RETENTION_DAYS, "DATA_RETENTION_DAYS": cls.DATA_RETENTION_DAYS,
'MAX_RETRIES': cls.MAX_RETRIES, "MAX_RETRIES": cls.MAX_RETRIES,
'RETRY_DELAY_SECONDS': cls.RETRY_DELAY_SECONDS, "RETRY_DELAY_SECONDS": cls.RETRY_DELAY_SECONDS,
'VM_HOST': cls.VM_HOST, "VM_HOST": cls.VM_HOST,
'VM_PORT': cls.VM_PORT, "VM_PORT": cls.VM_PORT,
'INFLUX_HOST': cls.INFLUX_HOST, "INFLUX_HOST": cls.INFLUX_HOST,
'INFLUX_PORT': cls.INFLUX_PORT, "INFLUX_PORT": cls.INFLUX_PORT,
'INFLUX_DATABASE': cls.INFLUX_DATABASE "INFLUX_DATABASE": cls.INFLUX_DATABASE,
} }
@classmethod @classmethod
@@ -208,18 +219,19 @@ class Config:
print("=== Water Level Monitor Configuration ===") print("=== Water Level Monitor Configuration ===")
for key, value in cls.get_all_settings().items(): for key, value in cls.get_all_settings().items():
# Hide sensitive information # Hide sensitive information
if 'PASSWORD' in key and value: if "PASSWORD" in key and value:
value = '*' * len(str(value)) value = "*" * len(str(value))
print(f"{key}: {value}") print(f"{key}: {value}")
print("=" * 45) print("=" * 45)
print("\nDatabase Configuration:") print("\nDatabase Configuration:")
db_config = cls.get_database_config() db_config = cls.get_database_config()
for key, value in db_config.items(): for key, value in db_config.items():
if 'password' in key and value: if "password" in key and value:
value = '*' * len(str(value)) value = "*" * len(str(value))
print(f" {key}: {value}") print(f" {key}: {value}")
print("=" * 45) print("=" * 45)
if __name__ == "__main__": if __name__ == "__main__":
Config.print_settings() Config.print_settings()
src/database_adapters.py
+225 -150
View File
@@ -5,8 +5,9 @@ Database adapters for different storage backends
import datetime import datetime
import logging import logging
from typing import List, Dict, Optional, Any
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from typing import Dict, List, Optional
# Base adapter interface # Base adapter interface
class DatabaseAdapter(ABC): class DatabaseAdapter(ABC):
@@ -23,19 +24,29 @@ class DatabaseAdapter(ABC):
pass pass
@abstractmethod @abstractmethod
def get_measurements_by_timerange(self, start_time: datetime.datetime, def get_measurements_by_timerange(
end_time: datetime.datetime, self,
station_codes: Optional[List[str]] = None) -> List[Dict]: start_time: datetime.datetime,
end_time: datetime.datetime,
station_codes: Optional[List[str]] = None,
) -> List[Dict]:
pass pass
@abstractmethod @abstractmethod
def get_measurements_for_date(self, target_date: datetime.datetime) -> List[Dict]: def get_measurements_for_date(self, target_date: datetime.datetime) -> List[Dict]:
pass pass
# InfluxDB Adapter # InfluxDB Adapter
class InfluxDBAdapter(DatabaseAdapter): class InfluxDBAdapter(DatabaseAdapter):
def __init__(self, host: str = "localhost", port: int = 8086, def __init__(
database: str = "water_monitoring", username: str = None, password: str = None): self,
host: str = "localhost",
port: int = 8086,
database: str = "water_monitoring",
username: str = None,
password: str = None,
):
self.host = host self.host = host
self.port = port self.port = port
self.database = database self.database = database
@@ -46,29 +57,30 @@ class InfluxDBAdapter(DatabaseAdapter):
def connect(self): def connect(self):
try: try:
from influxdb import InfluxDBClient from influxdb import InfluxDBClient
self.client = InfluxDBClient( self.client = InfluxDBClient(
host=self.host, host=self.host,
port=self.port, port=self.port,
username=self.username, username=self.username,
password=self.password, password=self.password,
database=self.database database=self.database,
) )
# Create database if it doesn't exist # Create database if it doesn't exist
databases = self.client.get_list_database() databases = self.client.get_list_database()
if not any(db['name'] == self.database for db in databases): if not any(db["name"] == self.database for db in databases):
self.client.create_database(self.database) self.client.create_database(self.database)
logging.info(f"Created InfluxDB database: {self.database}") logging.info(f"Created InfluxDB database: {self.database}")
# Create retention policy (keep data for 2 years, downsample after 30 days) # Create retention policy (keep data for 2 years, downsample after 30 days)
retention_policies = self.client.get_list_retention_policies(self.database) retention_policies = self.client.get_list_retention_policies(self.database)
if not any(rp['name'] == 'water_data_policy' for rp in retention_policies): if not any(rp["name"] == "water_data_policy" for rp in retention_policies):
self.client.create_retention_policy( self.client.create_retention_policy(
'water_data_policy', "water_data_policy",
'730d', # 2 years "730d", # 2 years
'1', # replication factor "1", # replication factor
database=self.database, database=self.database,
default=True default=True,
) )
logging.info("Connected to InfluxDB successfully") logging.info("Connected to InfluxDB successfully")
@@ -92,16 +104,18 @@ class InfluxDBAdapter(DatabaseAdapter):
point = { point = {
"measurement": "water_data", "measurement": "water_data",
"tags": { "tags": {
"station_code": measurement['station_code'], "station_code": measurement["station_code"],
"station_name_en": measurement['station_name_en'], "station_name_en": measurement["station_name_en"],
"station_name_th": measurement['station_name_th'] "station_name_th": measurement["station_name_th"],
}, },
"time": measurement['timestamp'].isoformat(), "time": measurement["timestamp"].isoformat(),
"fields": { "fields": {
"water_level": float(measurement['water_level']), "water_level": float(measurement["water_level"]),
"discharge": float(measurement['discharge']), "discharge": float(measurement["discharge"]),
"discharge_percent": float(measurement['discharge_percent']) if measurement['discharge_percent'] else None "discharge_percent": float(measurement["discharge_percent"])
} if measurement["discharge_percent"]
else None,
},
} }
points.append(point) points.append(point)
@@ -119,6 +133,8 @@ class InfluxDBAdapter(DatabaseAdapter):
return [] return []
try: try:
# Cast limit to int so it can never carry an injection payload.
limit = int(limit)
query = f""" query = f"""
SELECT last("water_level") as water_level, SELECT last("water_level") as water_level,
last("discharge") as discharge, last("discharge") as discharge,
@@ -132,15 +148,17 @@ class InfluxDBAdapter(DatabaseAdapter):
measurements = [] measurements = []
for point in result.get_points(): for point in result.get_points():
measurements.append({ measurements.append(
'timestamp': point['time'], {
'station_code': point.get('station_code'), "timestamp": point["time"],
'station_name_en': point.get('station_name_en'), "station_code": point.get("station_code"),
'station_name_th': point.get('station_name_th'), "station_name_en": point.get("station_name_en"),
'water_level': point.get('water_level'), "station_name_th": point.get("station_name_th"),
'discharge': point.get('discharge'), "water_level": point.get("water_level"),
'discharge_percent': point.get('discharge_percent') "discharge": point.get("discharge"),
}) "discharge_percent": point.get("discharge_percent"),
}
)
return measurements return measurements
@@ -148,17 +166,27 @@ class InfluxDBAdapter(DatabaseAdapter):
logging.error(f"Error querying InfluxDB: {e}") logging.error(f"Error querying InfluxDB: {e}")
return [] return []
def get_measurements_by_timerange(self, start_time: datetime.datetime, def get_measurements_by_timerange(
end_time: datetime.datetime, self,
station_codes: Optional[List[str]] = None) -> List[Dict]: start_time: datetime.datetime,
end_time: datetime.datetime,
station_codes: Optional[List[str]] = None,
) -> List[Dict]:
if not self.client: if not self.client:
return [] return []
try: try:
# start_time/end_time are datetime objects (fixed isoformat, injection-safe).
# station_codes are untrusted strings -> bind them as parameters.
bind_params = {}
where_clause = f"time >= '{start_time.isoformat()}' AND time <= '{end_time.isoformat()}'" where_clause = f"time >= '{start_time.isoformat()}' AND time <= '{end_time.isoformat()}'"
if station_codes: if station_codes:
station_filter = "'" + "','".join(station_codes) + "'" placeholders = []
where_clause += f" AND station_code IN ({station_filter})" for i, code in enumerate(station_codes):
key = f"sc{i}"
bind_params[key] = code
placeholders.append(f"station_code = ${key}")
where_clause += " AND (" + " OR ".join(placeholders) + ")"
query = f""" query = f"""
SELECT "water_level", "discharge", "discharge_percent", "station_code", "station_name_en", "station_name_th" SELECT "water_level", "discharge", "discharge_percent", "station_code", "station_name_en", "station_name_th"
@@ -167,19 +195,21 @@ class InfluxDBAdapter(DatabaseAdapter):
ORDER BY time DESC ORDER BY time DESC
""" """
result = self.client.query(query) result = self.client.query(query, bind_params=bind_params)
measurements = [] measurements = []
for point in result.get_points(): for point in result.get_points():
measurements.append({ measurements.append(
'timestamp': point['time'], {
'station_code': point.get('station_code'), "timestamp": point["time"],
'station_name_en': point.get('station_name_en'), "station_code": point.get("station_code"),
'station_name_th': point.get('station_name_th'), "station_name_en": point.get("station_name_en"),
'water_level': point.get('water_level'), "station_name_th": point.get("station_name_th"),
'discharge': point.get('discharge'), "water_level": point.get("water_level"),
'discharge_percent': point.get('discharge_percent') "discharge": point.get("discharge"),
}) "discharge_percent": point.get("discharge_percent"),
}
)
return measurements return measurements
@@ -187,6 +217,7 @@ class InfluxDBAdapter(DatabaseAdapter):
logging.error(f"Error querying InfluxDB: {e}") logging.error(f"Error querying InfluxDB: {e}")
return [] return []
# MySQL/PostgreSQL Adapter # MySQL/PostgreSQL Adapter
class SQLAdapter(DatabaseAdapter): class SQLAdapter(DatabaseAdapter):
def __init__(self, connection_string: str, db_type: str = "mysql"): def __init__(self, connection_string: str, db_type: str = "mysql"):
@@ -203,8 +234,7 @@ class SQLAdapter(DatabaseAdapter):
def connect(self): def connect(self):
try: try:
from sqlalchemy import create_engine, text from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
self.engine = create_engine(self.connection_string, pool_pre_ping=True) self.engine = create_engine(self.connection_string, pool_pre_ping=True)
@@ -258,7 +288,7 @@ class SQLAdapter(DatabaseAdapter):
# Create indexes separately for SQLite # Create indexes separately for SQLite
index_sql = [ index_sql = [
"CREATE INDEX IF NOT EXISTS idx_timestamp ON water_measurements(timestamp)", "CREATE INDEX IF NOT EXISTS idx_timestamp ON water_measurements(timestamp)",
"CREATE INDEX IF NOT EXISTS idx_station_timestamp ON water_measurements(station_id, timestamp)" "CREATE INDEX IF NOT EXISTS idx_station_timestamp ON water_measurements(station_id, timestamp)",
] ]
elif self.db_type == "postgresql": elif self.db_type == "postgresql":
@@ -293,7 +323,7 @@ class SQLAdapter(DatabaseAdapter):
index_sql = [ index_sql = [
"CREATE INDEX IF NOT EXISTS idx_timestamp ON water_measurements(timestamp)", "CREATE INDEX IF NOT EXISTS idx_timestamp ON water_measurements(timestamp)",
"CREATE INDEX IF NOT EXISTS idx_station_timestamp ON water_measurements(station_id, timestamp DESC)" "CREATE INDEX IF NOT EXISTS idx_station_timestamp ON water_measurements(station_id, timestamp DESC)",
] ]
else: # MySQL else: # MySQL
@@ -351,13 +381,21 @@ class SQLAdapter(DatabaseAdapter):
for measurement in measurements: for measurement in measurements:
if self.db_type == "sqlite": if self.db_type == "sqlite":
station_sql = """ station_sql = """
INSERT OR REPLACE INTO stations (id, station_code, thai_name, english_name, latitude, longitude, geohash, updated_at) INSERT OR REPLACE INTO stations
VALUES (:station_id, :station_code, :thai_name, :english_name, :latitude, :longitude, :geohash, CURRENT_TIMESTAMP) (id, station_code, thai_name, english_name,
latitude, longitude, geohash, updated_at)
VALUES
(:station_id, :station_code, :thai_name, :english_name,
:latitude, :longitude, :geohash, CURRENT_TIMESTAMP)
""" """
elif self.db_type == "postgresql": elif self.db_type == "postgresql":
station_sql = """ station_sql = """
INSERT INTO stations (id, station_code, thai_name, english_name, latitude, longitude, geohash, updated_at) INSERT INTO stations
VALUES (:station_id, :station_code, :thai_name, :english_name, :latitude, :longitude, :geohash, NOW()) (id, station_code, thai_name, english_name,
latitude, longitude, geohash, updated_at)
VALUES
(:station_id, :station_code, :thai_name, :english_name,
:latitude, :longitude, :geohash, NOW())
ON CONFLICT (id) DO UPDATE SET ON CONFLICT (id) DO UPDATE SET
thai_name = EXCLUDED.thai_name, thai_name = EXCLUDED.thai_name,
english_name = EXCLUDED.english_name, english_name = EXCLUDED.english_name,
@@ -368,8 +406,12 @@ class SQLAdapter(DatabaseAdapter):
""" """
else: # MySQL else: # MySQL
station_sql = """ station_sql = """
INSERT INTO stations (id, station_code, thai_name, english_name, latitude, longitude, geohash, updated_at) INSERT INTO stations
VALUES (:station_id, :station_code, :thai_name, :english_name, :latitude, :longitude, :geohash, NOW()) (id, station_code, thai_name, english_name,
latitude, longitude, geohash, updated_at)
VALUES
(:station_id, :station_code, :thai_name, :english_name,
:latitude, :longitude, :geohash, NOW())
ON DUPLICATE KEY UPDATE ON DUPLICATE KEY UPDATE
thai_name = VALUES(thai_name), thai_name = VALUES(thai_name),
english_name = VALUES(english_name), english_name = VALUES(english_name),
@@ -379,15 +421,18 @@ class SQLAdapter(DatabaseAdapter):
updated_at = NOW() updated_at = NOW()
""" """
conn.execute(text(station_sql), { conn.execute(
'station_id': measurement['station_id'], text(station_sql),
'station_code': measurement['station_code'], {
'thai_name': measurement['station_name_th'], "station_id": measurement["station_id"],
'english_name': measurement['station_name_en'], "station_code": measurement["station_code"],
'latitude': measurement.get('latitude'), "thai_name": measurement["station_name_th"],
'longitude': measurement.get('longitude'), "english_name": measurement["station_name_en"],
'geohash': measurement.get('geohash') "latitude": measurement.get("latitude"),
}) "longitude": measurement.get("longitude"),
"geohash": measurement.get("geohash"),
},
)
# Insert measurements # Insert measurements
for measurement in measurements: for measurement in measurements:
@@ -420,14 +465,17 @@ class SQLAdapter(DatabaseAdapter):
status = VALUES(status) status = VALUES(status)
""" """
conn.execute(text(measurement_sql), { conn.execute(
'timestamp': measurement['timestamp'], text(measurement_sql),
'station_id': measurement['station_id'], {
'water_level': measurement['water_level'], "timestamp": measurement["timestamp"],
'discharge': measurement['discharge'], "station_id": measurement["station_id"],
'discharge_percent': measurement['discharge_percent'], "water_level": measurement["water_level"],
'status': measurement['status'] "discharge": measurement["discharge"],
}) "discharge_percent": measurement["discharge_percent"],
"status": measurement["status"],
},
)
# Transaction is automatically committed when context manager exits # Transaction is automatically committed when context manager exits
logging.info(f"Successfully saved {len(measurements)} measurements to {self.db_type.upper()}") logging.info(f"Successfully saved {len(measurements)} measurements to {self.db_type.upper()}")
@@ -459,20 +507,22 @@ class SQLAdapter(DatabaseAdapter):
""" """
with self.engine.connect() as conn: with self.engine.connect() as conn:
result = conn.execute(text(query), {'limit': limit}) result = conn.execute(text(query), {"limit": limit})
measurements = [] measurements = []
for row in result: for row in result:
measurements.append({ measurements.append(
'timestamp': row[0], {
'station_code': row[1], "timestamp": row[0],
'station_name_en': row[2], "station_code": row[1],
'station_name_th': row[3], "station_name_en": row[2],
'water_level': float(row[4]) if row[4] else None, "station_name_th": row[3],
'discharge': float(row[5]) if row[5] else None, "water_level": float(row[4]) if row[4] else None,
'discharge_percent': float(row[6]) if row[6] else None, "discharge": float(row[5]) if row[5] else None,
'status': row[7] "discharge_percent": float(row[6]) if row[6] else None,
}) "status": row[7],
}
)
return measurements return measurements
@@ -480,9 +530,12 @@ class SQLAdapter(DatabaseAdapter):
logging.error(f"Error querying {self.db_type.upper()}: {e}") logging.error(f"Error querying {self.db_type.upper()}: {e}")
return [] return []
def get_measurements_by_timerange(self, start_time: datetime.datetime, def get_measurements_by_timerange(
end_time: datetime.datetime, self,
station_codes: Optional[List[str]] = None) -> List[Dict]: start_time: datetime.datetime,
end_time: datetime.datetime,
station_codes: Optional[List[str]] = None,
) -> List[Dict]:
if not self.engine: if not self.engine:
return [] return []
@@ -490,13 +543,13 @@ class SQLAdapter(DatabaseAdapter):
from sqlalchemy import text from sqlalchemy import text
where_clause = "m.timestamp BETWEEN :start_time AND :end_time" where_clause = "m.timestamp BETWEEN :start_time AND :end_time"
params = {'start_time': start_time, 'end_time': end_time} params = {"start_time": start_time, "end_time": end_time}
if station_codes: if station_codes:
placeholders = ','.join([f':station_{i}' for i in range(len(station_codes))]) placeholders = ",".join([f":station_{i}" for i in range(len(station_codes))])
where_clause += f" AND s.station_code IN ({placeholders})" where_clause += f" AND s.station_code IN ({placeholders})"
for i, code in enumerate(station_codes): for i, code in enumerate(station_codes):
params[f'station_{i}'] = code params[f"station_{i}"] = code
query = f""" query = f"""
SELECT m.timestamp, s.station_code, s.english_name, s.thai_name, SELECT m.timestamp, s.station_code, s.english_name, s.thai_name,
@@ -512,16 +565,18 @@ class SQLAdapter(DatabaseAdapter):
measurements = [] measurements = []
for row in result: for row in result:
measurements.append({ measurements.append(
'timestamp': row[0], {
'station_code': row[1], "timestamp": row[0],
'station_name_en': row[2], "station_code": row[1],
'station_name_th': row[3], "station_name_en": row[2],
'water_level': float(row[4]) if row[4] else None, "station_name_th": row[3],
'discharge': float(row[5]) if row[5] else None, "water_level": float(row[4]) if row[4] else None,
'discharge_percent': float(row[6]) if row[6] else None, "discharge": float(row[5]) if row[5] else None,
'status': row[7] "discharge_percent": float(row[6]) if row[6] else None,
}) "status": row[7],
}
)
return measurements return measurements
@@ -551,23 +606,22 @@ class SQLAdapter(DatabaseAdapter):
""" """
with self.engine.connect() as conn: with self.engine.connect() as conn:
result = conn.execute(text(query), { result = conn.execute(text(query), {"start_time": start_of_day, "end_time": end_of_day})
'start_time': start_of_day,
'end_time': end_of_day
})
measurements = [] measurements = []
for row in result: for row in result:
measurements.append({ measurements.append(
'timestamp': row[0], {
'station_id': row[1], "timestamp": row[0],
'station_code': row[2] or f"Station_{row[1]}", "station_id": row[1],
'station_name_th': row[3] or f"Station {row[1]}", "station_code": row[2] or f"Station_{row[1]}",
'water_level': float(row[4]) if row[4] else None, "station_name_th": row[3] or f"Station {row[1]}",
'discharge': float(row[5]) if row[5] else None, "water_level": float(row[4]) if row[4] else None,
'discharge_percent': float(row[6]) if row[6] else None, "discharge": float(row[5]) if row[5] else None,
'status': row[7] "discharge_percent": float(row[6]) if row[6] else None,
}) "status": row[7],
}
)
return measurements return measurements
@@ -575,6 +629,7 @@ class SQLAdapter(DatabaseAdapter):
logging.error(f"Error querying {self.db_type.upper()} for date {target_date.date()}: {e}") logging.error(f"Error querying {self.db_type.upper()} for date {target_date.date()}: {e}")
return [] return []
# VictoriaMetrics Adapter (using Prometheus format) # VictoriaMetrics Adapter (using Prometheus format)
class VictoriaMetricsAdapter(DatabaseAdapter): class VictoriaMetricsAdapter(DatabaseAdapter):
def __init__(self, host: str = "localhost", port: int = 8428): def __init__(self, host: str = "localhost", port: int = 8428):
@@ -582,11 +637,11 @@ class VictoriaMetricsAdapter(DatabaseAdapter):
self.port = port self.port = port
# Handle HTTPS URLs and reverse proxy configurations # Handle HTTPS URLs and reverse proxy configurations
if host.startswith(('http://', 'https://')): if host.startswith(("http://", "https://")):
self.base_url = host self.base_url = host
if port != 80 and port != 443 and not host.endswith(f':{port}'): if port != 80 and port != 443 and not host.endswith(f":{port}"):
# Only add port if it's not standard and not already in URL # Only add port if it's not standard and not already in URL
if '://' in host and ':' not in host.split('://')[1]: if "://" in host and ":" not in host.split("://")[1]:
self.base_url = f"{host}:{port}" self.base_url = f"{host}:{port}"
else: else:
# Default to HTTP for localhost, HTTPS for remote hosts # Default to HTTP for localhost, HTTPS for remote hosts
@@ -596,14 +651,33 @@ class VictoriaMetricsAdapter(DatabaseAdapter):
else: else:
self.base_url = f"{protocol}://{host}:{port}" self.base_url = f"{protocol}://{host}:{port}"
@staticmethod
def _escape_label(value) -> str:
"""Escape a Prometheus label value per the exposition format spec.
Station names include arbitrary Thai text (and could be set via the API),
so backslashes, double-quotes and newlines must be escaped to avoid
producing malformed or injected exposition lines.
"""
return str(value).replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n")
@staticmethod
def _metric_value(value) -> Optional[float]:
"""Coerce a numeric field to float, or None if it isn't a valid number."""
try:
return float(value)
except (TypeError, ValueError):
return None
def connect(self): def connect(self):
try: try:
import requests import requests
# Test connection with SSL verification and timeout # Test connection with SSL verification and timeout
response = requests.get( response = requests.get(
f"{self.base_url}/api/v1/status/config", f"{self.base_url}/api/v1/status/config",
timeout=10, timeout=10,
verify=True # Enable SSL verification for HTTPS verify=True, # Enable SSL verification for HTTPS
) )
if response.status_code == 200: if response.status_code == 200:
logging.info(f"Connected to VictoriaMetrics successfully at {self.base_url}") logging.info(f"Connected to VictoriaMetrics successfully at {self.base_url}")
@@ -630,38 +704,35 @@ class VictoriaMetricsAdapter(DatabaseAdapter):
timestamp_ms = int(datetime.datetime.now().timestamp() * 1000) timestamp_ms = int(datetime.datetime.now().timestamp() * 1000)
for measurement in measurements: for measurement in measurements:
# Water level metric # Escape label values once per measurement (untrusted Thai/English names).
metrics_data.append( labels = (
f'water_level{{station_code="{measurement["station_code"]}",' f'station_code="{self._escape_label(measurement["station_code"])}",'
f'station_name_en="{measurement["station_name_en"]}",' f'station_name_en="{self._escape_label(measurement["station_name_en"])}",'
f'station_name_th="{measurement["station_name_th"]}"}} ' f'station_name_th="{self._escape_label(measurement["station_name_th"])}"'
f'{measurement["water_level"]} {timestamp_ms}'
) )
# Water level metric
water_level = self._metric_value(measurement.get("water_level"))
if water_level is not None:
metrics_data.append(f"water_level{{{labels}}} {water_level} {timestamp_ms}")
# Discharge metric # Discharge metric
metrics_data.append( discharge = self._metric_value(measurement.get("discharge"))
f'water_discharge{{station_code="{measurement["station_code"]}",' if discharge is not None:
f'station_name_en="{measurement["station_name_en"]}",' metrics_data.append(f"water_discharge{{{labels}}} {discharge} {timestamp_ms}")
f'station_name_th="{measurement["station_name_th"]}"}} '
f'{measurement["discharge"]} {timestamp_ms}'
)
# Discharge percentage metric # Discharge percentage metric
if measurement["discharge_percent"]: discharge_percent = self._metric_value(measurement.get("discharge_percent"))
metrics_data.append( if discharge_percent is not None:
f'water_discharge_percent{{station_code="{measurement["station_code"]}",' metrics_data.append(f"water_discharge_percent{{{labels}}} {discharge_percent} {timestamp_ms}")
f'station_name_en="{measurement["station_name_en"]}",'
f'station_name_th="{measurement["station_name_th"]}"}} '
f'{measurement["discharge_percent"]} {timestamp_ms}'
)
# Send to VictoriaMetrics # Send to VictoriaMetrics
data = '\n'.join(metrics_data) data = "\n".join(metrics_data)
response = requests.post( response = requests.post(
f"{self.base_url}/api/v1/import/prometheus", f"{self.base_url}/api/v1/import/prometheus",
data=data, data=data,
headers={'Content-Type': 'text/plain'}, headers={"Content-Type": "text/plain"},
timeout=30 timeout=30,
) )
if response.status_code == 204: if response.status_code == 204:
@@ -681,9 +752,12 @@ class VictoriaMetricsAdapter(DatabaseAdapter):
logging.warning("get_latest_measurements not fully implemented for VictoriaMetrics") logging.warning("get_latest_measurements not fully implemented for VictoriaMetrics")
return [] return []
def get_measurements_by_timerange(self, start_time: datetime.datetime, def get_measurements_by_timerange(
end_time: datetime.datetime, self,
station_codes: Optional[List[str]] = None) -> List[Dict]: start_time: datetime.datetime,
end_time: datetime.datetime,
station_codes: Optional[List[str]] = None,
) -> List[Dict]:
# VictoriaMetrics range queries would be implemented here # VictoriaMetrics range queries would be implemented here
logging.warning("get_measurements_by_timerange not fully implemented for VictoriaMetrics") logging.warning("get_measurements_by_timerange not fully implemented for VictoriaMetrics")
return [] return []
@@ -693,6 +767,7 @@ class VictoriaMetricsAdapter(DatabaseAdapter):
logging.warning("get_measurements_for_date not fully implemented for VictoriaMetrics") logging.warning("get_measurements_for_date not fully implemented for VictoriaMetrics")
return [] return []
# Factory function to create appropriate adapter # Factory function to create appropriate adapter
def create_database_adapter(db_type: str, **kwargs) -> DatabaseAdapter: def create_database_adapter(db_type: str, **kwargs) -> DatabaseAdapter:
""" """
@@ -704,15 +779,15 @@ def create_database_adapter(db_type: str, **kwargs) -> DatabaseAdapter:
""" """
db_type = db_type.lower() db_type = db_type.lower()
if db_type == 'influxdb': if db_type == "influxdb":
return InfluxDBAdapter(**kwargs) return InfluxDBAdapter(**kwargs)
elif db_type == 'mysql': elif db_type == "mysql":
return SQLAdapter(db_type='mysql', **kwargs) return SQLAdapter(db_type="mysql", **kwargs)
elif db_type == 'postgresql': elif db_type == "postgresql":
return SQLAdapter(db_type='postgresql', **kwargs) return SQLAdapter(db_type="postgresql", **kwargs)
elif db_type == 'sqlite': elif db_type == "sqlite":
return SQLAdapter(db_type='sqlite', **kwargs) return SQLAdapter(db_type="sqlite", **kwargs)
elif db_type == 'victoriametrics': elif db_type == "victoriametrics":
return VictoriaMetricsAdapter(**kwargs) return VictoriaMetricsAdapter(**kwargs)
else: else:
raise ValueError(f"Unsupported database type: {db_type}") raise ValueError(f"Unsupported database type: {db_type}")
src/web_api.py
+116 -90
View File
@@ -4,26 +4,24 @@ FastAPI web interface for water monitoring system
""" """
import asyncio import asyncio
import threading
from datetime import datetime, timedelta
from typing import List, Dict, Any, Optional
from contextlib import asynccontextmanager from contextlib import asynccontextmanager
from datetime import datetime, timedelta
from typing import Any, Dict, List, Optional
from fastapi import FastAPI, HTTPException, BackgroundTasks, Depends from fastapi import BackgroundTasks, FastAPI, HTTPException
from fastapi.responses import HTMLResponse, JSONResponse
from fastapi.staticfiles import StaticFiles
from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import HTMLResponse
from pydantic import BaseModel, Field from pydantic import BaseModel, Field
from .water_scraper_v3 import EnhancedWaterMonitorScraper
from .config import Config from .config import Config
from .models import WaterMeasurement, StationInfo, ScrapingResult, StationCreateRequest, StationUpdateRequest, StationStatus from .health_check import APIHealthCheck, DatabaseHealthCheck, HealthCheckManager, MemoryHealthCheck
from .health_check import HealthCheckManager, DatabaseHealthCheck, APIHealthCheck, MemoryHealthCheck from .logging_config import get_logger, setup_logging
from .metrics import get_metrics_collector, increment_counter, set_gauge from .metrics import get_metrics_collector, increment_counter, set_gauge
from .logging_config import setup_logging, get_logger from .water_scraper_v3 import EnhancedWaterMonitorScraper
logger = get_logger(__name__) logger = get_logger(__name__)
# Pydantic models for API responses # Pydantic models for API responses
class StationResponse(BaseModel): class StationResponse(BaseModel):
station_id: int station_id: int
@@ -35,6 +33,7 @@ class StationResponse(BaseModel):
geohash: Optional[str] = None geohash: Optional[str] = None
status: str = "active" status: str = "active"
class StationCreateModel(BaseModel): class StationCreateModel(BaseModel):
station_code: str = Field(..., description="Station code (e.g., P.1, P.20)") station_code: str = Field(..., description="Station code (e.g., P.1, P.20)")
thai_name: str = Field(..., description="Thai name of the station") thai_name: str = Field(..., description="Thai name of the station")
@@ -44,6 +43,7 @@ class StationCreateModel(BaseModel):
geohash: Optional[str] = Field(None, description="Geohash for the location") geohash: Optional[str] = Field(None, description="Geohash for the location")
status: str = Field("active", description="Station status") status: str = Field("active", description="Station status")
class StationUpdateModel(BaseModel): class StationUpdateModel(BaseModel):
thai_name: Optional[str] = Field(None, description="Thai name of the station") thai_name: Optional[str] = Field(None, description="Thai name of the station")
english_name: Optional[str] = Field(None, description="English name of the station") english_name: Optional[str] = Field(None, description="English name of the station")
@@ -52,6 +52,7 @@ class StationUpdateModel(BaseModel):
geohash: Optional[str] = Field(None, description="Geohash for the location") geohash: Optional[str] = Field(None, description="Geohash for the location")
status: Optional[str] = Field(None, description="Station status") status: Optional[str] = Field(None, description="Station status")
class MeasurementResponse(BaseModel): class MeasurementResponse(BaseModel):
timestamp: datetime timestamp: datetime
station_code: str station_code: str
@@ -62,16 +63,19 @@ class MeasurementResponse(BaseModel):
discharge_percent: Optional[float] = None discharge_percent: Optional[float] = None
status: str = "active" status: str = "active"
class HealthResponse(BaseModel): class HealthResponse(BaseModel):
overall_status: str overall_status: str
timestamp: str timestamp: str
checks: Dict[str, Dict[str, Any]] checks: Dict[str, Dict[str, Any]]
class MetricsResponse(BaseModel): class MetricsResponse(BaseModel):
counters: Dict[str, float] counters: Dict[str, float]
gauges: Dict[str, float] gauges: Dict[str, float]
histograms: Dict[str, Dict[str, float]] histograms: Dict[str, Dict[str, float]]
class ScrapingStatusResponse(BaseModel): class ScrapingStatusResponse(BaseModel):
is_running: bool is_running: bool
last_run: Optional[datetime] = None last_run: Optional[datetime] = None
@@ -80,6 +84,7 @@ class ScrapingStatusResponse(BaseModel):
successful_runs: int = 0 successful_runs: int = 0
failed_runs: int = 0 failed_runs: int = 0
# Global application state # Global application state
app_state = { app_state = {
"scraper": None, "scraper": None,
@@ -91,10 +96,11 @@ app_state = {
"successful_runs": 0, "successful_runs": 0,
"failed_runs": 0, "failed_runs": 0,
"last_run": None, "last_run": None,
"next_run": None "next_run": None,
} },
} }
@asynccontextmanager @asynccontextmanager
async def lifespan(app: FastAPI): async def lifespan(app: FastAPI):
"""Application lifespan manager""" """Application lifespan manager"""
@@ -139,23 +145,30 @@ async def lifespan(app: FastAPI):
logger.info("Water Monitor API shutdown complete") logger.info("Water Monitor API shutdown complete")
# Create FastAPI app # Create FastAPI app
app = FastAPI( app = FastAPI(
title="Northern Thailand Ping River Monitor API", title="Northern Thailand Ping River Monitor API",
description="Real-time water level monitoring system for Northern Thailand's Ping River Basin stations", description="Real-time water level monitoring system for Northern Thailand's Ping River Basin stations",
version="3.1.3", version="3.1.3",
lifespan=lifespan lifespan=lifespan,
) )
# Add CORS middleware # Add CORS middleware.
# Origins come from CORS_ALLOW_ORIGINS (comma-separated). When none are configured
# we fall back to a wildcard WITHOUT credentials (a safe, spec-valid combination);
# credentials are only enabled when explicit origins are provided.
_cors_origins = Config.CORS_ALLOW_ORIGINS or ["*"]
_cors_allow_credentials = bool(Config.CORS_ALLOW_ORIGINS)
app.add_middleware( app.add_middleware(
CORSMiddleware, CORSMiddleware,
allow_origins=["*"], # Configure appropriately for production allow_origins=_cors_origins,
allow_credentials=True, allow_credentials=_cors_allow_credentials,
allow_methods=["*"], allow_methods=["*"],
allow_headers=["*"], allow_headers=["*"],
) )
async def background_scraping_task(): async def background_scraping_task():
"""Background task for periodic data scraping""" """Background task for periodic data scraping"""
while True: while True:
@@ -170,7 +183,9 @@ async def background_scraping_task():
start_time = datetime.now() start_time = datetime.now()
try: try:
result = scraper.run_scraping_cycle() # run_scraping_cycle() does blocking network/DB I/O and time.sleep
# retries; run it in a thread so it doesn't freeze the event loop.
result = await asyncio.get_event_loop().run_in_executor(None, scraper.run_scraping_cycle)
# Update stats # Update stats
app_state["scraping_stats"]["total_runs"] += 1 app_state["scraping_stats"]["total_runs"] += 1
@@ -209,8 +224,10 @@ async def background_scraping_task():
logger.error(f"Error in background scraping task: {e}") logger.error(f"Error in background scraping task: {e}")
await asyncio.sleep(60) # Wait a minute before retrying await asyncio.sleep(60) # Wait a minute before retrying
# API Routes # API Routes
@app.get("/", response_class=HTMLResponse) @app.get("/", response_class=HTMLResponse)
async def root(): async def root():
"""Root endpoint with basic dashboard""" """Root endpoint with basic dashboard"""
@@ -268,6 +285,7 @@ async def root():
""" """
return HTMLResponse(content=html_content) return HTMLResponse(content=html_content)
@app.get("/health", response_model=HealthResponse) @app.get("/health", response_model=HealthResponse)
async def get_health(): async def get_health():
"""Get system health status""" """Get system health status"""
@@ -277,12 +295,13 @@ async def get_health():
if not health_manager: if not health_manager:
raise HTTPException(status_code=503, detail="Health manager not initialized") raise HTTPException(status_code=503, detail="Health manager not initialized")
# Run health checks # Run health checks (populates state read by get_health_summary)
results = health_manager.run_all_checks() health_manager.run_all_checks()
summary = health_manager.get_health_summary() summary = health_manager.get_health_summary()
return HealthResponse(**summary) return HealthResponse(**summary)
@app.get("/metrics", response_model=MetricsResponse) @app.get("/metrics", response_model=MetricsResponse)
async def get_metrics(): async def get_metrics():
"""Get application metrics""" """Get application metrics"""
@@ -293,6 +312,7 @@ async def get_metrics():
return MetricsResponse(**metrics) return MetricsResponse(**metrics)
@app.get("/stations", response_model=List[StationResponse]) @app.get("/stations", response_model=List[StationResponse])
async def get_stations(): async def get_stations():
"""Get list of all monitoring stations""" """Get list of all monitoring stations"""
@@ -304,18 +324,21 @@ async def get_stations():
stations = [] stations = []
for station_id, station_info in scraper.station_mapping.items(): for station_id, station_info in scraper.station_mapping.items():
stations.append(StationResponse( stations.append(
station_id=int(station_id), StationResponse(
station_code=station_info["code"], station_id=int(station_id),
thai_name=station_info["thai_name"], station_code=station_info["code"],
english_name=station_info["english_name"], thai_name=station_info["thai_name"],
latitude=station_info.get("latitude"), english_name=station_info["english_name"],
longitude=station_info.get("longitude"), latitude=station_info.get("latitude"),
status="active" longitude=station_info.get("longitude"),
)) status="active",
)
)
return stations return stations
@app.post("/stations", response_model=StationResponse) @app.post("/stations", response_model=StationResponse)
async def create_station(station: StationCreateModel): async def create_station(station: StationCreateModel):
"""Create a new monitoring station""" """Create a new monitoring station"""
@@ -332,12 +355,12 @@ async def create_station(station: StationCreateModel):
# Add to station mapping # Add to station mapping
scraper.station_mapping[str(new_station_id)] = { scraper.station_mapping[str(new_station_id)] = {
'code': station.station_code, "code": station.station_code,
'thai_name': station.thai_name, "thai_name": station.thai_name,
'english_name': station.english_name, "english_name": station.english_name,
'latitude': station.latitude, "latitude": station.latitude,
'longitude': station.longitude, "longitude": station.longitude,
'geohash': station.geohash "geohash": station.geohash,
} }
logger.info(f"Created new station: {station.station_code} ({station.english_name})") logger.info(f"Created new station: {station.station_code} ({station.english_name})")
@@ -350,13 +373,14 @@ async def create_station(station: StationCreateModel):
latitude=station.latitude, latitude=station.latitude,
longitude=station.longitude, longitude=station.longitude,
geohash=station.geohash, geohash=station.geohash,
status=station.status status=station.status,
) )
except Exception as e: except Exception as e:
logger.error(f"Error creating station: {e}") logger.error(f"Error creating station: {e}")
raise HTTPException(status_code=500, detail=str(e)) raise HTTPException(status_code=500, detail=str(e))
@app.put("/stations/{station_id}", response_model=StationResponse) @app.put("/stations/{station_id}", response_model=StationResponse)
async def update_station(station_id: int, updates: StationUpdateModel): async def update_station(station_id: int, updates: StationUpdateModel):
"""Update an existing monitoring station""" """Update an existing monitoring station"""
@@ -375,33 +399,34 @@ async def update_station(station_id: int, updates: StationUpdateModel):
# Update fields if provided # Update fields if provided
if updates.thai_name is not None: if updates.thai_name is not None:
station_info['thai_name'] = updates.thai_name station_info["thai_name"] = updates.thai_name
if updates.english_name is not None: if updates.english_name is not None:
station_info['english_name'] = updates.english_name station_info["english_name"] = updates.english_name
if updates.latitude is not None: if updates.latitude is not None:
station_info['latitude'] = updates.latitude station_info["latitude"] = updates.latitude
if updates.longitude is not None: if updates.longitude is not None:
station_info['longitude'] = updates.longitude station_info["longitude"] = updates.longitude
if updates.geohash is not None: if updates.geohash is not None:
station_info['geohash'] = updates.geohash station_info["geohash"] = updates.geohash
logger.info(f"Updated station {station_id}: {station_info['code']}") logger.info(f"Updated station {station_id}: {station_info['code']}")
return StationResponse( return StationResponse(
station_id=station_id, station_id=station_id,
station_code=station_info['code'], station_code=station_info["code"],
thai_name=station_info['thai_name'], thai_name=station_info["thai_name"],
english_name=station_info['english_name'], english_name=station_info["english_name"],
latitude=station_info.get('latitude'), latitude=station_info.get("latitude"),
longitude=station_info.get('longitude'), longitude=station_info.get("longitude"),
geohash=station_info.get('geohash'), geohash=station_info.get("geohash"),
status=updates.status or "active" status=updates.status or "active",
) )
except Exception as e: except Exception as e:
logger.error(f"Error updating station {station_id}: {e}") logger.error(f"Error updating station {station_id}: {e}")
raise HTTPException(status_code=500, detail=str(e)) raise HTTPException(status_code=500, detail=str(e))
@app.delete("/stations/{station_id}") @app.delete("/stations/{station_id}")
async def delete_station(station_id: int): async def delete_station(station_id: int):
"""Delete a monitoring station""" """Delete a monitoring station"""
@@ -425,6 +450,7 @@ async def delete_station(station_id: int):
logger.error(f"Error deleting station {station_id}: {e}") logger.error(f"Error deleting station {station_id}: {e}")
raise HTTPException(status_code=500, detail=str(e)) raise HTTPException(status_code=500, detail=str(e))
@app.get("/stations/{station_id}", response_model=StationResponse) @app.get("/stations/{station_id}", response_model=StationResponse)
async def get_station(station_id: int): async def get_station(station_id: int):
"""Get details of a specific monitoring station""" """Get details of a specific monitoring station"""
@@ -442,15 +468,16 @@ async def get_station(station_id: int):
return StationResponse( return StationResponse(
station_id=station_id, station_id=station_id,
station_code=station_info['code'], station_code=station_info["code"],
thai_name=station_info['thai_name'], thai_name=station_info["thai_name"],
english_name=station_info['english_name'], english_name=station_info["english_name"],
latitude=station_info.get('latitude'), latitude=station_info.get("latitude"),
longitude=station_info.get('longitude'), longitude=station_info.get("longitude"),
geohash=station_info.get('geohash'), geohash=station_info.get("geohash"),
status="active" status="active",
) )
@app.get("/measurements/latest", response_model=List[MeasurementResponse]) @app.get("/measurements/latest", response_model=List[MeasurementResponse])
async def get_latest_measurements(limit: int = 100): async def get_latest_measurements(limit: int = 100):
"""Get latest measurements from all stations""" """Get latest measurements from all stations"""
@@ -465,16 +492,18 @@ async def get_latest_measurements(limit: int = 100):
response = [] response = []
for measurement in measurements: for measurement in measurements:
response.append(MeasurementResponse( response.append(
timestamp=measurement["timestamp"], MeasurementResponse(
station_code=measurement["station_code"], timestamp=measurement["timestamp"],
station_name_en=measurement["station_name_en"], station_code=measurement["station_code"],
station_name_th=measurement["station_name_th"], station_name_en=measurement["station_name_en"],
water_level=measurement["water_level"], station_name_th=measurement["station_name_th"],
discharge=measurement["discharge"], water_level=measurement["water_level"],
discharge_percent=measurement.get("discharge_percent"), discharge=measurement["discharge"],
status=measurement.get("status", "active") discharge_percent=measurement.get("discharge_percent"),
)) status=measurement.get("status", "active"),
)
)
return response return response
@@ -482,12 +511,9 @@ async def get_latest_measurements(limit: int = 100):
logger.error(f"Error fetching latest measurements: {e}") logger.error(f"Error fetching latest measurements: {e}")
raise HTTPException(status_code=500, detail=str(e)) raise HTTPException(status_code=500, detail=str(e))
@app.get("/measurements/station/{station_code}", response_model=List[MeasurementResponse]) @app.get("/measurements/station/{station_code}", response_model=List[MeasurementResponse])
async def get_station_measurements( async def get_station_measurements(station_code: str, hours: int = 24, limit: int = 1000):
station_code: str,
hours: int = 24,
limit: int = 1000
):
"""Get measurements for a specific station""" """Get measurements for a specific station"""
increment_counter("api_requests", labels={"endpoint": "measurements_station"}) increment_counter("api_requests", labels={"endpoint": "measurements_station"})
@@ -509,16 +535,18 @@ async def get_station_measurements(
response = [] response = []
for measurement in measurements: for measurement in measurements:
response.append(MeasurementResponse( response.append(
timestamp=measurement["timestamp"], MeasurementResponse(
station_code=measurement["station_code"], timestamp=measurement["timestamp"],
station_name_en=measurement["station_name_en"], station_code=measurement["station_code"],
station_name_th=measurement["station_name_th"], station_name_en=measurement["station_name_en"],
water_level=measurement["water_level"], station_name_th=measurement["station_name_th"],
discharge=measurement["discharge"], water_level=measurement["water_level"],
discharge_percent=measurement.get("discharge_percent"), discharge=measurement["discharge"],
status=measurement.get("status", "active") discharge_percent=measurement.get("discharge_percent"),
)) status=measurement.get("status", "active"),
)
)
return response return response
@@ -526,6 +554,7 @@ async def get_station_measurements(
logger.error(f"Error fetching station measurements: {e}") logger.error(f"Error fetching station measurements: {e}")
raise HTTPException(status_code=500, detail=str(e)) raise HTTPException(status_code=500, detail=str(e))
@app.post("/scrape/trigger") @app.post("/scrape/trigger")
async def trigger_scraping(background_tasks: BackgroundTasks): async def trigger_scraping(background_tasks: BackgroundTasks):
"""Trigger manual data scraping""" """Trigger manual data scraping"""
@@ -568,6 +597,7 @@ async def trigger_scraping(background_tasks: BackgroundTasks):
return {"message": "Scraping triggered", "status": "started"} return {"message": "Scraping triggered", "status": "started"}
@app.get("/scraping/status", response_model=ScrapingStatusResponse) @app.get("/scraping/status", response_model=ScrapingStatusResponse)
async def get_scraping_status(): async def get_scraping_status():
"""Get current scraping status""" """Get current scraping status"""
@@ -581,9 +611,10 @@ async def get_scraping_status():
next_run=stats["next_run"], next_run=stats["next_run"],
total_runs=stats["total_runs"], total_runs=stats["total_runs"],
successful_runs=stats["successful_runs"], successful_runs=stats["successful_runs"],
failed_runs=stats["failed_runs"] failed_runs=stats["failed_runs"],
) )
@app.get("/config") @app.get("/config")
async def get_config(): async def get_config():
"""Get current configuration (sensitive data masked)""" """Get current configuration (sensitive data masked)"""
@@ -593,12 +624,13 @@ async def get_config():
# Mask sensitive information # Mask sensitive information
for key in config: for key in config:
if 'password' in key.lower() or 'secret' in key.lower(): if "password" in key.lower() or "secret" in key.lower():
if config[key]: if config[key]:
config[key] = '*' * 8 config[key] = "*" * 8
return config return config
if __name__ == "__main__": if __name__ == "__main__":
import uvicorn import uvicorn
@@ -607,14 +639,8 @@ if __name__ == "__main__":
log_level=Config.LOG_LEVEL, log_level=Config.LOG_LEVEL,
log_file=Config.LOG_FILE, log_file=Config.LOG_FILE,
enable_console=True, enable_console=True,
enable_colors=True enable_colors=True,
) )
# Run the API server # Run the API server
uvicorn.run( uvicorn.run("web_api:app", host="0.0.0.0", port=8000, reload=False, log_config=None) # Use our custom logging
"web_api:app",
host="0.0.0.0",
port=8000,
reload=False,
log_config=None # Use our custom logging
)