Fix Matrix message formatting and harden security

Matrix alerts:
- Send HTML formatted_body (org.matrix.custom.html) so **bold** and URLs
  render instead of showing literal Markdown; add plain-text body fallback.
  Add dependency-free markdown_to_matrix_html/strip_markdown helpers with
  HTML escaping of station/message data.

Security:
- InfluxDB: bind untrusted station_codes as query params and cast limit to
  int (was f-string interpolation / injection risk).
- VictoriaMetrics: escape Prometheus label values and coerce metric values
  to float, preventing exposition-format injection and None crashes.
- web_api: run blocking scrape cycle via run_in_executor so it no longer
  freezes the event loop; make CORS origins configurable and only allow
  credentials with explicit origins ("*" + credentials is invalid/unsafe).
- config: remove hardcoded root/postgres password fallbacks (raise instead)
  and stop defaulting VM_HOST to a real infrastructure hostname.

Also remove unused imports and wrap long lines to satisfy flake8.
This commit is contained in:
2026-07-22 12:07:05 +07:00
parent d3ec5a77e6
commit f4c63cabef
4 changed files with 688 additions and 525 deletions
+56 -6
View File
@@ -4,7 +4,9 @@ Water Level Alerting System with Matrix Integration
"""
import datetime
import html
import os
import re
from dataclasses import dataclass
from enum import Enum
from typing import Dict, List, Optional
@@ -28,6 +30,31 @@ except ImportError:
logger = get_logger(__name__)
_BOLD_RE = re.compile(r"\*\*(.+?)\*\*")
_URL_RE = re.compile(r"(https?://[^\s<]+)")
def markdown_to_matrix_html(text: str) -> str:
"""Convert the small Markdown subset we emit into Matrix-compatible HTML.
Matrix clients do NOT render Markdown in the plain ``body`` field; formatting
only shows when an HTML ``formatted_body`` is sent alongside it. We only use
``**bold**``, bare URLs and newlines, so a minimal converter is sufficient and
avoids adding a Markdown dependency.
"""
# Escape HTML special chars first so station/message data can't inject markup.
result = html.escape(text, quote=False)
result = _BOLD_RE.sub(r"<strong>\1</strong>", result)
result = _URL_RE.sub(r'<a href="\1">\1</a>', result)
result = result.replace("\n", "<br/>")
return result
def strip_markdown(text: str) -> str:
"""Produce a clean plain-text fallback for the Matrix ``body`` field."""
return _BOLD_RE.sub(r"\1", text)
class AlertLevel(Enum):
INFO = "info"
WARNING = "warning"
@@ -55,16 +82,32 @@ class MatrixNotifier:
self.room_id = room_id
self.session = requests.Session()
def send_message(self, message: str, msgtype: str = "m.text") -> bool:
"""Send message to Matrix room"""
def send_message(self, message: str, msgtype: str = "m.text", markdown: bool = True) -> bool:
"""Send a message to the Matrix room.
When ``markdown`` is True (default) the ``message`` is treated as Markdown:
a rendered HTML ``formatted_body`` is sent so clients show real formatting,
with a plain-text ``body`` fallback for clients that ignore HTML.
"""
try:
# Add transaction ID to prevent duplicates
txn_id = datetime.datetime.now().strftime("%Y%m%d_%H%M%S_%f")
url = f"{self.homeserver}/_matrix/client/v3/rooms/{self.room_id}/send/m.room.message/{txn_id}"
headers = {"Authorization": f"Bearer {self.access_token}", "Content-Type": "application/json"}
headers = {
"Authorization": f"Bearer {self.access_token}",
"Content-Type": "application/json",
}
data = {"msgtype": msgtype, "body": message}
if markdown:
data = {
"msgtype": msgtype,
"body": strip_markdown(message),
"format": "org.matrix.custom.html",
"formatted_body": markdown_to_matrix_html(message),
}
else:
data = {"msgtype": msgtype, "body": message}
# Matrix API requires PUT when transaction ID is in the URL path
response = self.session.put(url, headers=headers, json=data, timeout=10)
@@ -239,7 +282,12 @@ class WaterLevelAlertSystem:
("zone_1", 3.7, AlertLevel.INFO, "Zone 1 - Info"),
]
for zone_name, zone_threshold, zone_alert_level, zone_description in zones:
for (
zone_name,
zone_threshold,
zone_alert_level,
zone_description,
) in zones:
if water_level >= zone_threshold:
alert_level = zone_alert_level
threshold_value = zone_threshold
@@ -348,7 +396,9 @@ class WaterLevelAlertSystem:
# Get measurements for this station in the time window
current_time = datetime.datetime.now()
measurements = self.db_adapter.get_measurements_by_timerange(
start_time=cutoff_time, end_time=current_time, station_codes=[station_code]
start_time=cutoff_time,
end_time=current_time,
station_codes=[station_code],
)
if len(measurements) < 2:
+94 -82
View File
@@ -1,9 +1,10 @@
import os
from typing import Dict, Any, Optional
from typing import Any, Dict
# Load environment variables from .env file
try:
from dotenv import load_dotenv
load_dotenv()
except ImportError:
# python-dotenv not installed, continue without it
@@ -11,7 +12,7 @@ except ImportError:
try:
from .exceptions import ConfigurationError
from .models import DatabaseType, DatabaseConfig
from .models import DatabaseType
except ImportError:
# Handle case when running as standalone script
class ConfigurationError(Exception):
@@ -26,60 +27,73 @@ except ImportError:
INFLUXDB = "influxdb"
VICTORIAMETRICS = "victoriametrics"
class Config:
"""Configuration class for the Water Level Monitor"""
# Database settings
DATABASE_PATH = os.getenv('WATER_DB_PATH', 'water_levels.db')
DATABASE_PATH = os.getenv("WATER_DB_PATH", "water_levels.db")
# Website settings
TARGET_URL = "https://hyd-app-db.rid.go.th/hydro1h.html"
API_URL = "https://hyd-app-db.rid.go.th/webservice/getGroupHourlyWaterLevelReportAllHL.ashx"
REQUEST_TIMEOUT = int(os.getenv('REQUEST_TIMEOUT', '30'))
USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
REQUEST_TIMEOUT = int(os.getenv("REQUEST_TIMEOUT", "30"))
USER_AGENT = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
)
# Database configuration
DB_TYPE = os.getenv('DB_TYPE', 'sqlite').lower()
DB_TYPE = os.getenv("DB_TYPE", "sqlite").lower()
# VictoriaMetrics settings
VM_HOST = os.getenv('VM_HOST', 'vm.newedge.house')
VM_PORT = int(os.getenv('VM_PORT', '443'))
# Default to localhost; set VM_HOST in the environment for real deployments
# (avoids committing infrastructure hostnames to the repo).
VM_HOST = os.getenv("VM_HOST", "localhost")
VM_PORT = int(os.getenv("VM_PORT", "443"))
# Support for HTTPS URLs (e.g., behind reverse proxy)
VM_URL = os.getenv('VM_URL') # Full URL override (e.g., https://vm.example.com)
VM_URL = os.getenv("VM_URL") # Full URL override (e.g., https://vm.example.com)
# InfluxDB settings
INFLUX_HOST = os.getenv('INFLUX_HOST', 'localhost')
INFLUX_PORT = int(os.getenv('INFLUX_PORT', '8086'))
INFLUX_DATABASE = os.getenv('INFLUX_DATABASE', 'water_monitoring')
INFLUX_USERNAME = os.getenv('INFLUX_USERNAME')
INFLUX_PASSWORD = os.getenv('INFLUX_PASSWORD')
INFLUX_HOST = os.getenv("INFLUX_HOST", "localhost")
INFLUX_PORT = int(os.getenv("INFLUX_PORT", "8086"))
INFLUX_DATABASE = os.getenv("INFLUX_DATABASE", "water_monitoring")
INFLUX_USERNAME = os.getenv("INFLUX_USERNAME")
INFLUX_PASSWORD = os.getenv("INFLUX_PASSWORD")
# PostgreSQL settings
POSTGRES_CONNECTION_STRING = os.getenv('POSTGRES_CONNECTION_STRING')
POSTGRES_HOST = os.getenv('POSTGRES_HOST', 'localhost')
POSTGRES_PORT = int(os.getenv('POSTGRES_PORT', '5432'))
POSTGRES_DB = os.getenv('POSTGRES_DB', 'water_monitoring')
POSTGRES_USER = os.getenv('POSTGRES_USER', 'postgres')
POSTGRES_PASSWORD = os.getenv('POSTGRES_PASSWORD')
POSTGRES_CONNECTION_STRING = os.getenv("POSTGRES_CONNECTION_STRING")
POSTGRES_HOST = os.getenv("POSTGRES_HOST", "localhost")
POSTGRES_PORT = int(os.getenv("POSTGRES_PORT", "5432"))
POSTGRES_DB = os.getenv("POSTGRES_DB", "water_monitoring")
POSTGRES_USER = os.getenv("POSTGRES_USER", "postgres")
POSTGRES_PASSWORD = os.getenv("POSTGRES_PASSWORD")
# MySQL settings
MYSQL_CONNECTION_STRING = os.getenv('MYSQL_CONNECTION_STRING')
MYSQL_CONNECTION_STRING = os.getenv("MYSQL_CONNECTION_STRING")
# Scheduler settings
SCRAPING_INTERVAL_HOURS = int(os.getenv('SCRAPING_INTERVAL_HOURS', '1'))
SCRAPING_INTERVAL_HOURS = int(os.getenv("SCRAPING_INTERVAL_HOURS", "1"))
# Logging settings
LOG_LEVEL = os.getenv('LOG_LEVEL', 'INFO')
LOG_FILE = os.getenv('LOG_FILE', 'water_monitor.log')
LOG_FORMAT = '%(asctime)s - %(levelname)s - %(message)s'
LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO")
LOG_FILE = os.getenv("LOG_FILE", "water_monitor.log")
LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s"
# Data retention
DATA_RETENTION_DAYS = int(os.getenv('DATA_RETENTION_DAYS', '365'))
DATA_RETENTION_DAYS = int(os.getenv("DATA_RETENTION_DAYS", "365"))
# Retry settings
MAX_RETRIES = int(os.getenv('MAX_RETRIES', '3'))
RETRY_DELAY_SECONDS = int(os.getenv('RETRY_DELAY_SECONDS', '60'))
MAX_RETRIES = int(os.getenv("MAX_RETRIES", "3"))
RETRY_DELAY_SECONDS = int(os.getenv("RETRY_DELAY_SECONDS", "60"))
# Web API / CORS settings
# Comma-separated list of allowed origins. Defaults to none (same-origin only);
# set CORS_ALLOW_ORIGINS to a specific list of front-end origins in production.
# Credentials are only enabled when explicit (non-wildcard) origins are set,
# because "*" + credentials is rejected by browsers and unsafe.
CORS_ALLOW_ORIGINS = [origin.strip() for origin in os.getenv("CORS_ALLOW_ORIGINS", "").split(",") if origin.strip()]
@classmethod
def validate_config(cls) -> bool:
@@ -93,20 +107,20 @@ class Config:
errors.append(f"Invalid DB_TYPE: {cls.DB_TYPE}")
# Validate database-specific settings
if cls.DB_TYPE == 'victoriametrics':
if cls.DB_TYPE == "victoriametrics":
if not cls.VM_HOST:
errors.append("VM_HOST is required for VictoriaMetrics")
if not isinstance(cls.VM_PORT, int) or cls.VM_PORT <= 0:
errors.append("VM_PORT must be a positive integer")
elif cls.DB_TYPE == 'influxdb':
elif cls.DB_TYPE == "influxdb":
if not cls.INFLUX_HOST:
errors.append("INFLUX_HOST is required for InfluxDB")
if not cls.INFLUX_DATABASE:
errors.append("INFLUX_DATABASE is required for InfluxDB")
elif cls.DB_TYPE in ['postgresql', 'mysql']:
if cls.DB_TYPE == 'postgresql':
elif cls.DB_TYPE in ["postgresql", "mysql"]:
if cls.DB_TYPE == "postgresql":
# Check if either connection string or individual components are provided
if not cls.POSTGRES_CONNECTION_STRING:
# If no connection string, check individual components
@@ -137,69 +151,66 @@ class Config:
@classmethod
def get_database_config(cls) -> Dict[str, Any]:
"""Returns database configuration based on DB_TYPE"""
if cls.DB_TYPE == 'victoriametrics':
if cls.DB_TYPE == "victoriametrics":
return {"type": "victoriametrics", "host": cls.VM_HOST, "port": cls.VM_PORT}
elif cls.DB_TYPE == "influxdb":
return {
'type': 'victoriametrics',
'host': cls.VM_HOST,
'port': cls.VM_PORT
"type": "influxdb",
"host": cls.INFLUX_HOST,
"port": cls.INFLUX_PORT,
"database": cls.INFLUX_DATABASE,
"username": cls.INFLUX_USERNAME,
"password": cls.INFLUX_PASSWORD,
}
elif cls.DB_TYPE == 'influxdb':
return {
'type': 'influxdb',
'host': cls.INFLUX_HOST,
'port': cls.INFLUX_PORT,
'database': cls.INFLUX_DATABASE,
'username': cls.INFLUX_USERNAME,
'password': cls.INFLUX_PASSWORD
}
elif cls.DB_TYPE == 'postgresql':
elif cls.DB_TYPE == "postgresql":
# Use individual components if POSTGRES_CONNECTION_STRING is not provided
if cls.POSTGRES_CONNECTION_STRING:
return {
'type': 'postgresql',
'connection_string': cls.POSTGRES_CONNECTION_STRING
"type": "postgresql",
"connection_string": cls.POSTGRES_CONNECTION_STRING,
}
else:
# Build connection string from components (automatically URL-encodes password)
import urllib.parse
password = urllib.parse.quote(cls.POSTGRES_PASSWORD or 'password', safe='')
connection_string = f'postgresql://{cls.POSTGRES_USER}:{password}@{cls.POSTGRES_HOST}:{cls.POSTGRES_PORT}/{cls.POSTGRES_DB}'
return {
'type': 'postgresql',
'connection_string': connection_string
}
elif cls.DB_TYPE == 'mysql':
return {
'type': 'mysql',
'connection_string': cls.MYSQL_CONNECTION_STRING or
'mysql://root:password@localhost:3306/water_monitoring'
}
if not cls.POSTGRES_PASSWORD:
raise ConfigurationError("POSTGRES_PASSWORD is required for PostgreSQL (no default is provided)")
password = urllib.parse.quote(cls.POSTGRES_PASSWORD, safe="")
connection_string = (
f"postgresql://{cls.POSTGRES_USER}:{password}"
f"@{cls.POSTGRES_HOST}:{cls.POSTGRES_PORT}/{cls.POSTGRES_DB}"
)
return {"type": "postgresql", "connection_string": connection_string}
elif cls.DB_TYPE == "mysql":
if not cls.MYSQL_CONNECTION_STRING:
raise ConfigurationError("MYSQL_CONNECTION_STRING is required for MySQL (no default is provided)")
return {"type": "mysql", "connection_string": cls.MYSQL_CONNECTION_STRING}
else: # sqlite
return {
'type': 'sqlite',
'connection_string': f'sqlite:///{cls.DATABASE_PATH}'
"type": "sqlite",
"connection_string": f"sqlite:///{cls.DATABASE_PATH}",
}
@classmethod
def get_all_settings(cls) -> Dict[str, Any]:
"""Returns all configuration settings"""
return {
'DB_TYPE': cls.DB_TYPE,
'DATABASE_PATH': cls.DATABASE_PATH,
'TARGET_URL': cls.TARGET_URL,
'API_URL': cls.API_URL,
'REQUEST_TIMEOUT': cls.REQUEST_TIMEOUT,
'SCRAPING_INTERVAL_HOURS': cls.SCRAPING_INTERVAL_HOURS,
'LOG_LEVEL': cls.LOG_LEVEL,
'LOG_FILE': cls.LOG_FILE,
'DATA_RETENTION_DAYS': cls.DATA_RETENTION_DAYS,
'MAX_RETRIES': cls.MAX_RETRIES,
'RETRY_DELAY_SECONDS': cls.RETRY_DELAY_SECONDS,
'VM_HOST': cls.VM_HOST,
'VM_PORT': cls.VM_PORT,
'INFLUX_HOST': cls.INFLUX_HOST,
'INFLUX_PORT': cls.INFLUX_PORT,
'INFLUX_DATABASE': cls.INFLUX_DATABASE
"DB_TYPE": cls.DB_TYPE,
"DATABASE_PATH": cls.DATABASE_PATH,
"TARGET_URL": cls.TARGET_URL,
"API_URL": cls.API_URL,
"REQUEST_TIMEOUT": cls.REQUEST_TIMEOUT,
"SCRAPING_INTERVAL_HOURS": cls.SCRAPING_INTERVAL_HOURS,
"LOG_LEVEL": cls.LOG_LEVEL,
"LOG_FILE": cls.LOG_FILE,
"DATA_RETENTION_DAYS": cls.DATA_RETENTION_DAYS,
"MAX_RETRIES": cls.MAX_RETRIES,
"RETRY_DELAY_SECONDS": cls.RETRY_DELAY_SECONDS,
"VM_HOST": cls.VM_HOST,
"VM_PORT": cls.VM_PORT,
"INFLUX_HOST": cls.INFLUX_HOST,
"INFLUX_PORT": cls.INFLUX_PORT,
"INFLUX_DATABASE": cls.INFLUX_DATABASE,
}
@classmethod
@@ -208,18 +219,19 @@ class Config:
print("=== Water Level Monitor Configuration ===")
for key, value in cls.get_all_settings().items():
# Hide sensitive information
if 'PASSWORD' in key and value:
value = '*' * len(str(value))
if "PASSWORD" in key and value:
value = "*" * len(str(value))
print(f"{key}: {value}")
print("=" * 45)
print("\nDatabase Configuration:")
db_config = cls.get_database_config()
for key, value in db_config.items():
if 'password' in key and value:
value = '*' * len(str(value))
if "password" in key and value:
value = "*" * len(str(value))
print(f" {key}: {value}")
print("=" * 45)
if __name__ == "__main__":
Config.print_settings()
+225 -150
View File
@@ -5,8 +5,9 @@ Database adapters for different storage backends
import datetime
import logging
from typing import List, Dict, Optional, Any
from abc import ABC, abstractmethod
from typing import Dict, List, Optional
# Base adapter interface
class DatabaseAdapter(ABC):
@@ -23,19 +24,29 @@ class DatabaseAdapter(ABC):
pass
@abstractmethod
def get_measurements_by_timerange(self, start_time: datetime.datetime,
end_time: datetime.datetime,
station_codes: Optional[List[str]] = None) -> List[Dict]:
def get_measurements_by_timerange(
self,
start_time: datetime.datetime,
end_time: datetime.datetime,
station_codes: Optional[List[str]] = None,
) -> List[Dict]:
pass
@abstractmethod
def get_measurements_for_date(self, target_date: datetime.datetime) -> List[Dict]:
pass
# InfluxDB Adapter
class InfluxDBAdapter(DatabaseAdapter):
def __init__(self, host: str = "localhost", port: int = 8086,
database: str = "water_monitoring", username: str = None, password: str = None):
def __init__(
self,
host: str = "localhost",
port: int = 8086,
database: str = "water_monitoring",
username: str = None,
password: str = None,
):
self.host = host
self.port = port
self.database = database
@@ -46,29 +57,30 @@ class InfluxDBAdapter(DatabaseAdapter):
def connect(self):
try:
from influxdb import InfluxDBClient
self.client = InfluxDBClient(
host=self.host,
port=self.port,
username=self.username,
password=self.password,
database=self.database
database=self.database,
)
# Create database if it doesn't exist
databases = self.client.get_list_database()
if not any(db['name'] == self.database for db in databases):
if not any(db["name"] == self.database for db in databases):
self.client.create_database(self.database)
logging.info(f"Created InfluxDB database: {self.database}")
# Create retention policy (keep data for 2 years, downsample after 30 days)
retention_policies = self.client.get_list_retention_policies(self.database)
if not any(rp['name'] == 'water_data_policy' for rp in retention_policies):
if not any(rp["name"] == "water_data_policy" for rp in retention_policies):
self.client.create_retention_policy(
'water_data_policy',
'730d', # 2 years
'1', # replication factor
"water_data_policy",
"730d", # 2 years
"1", # replication factor
database=self.database,
default=True
default=True,
)
logging.info("Connected to InfluxDB successfully")
@@ -92,16 +104,18 @@ class InfluxDBAdapter(DatabaseAdapter):
point = {
"measurement": "water_data",
"tags": {
"station_code": measurement['station_code'],
"station_name_en": measurement['station_name_en'],
"station_name_th": measurement['station_name_th']
"station_code": measurement["station_code"],
"station_name_en": measurement["station_name_en"],
"station_name_th": measurement["station_name_th"],
},
"time": measurement['timestamp'].isoformat(),
"time": measurement["timestamp"].isoformat(),
"fields": {
"water_level": float(measurement['water_level']),
"discharge": float(measurement['discharge']),
"discharge_percent": float(measurement['discharge_percent']) if measurement['discharge_percent'] else None
}
"water_level": float(measurement["water_level"]),
"discharge": float(measurement["discharge"]),
"discharge_percent": float(measurement["discharge_percent"])
if measurement["discharge_percent"]
else None,
},
}
points.append(point)
@@ -119,6 +133,8 @@ class InfluxDBAdapter(DatabaseAdapter):
return []
try:
# Cast limit to int so it can never carry an injection payload.
limit = int(limit)
query = f"""
SELECT last("water_level") as water_level,
last("discharge") as discharge,
@@ -132,15 +148,17 @@ class InfluxDBAdapter(DatabaseAdapter):
measurements = []
for point in result.get_points():
measurements.append({
'timestamp': point['time'],
'station_code': point.get('station_code'),
'station_name_en': point.get('station_name_en'),
'station_name_th': point.get('station_name_th'),
'water_level': point.get('water_level'),
'discharge': point.get('discharge'),
'discharge_percent': point.get('discharge_percent')
})
measurements.append(
{
"timestamp": point["time"],
"station_code": point.get("station_code"),
"station_name_en": point.get("station_name_en"),
"station_name_th": point.get("station_name_th"),
"water_level": point.get("water_level"),
"discharge": point.get("discharge"),
"discharge_percent": point.get("discharge_percent"),
}
)
return measurements
@@ -148,17 +166,27 @@ class InfluxDBAdapter(DatabaseAdapter):
logging.error(f"Error querying InfluxDB: {e}")
return []
def get_measurements_by_timerange(self, start_time: datetime.datetime,
end_time: datetime.datetime,
station_codes: Optional[List[str]] = None) -> List[Dict]:
def get_measurements_by_timerange(
self,
start_time: datetime.datetime,
end_time: datetime.datetime,
station_codes: Optional[List[str]] = None,
) -> List[Dict]:
if not self.client:
return []
try:
# start_time/end_time are datetime objects (fixed isoformat, injection-safe).
# station_codes are untrusted strings -> bind them as parameters.
bind_params = {}
where_clause = f"time >= '{start_time.isoformat()}' AND time <= '{end_time.isoformat()}'"
if station_codes:
station_filter = "'" + "','".join(station_codes) + "'"
where_clause += f" AND station_code IN ({station_filter})"
placeholders = []
for i, code in enumerate(station_codes):
key = f"sc{i}"
bind_params[key] = code
placeholders.append(f"station_code = ${key}")
where_clause += " AND (" + " OR ".join(placeholders) + ")"
query = f"""
SELECT "water_level", "discharge", "discharge_percent", "station_code", "station_name_en", "station_name_th"
@@ -167,19 +195,21 @@ class InfluxDBAdapter(DatabaseAdapter):
ORDER BY time DESC
"""
result = self.client.query(query)
result = self.client.query(query, bind_params=bind_params)
measurements = []
for point in result.get_points():
measurements.append({
'timestamp': point['time'],
'station_code': point.get('station_code'),
'station_name_en': point.get('station_name_en'),
'station_name_th': point.get('station_name_th'),
'water_level': point.get('water_level'),
'discharge': point.get('discharge'),
'discharge_percent': point.get('discharge_percent')
})
measurements.append(
{
"timestamp": point["time"],
"station_code": point.get("station_code"),
"station_name_en": point.get("station_name_en"),
"station_name_th": point.get("station_name_th"),
"water_level": point.get("water_level"),
"discharge": point.get("discharge"),
"discharge_percent": point.get("discharge_percent"),
}
)
return measurements
@@ -187,6 +217,7 @@ class InfluxDBAdapter(DatabaseAdapter):
logging.error(f"Error querying InfluxDB: {e}")
return []
# MySQL/PostgreSQL Adapter
class SQLAdapter(DatabaseAdapter):
def __init__(self, connection_string: str, db_type: str = "mysql"):
@@ -203,8 +234,7 @@ class SQLAdapter(DatabaseAdapter):
def connect(self):
try:
from sqlalchemy import create_engine, text
from sqlalchemy.orm import sessionmaker
from sqlalchemy import create_engine
self.engine = create_engine(self.connection_string, pool_pre_ping=True)
@@ -258,7 +288,7 @@ class SQLAdapter(DatabaseAdapter):
# Create indexes separately for SQLite
index_sql = [
"CREATE INDEX IF NOT EXISTS idx_timestamp ON water_measurements(timestamp)",
"CREATE INDEX IF NOT EXISTS idx_station_timestamp ON water_measurements(station_id, timestamp)"
"CREATE INDEX IF NOT EXISTS idx_station_timestamp ON water_measurements(station_id, timestamp)",
]
elif self.db_type == "postgresql":
@@ -293,7 +323,7 @@ class SQLAdapter(DatabaseAdapter):
index_sql = [
"CREATE INDEX IF NOT EXISTS idx_timestamp ON water_measurements(timestamp)",
"CREATE INDEX IF NOT EXISTS idx_station_timestamp ON water_measurements(station_id, timestamp DESC)"
"CREATE INDEX IF NOT EXISTS idx_station_timestamp ON water_measurements(station_id, timestamp DESC)",
]
else: # MySQL
@@ -351,13 +381,21 @@ class SQLAdapter(DatabaseAdapter):
for measurement in measurements:
if self.db_type == "sqlite":
station_sql = """
INSERT OR REPLACE INTO stations (id, station_code, thai_name, english_name, latitude, longitude, geohash, updated_at)
VALUES (:station_id, :station_code, :thai_name, :english_name, :latitude, :longitude, :geohash, CURRENT_TIMESTAMP)
INSERT OR REPLACE INTO stations
(id, station_code, thai_name, english_name,
latitude, longitude, geohash, updated_at)
VALUES
(:station_id, :station_code, :thai_name, :english_name,
:latitude, :longitude, :geohash, CURRENT_TIMESTAMP)
"""
elif self.db_type == "postgresql":
station_sql = """
INSERT INTO stations (id, station_code, thai_name, english_name, latitude, longitude, geohash, updated_at)
VALUES (:station_id, :station_code, :thai_name, :english_name, :latitude, :longitude, :geohash, NOW())
INSERT INTO stations
(id, station_code, thai_name, english_name,
latitude, longitude, geohash, updated_at)
VALUES
(:station_id, :station_code, :thai_name, :english_name,
:latitude, :longitude, :geohash, NOW())
ON CONFLICT (id) DO UPDATE SET
thai_name = EXCLUDED.thai_name,
english_name = EXCLUDED.english_name,
@@ -368,8 +406,12 @@ class SQLAdapter(DatabaseAdapter):
"""
else: # MySQL
station_sql = """
INSERT INTO stations (id, station_code, thai_name, english_name, latitude, longitude, geohash, updated_at)
VALUES (:station_id, :station_code, :thai_name, :english_name, :latitude, :longitude, :geohash, NOW())
INSERT INTO stations
(id, station_code, thai_name, english_name,
latitude, longitude, geohash, updated_at)
VALUES
(:station_id, :station_code, :thai_name, :english_name,
:latitude, :longitude, :geohash, NOW())
ON DUPLICATE KEY UPDATE
thai_name = VALUES(thai_name),
english_name = VALUES(english_name),
@@ -379,15 +421,18 @@ class SQLAdapter(DatabaseAdapter):
updated_at = NOW()
"""
conn.execute(text(station_sql), {
'station_id': measurement['station_id'],
'station_code': measurement['station_code'],
'thai_name': measurement['station_name_th'],
'english_name': measurement['station_name_en'],
'latitude': measurement.get('latitude'),
'longitude': measurement.get('longitude'),
'geohash': measurement.get('geohash')
})
conn.execute(
text(station_sql),
{
"station_id": measurement["station_id"],
"station_code": measurement["station_code"],
"thai_name": measurement["station_name_th"],
"english_name": measurement["station_name_en"],
"latitude": measurement.get("latitude"),
"longitude": measurement.get("longitude"),
"geohash": measurement.get("geohash"),
},
)
# Insert measurements
for measurement in measurements:
@@ -420,14 +465,17 @@ class SQLAdapter(DatabaseAdapter):
status = VALUES(status)
"""
conn.execute(text(measurement_sql), {
'timestamp': measurement['timestamp'],
'station_id': measurement['station_id'],
'water_level': measurement['water_level'],
'discharge': measurement['discharge'],
'discharge_percent': measurement['discharge_percent'],
'status': measurement['status']
})
conn.execute(
text(measurement_sql),
{
"timestamp": measurement["timestamp"],
"station_id": measurement["station_id"],
"water_level": measurement["water_level"],
"discharge": measurement["discharge"],
"discharge_percent": measurement["discharge_percent"],
"status": measurement["status"],
},
)
# Transaction is automatically committed when context manager exits
logging.info(f"Successfully saved {len(measurements)} measurements to {self.db_type.upper()}")
@@ -459,20 +507,22 @@ class SQLAdapter(DatabaseAdapter):
"""
with self.engine.connect() as conn:
result = conn.execute(text(query), {'limit': limit})
result = conn.execute(text(query), {"limit": limit})
measurements = []
for row in result:
measurements.append({
'timestamp': row[0],
'station_code': row[1],
'station_name_en': row[2],
'station_name_th': row[3],
'water_level': float(row[4]) if row[4] else None,
'discharge': float(row[5]) if row[5] else None,
'discharge_percent': float(row[6]) if row[6] else None,
'status': row[7]
})
measurements.append(
{
"timestamp": row[0],
"station_code": row[1],
"station_name_en": row[2],
"station_name_th": row[3],
"water_level": float(row[4]) if row[4] else None,
"discharge": float(row[5]) if row[5] else None,
"discharge_percent": float(row[6]) if row[6] else None,
"status": row[7],
}
)
return measurements
@@ -480,9 +530,12 @@ class SQLAdapter(DatabaseAdapter):
logging.error(f"Error querying {self.db_type.upper()}: {e}")
return []
def get_measurements_by_timerange(self, start_time: datetime.datetime,
end_time: datetime.datetime,
station_codes: Optional[List[str]] = None) -> List[Dict]:
def get_measurements_by_timerange(
self,
start_time: datetime.datetime,
end_time: datetime.datetime,
station_codes: Optional[List[str]] = None,
) -> List[Dict]:
if not self.engine:
return []
@@ -490,13 +543,13 @@ class SQLAdapter(DatabaseAdapter):
from sqlalchemy import text
where_clause = "m.timestamp BETWEEN :start_time AND :end_time"
params = {'start_time': start_time, 'end_time': end_time}
params = {"start_time": start_time, "end_time": end_time}
if station_codes:
placeholders = ','.join([f':station_{i}' for i in range(len(station_codes))])
placeholders = ",".join([f":station_{i}" for i in range(len(station_codes))])
where_clause += f" AND s.station_code IN ({placeholders})"
for i, code in enumerate(station_codes):
params[f'station_{i}'] = code
params[f"station_{i}"] = code
query = f"""
SELECT m.timestamp, s.station_code, s.english_name, s.thai_name,
@@ -512,16 +565,18 @@ class SQLAdapter(DatabaseAdapter):
measurements = []
for row in result:
measurements.append({
'timestamp': row[0],
'station_code': row[1],
'station_name_en': row[2],
'station_name_th': row[3],
'water_level': float(row[4]) if row[4] else None,
'discharge': float(row[5]) if row[5] else None,
'discharge_percent': float(row[6]) if row[6] else None,
'status': row[7]
})
measurements.append(
{
"timestamp": row[0],
"station_code": row[1],
"station_name_en": row[2],
"station_name_th": row[3],
"water_level": float(row[4]) if row[4] else None,
"discharge": float(row[5]) if row[5] else None,
"discharge_percent": float(row[6]) if row[6] else None,
"status": row[7],
}
)
return measurements
@@ -551,23 +606,22 @@ class SQLAdapter(DatabaseAdapter):
"""
with self.engine.connect() as conn:
result = conn.execute(text(query), {
'start_time': start_of_day,
'end_time': end_of_day
})
result = conn.execute(text(query), {"start_time": start_of_day, "end_time": end_of_day})
measurements = []
for row in result:
measurements.append({
'timestamp': row[0],
'station_id': row[1],
'station_code': row[2] or f"Station_{row[1]}",
'station_name_th': row[3] or f"Station {row[1]}",
'water_level': float(row[4]) if row[4] else None,
'discharge': float(row[5]) if row[5] else None,
'discharge_percent': float(row[6]) if row[6] else None,
'status': row[7]
})
measurements.append(
{
"timestamp": row[0],
"station_id": row[1],
"station_code": row[2] or f"Station_{row[1]}",
"station_name_th": row[3] or f"Station {row[1]}",
"water_level": float(row[4]) if row[4] else None,
"discharge": float(row[5]) if row[5] else None,
"discharge_percent": float(row[6]) if row[6] else None,
"status": row[7],
}
)
return measurements
@@ -575,6 +629,7 @@ class SQLAdapter(DatabaseAdapter):
logging.error(f"Error querying {self.db_type.upper()} for date {target_date.date()}: {e}")
return []
# VictoriaMetrics Adapter (using Prometheus format)
class VictoriaMetricsAdapter(DatabaseAdapter):
def __init__(self, host: str = "localhost", port: int = 8428):
@@ -582,11 +637,11 @@ class VictoriaMetricsAdapter(DatabaseAdapter):
self.port = port
# Handle HTTPS URLs and reverse proxy configurations
if host.startswith(('http://', 'https://')):
if host.startswith(("http://", "https://")):
self.base_url = host
if port != 80 and port != 443 and not host.endswith(f':{port}'):
if port != 80 and port != 443 and not host.endswith(f":{port}"):
# Only add port if it's not standard and not already in URL
if '://' in host and ':' not in host.split('://')[1]:
if "://" in host and ":" not in host.split("://")[1]:
self.base_url = f"{host}:{port}"
else:
# Default to HTTP for localhost, HTTPS for remote hosts
@@ -596,14 +651,33 @@ class VictoriaMetricsAdapter(DatabaseAdapter):
else:
self.base_url = f"{protocol}://{host}:{port}"
@staticmethod
def _escape_label(value) -> str:
"""Escape a Prometheus label value per the exposition format spec.
Station names include arbitrary Thai text (and could be set via the API),
so backslashes, double-quotes and newlines must be escaped to avoid
producing malformed or injected exposition lines.
"""
return str(value).replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n")
@staticmethod
def _metric_value(value) -> Optional[float]:
"""Coerce a numeric field to float, or None if it isn't a valid number."""
try:
return float(value)
except (TypeError, ValueError):
return None
def connect(self):
try:
import requests
# Test connection with SSL verification and timeout
response = requests.get(
f"{self.base_url}/api/v1/status/config",
timeout=10,
verify=True # Enable SSL verification for HTTPS
verify=True, # Enable SSL verification for HTTPS
)
if response.status_code == 200:
logging.info(f"Connected to VictoriaMetrics successfully at {self.base_url}")
@@ -630,38 +704,35 @@ class VictoriaMetricsAdapter(DatabaseAdapter):
timestamp_ms = int(datetime.datetime.now().timestamp() * 1000)
for measurement in measurements:
# Water level metric
metrics_data.append(
f'water_level{{station_code="{measurement["station_code"]}",'
f'station_name_en="{measurement["station_name_en"]}",'
f'station_name_th="{measurement["station_name_th"]}"}} '
f'{measurement["water_level"]} {timestamp_ms}'
# Escape label values once per measurement (untrusted Thai/English names).
labels = (
f'station_code="{self._escape_label(measurement["station_code"])}",'
f'station_name_en="{self._escape_label(measurement["station_name_en"])}",'
f'station_name_th="{self._escape_label(measurement["station_name_th"])}"'
)
# Water level metric
water_level = self._metric_value(measurement.get("water_level"))
if water_level is not None:
metrics_data.append(f"water_level{{{labels}}} {water_level} {timestamp_ms}")
# Discharge metric
metrics_data.append(
f'water_discharge{{station_code="{measurement["station_code"]}",'
f'station_name_en="{measurement["station_name_en"]}",'
f'station_name_th="{measurement["station_name_th"]}"}} '
f'{measurement["discharge"]} {timestamp_ms}'
)
discharge = self._metric_value(measurement.get("discharge"))
if discharge is not None:
metrics_data.append(f"water_discharge{{{labels}}} {discharge} {timestamp_ms}")
# Discharge percentage metric
if measurement["discharge_percent"]:
metrics_data.append(
f'water_discharge_percent{{station_code="{measurement["station_code"]}",'
f'station_name_en="{measurement["station_name_en"]}",'
f'station_name_th="{measurement["station_name_th"]}"}} '
f'{measurement["discharge_percent"]} {timestamp_ms}'
)
discharge_percent = self._metric_value(measurement.get("discharge_percent"))
if discharge_percent is not None:
metrics_data.append(f"water_discharge_percent{{{labels}}} {discharge_percent} {timestamp_ms}")
# Send to VictoriaMetrics
data = '\n'.join(metrics_data)
data = "\n".join(metrics_data)
response = requests.post(
f"{self.base_url}/api/v1/import/prometheus",
data=data,
headers={'Content-Type': 'text/plain'},
timeout=30
headers={"Content-Type": "text/plain"},
timeout=30,
)
if response.status_code == 204:
@@ -681,9 +752,12 @@ class VictoriaMetricsAdapter(DatabaseAdapter):
logging.warning("get_latest_measurements not fully implemented for VictoriaMetrics")
return []
def get_measurements_by_timerange(self, start_time: datetime.datetime,
end_time: datetime.datetime,
station_codes: Optional[List[str]] = None) -> List[Dict]:
def get_measurements_by_timerange(
self,
start_time: datetime.datetime,
end_time: datetime.datetime,
station_codes: Optional[List[str]] = None,
) -> List[Dict]:
# VictoriaMetrics range queries would be implemented here
logging.warning("get_measurements_by_timerange not fully implemented for VictoriaMetrics")
return []
@@ -693,6 +767,7 @@ class VictoriaMetricsAdapter(DatabaseAdapter):
logging.warning("get_measurements_for_date not fully implemented for VictoriaMetrics")
return []
# Factory function to create appropriate adapter
def create_database_adapter(db_type: str, **kwargs) -> DatabaseAdapter:
"""
@@ -704,15 +779,15 @@ def create_database_adapter(db_type: str, **kwargs) -> DatabaseAdapter:
"""
db_type = db_type.lower()
if db_type == 'influxdb':
if db_type == "influxdb":
return InfluxDBAdapter(**kwargs)
elif db_type == 'mysql':
return SQLAdapter(db_type='mysql', **kwargs)
elif db_type == 'postgresql':
return SQLAdapter(db_type='postgresql', **kwargs)
elif db_type == 'sqlite':
return SQLAdapter(db_type='sqlite', **kwargs)
elif db_type == 'victoriametrics':
elif db_type == "mysql":
return SQLAdapter(db_type="mysql", **kwargs)
elif db_type == "postgresql":
return SQLAdapter(db_type="postgresql", **kwargs)
elif db_type == "sqlite":
return SQLAdapter(db_type="sqlite", **kwargs)
elif db_type == "victoriametrics":
return VictoriaMetricsAdapter(**kwargs)
else:
raise ValueError(f"Unsupported database type: {db_type}")
+116 -90
View File
@@ -4,26 +4,24 @@ FastAPI web interface for water monitoring system
"""
import asyncio
import threading
from datetime import datetime, timedelta
from typing import List, Dict, Any, Optional
from contextlib import asynccontextmanager
from datetime import datetime, timedelta
from typing import Any, Dict, List, Optional
from fastapi import FastAPI, HTTPException, BackgroundTasks, Depends
from fastapi.responses import HTMLResponse, JSONResponse
from fastapi.staticfiles import StaticFiles
from fastapi import BackgroundTasks, FastAPI, HTTPException
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import HTMLResponse
from pydantic import BaseModel, Field
from .water_scraper_v3 import EnhancedWaterMonitorScraper
from .config import Config
from .models import WaterMeasurement, StationInfo, ScrapingResult, StationCreateRequest, StationUpdateRequest, StationStatus
from .health_check import HealthCheckManager, DatabaseHealthCheck, APIHealthCheck, MemoryHealthCheck
from .health_check import APIHealthCheck, DatabaseHealthCheck, HealthCheckManager, MemoryHealthCheck
from .logging_config import get_logger, setup_logging
from .metrics import get_metrics_collector, increment_counter, set_gauge
from .logging_config import setup_logging, get_logger
from .water_scraper_v3 import EnhancedWaterMonitorScraper
logger = get_logger(__name__)
# Pydantic models for API responses
class StationResponse(BaseModel):
station_id: int
@@ -35,6 +33,7 @@ class StationResponse(BaseModel):
geohash: Optional[str] = None
status: str = "active"
class StationCreateModel(BaseModel):
station_code: str = Field(..., description="Station code (e.g., P.1, P.20)")
thai_name: str = Field(..., description="Thai name of the station")
@@ -44,6 +43,7 @@ class StationCreateModel(BaseModel):
geohash: Optional[str] = Field(None, description="Geohash for the location")
status: str = Field("active", description="Station status")
class StationUpdateModel(BaseModel):
thai_name: Optional[str] = Field(None, description="Thai name of the station")
english_name: Optional[str] = Field(None, description="English name of the station")
@@ -52,6 +52,7 @@ class StationUpdateModel(BaseModel):
geohash: Optional[str] = Field(None, description="Geohash for the location")
status: Optional[str] = Field(None, description="Station status")
class MeasurementResponse(BaseModel):
timestamp: datetime
station_code: str
@@ -62,16 +63,19 @@ class MeasurementResponse(BaseModel):
discharge_percent: Optional[float] = None
status: str = "active"
class HealthResponse(BaseModel):
overall_status: str
timestamp: str
checks: Dict[str, Dict[str, Any]]
class MetricsResponse(BaseModel):
counters: Dict[str, float]
gauges: Dict[str, float]
histograms: Dict[str, Dict[str, float]]
class ScrapingStatusResponse(BaseModel):
is_running: bool
last_run: Optional[datetime] = None
@@ -80,6 +84,7 @@ class ScrapingStatusResponse(BaseModel):
successful_runs: int = 0
failed_runs: int = 0
# Global application state
app_state = {
"scraper": None,
@@ -91,10 +96,11 @@ app_state = {
"successful_runs": 0,
"failed_runs": 0,
"last_run": None,
"next_run": None
}
"next_run": None,
},
}
@asynccontextmanager
async def lifespan(app: FastAPI):
"""Application lifespan manager"""
@@ -139,23 +145,30 @@ async def lifespan(app: FastAPI):
logger.info("Water Monitor API shutdown complete")
# Create FastAPI app
app = FastAPI(
title="Northern Thailand Ping River Monitor API",
description="Real-time water level monitoring system for Northern Thailand's Ping River Basin stations",
version="3.1.3",
lifespan=lifespan
lifespan=lifespan,
)
# Add CORS middleware
# Add CORS middleware.
# Origins come from CORS_ALLOW_ORIGINS (comma-separated). When none are configured
# we fall back to a wildcard WITHOUT credentials (a safe, spec-valid combination);
# credentials are only enabled when explicit origins are provided.
_cors_origins = Config.CORS_ALLOW_ORIGINS or ["*"]
_cors_allow_credentials = bool(Config.CORS_ALLOW_ORIGINS)
app.add_middleware(
CORSMiddleware,
allow_origins=["*"], # Configure appropriately for production
allow_credentials=True,
allow_origins=_cors_origins,
allow_credentials=_cors_allow_credentials,
allow_methods=["*"],
allow_headers=["*"],
)
async def background_scraping_task():
"""Background task for periodic data scraping"""
while True:
@@ -170,7 +183,9 @@ async def background_scraping_task():
start_time = datetime.now()
try:
result = scraper.run_scraping_cycle()
# run_scraping_cycle() does blocking network/DB I/O and time.sleep
# retries; run it in a thread so it doesn't freeze the event loop.
result = await asyncio.get_event_loop().run_in_executor(None, scraper.run_scraping_cycle)
# Update stats
app_state["scraping_stats"]["total_runs"] += 1
@@ -209,8 +224,10 @@ async def background_scraping_task():
logger.error(f"Error in background scraping task: {e}")
await asyncio.sleep(60) # Wait a minute before retrying
# API Routes
@app.get("/", response_class=HTMLResponse)
async def root():
"""Root endpoint with basic dashboard"""
@@ -268,6 +285,7 @@ async def root():
"""
return HTMLResponse(content=html_content)
@app.get("/health", response_model=HealthResponse)
async def get_health():
"""Get system health status"""
@@ -277,12 +295,13 @@ async def get_health():
if not health_manager:
raise HTTPException(status_code=503, detail="Health manager not initialized")
# Run health checks
results = health_manager.run_all_checks()
# Run health checks (populates state read by get_health_summary)
health_manager.run_all_checks()
summary = health_manager.get_health_summary()
return HealthResponse(**summary)
@app.get("/metrics", response_model=MetricsResponse)
async def get_metrics():
"""Get application metrics"""
@@ -293,6 +312,7 @@ async def get_metrics():
return MetricsResponse(**metrics)
@app.get("/stations", response_model=List[StationResponse])
async def get_stations():
"""Get list of all monitoring stations"""
@@ -304,18 +324,21 @@ async def get_stations():
stations = []
for station_id, station_info in scraper.station_mapping.items():
stations.append(StationResponse(
station_id=int(station_id),
station_code=station_info["code"],
thai_name=station_info["thai_name"],
english_name=station_info["english_name"],
latitude=station_info.get("latitude"),
longitude=station_info.get("longitude"),
status="active"
))
stations.append(
StationResponse(
station_id=int(station_id),
station_code=station_info["code"],
thai_name=station_info["thai_name"],
english_name=station_info["english_name"],
latitude=station_info.get("latitude"),
longitude=station_info.get("longitude"),
status="active",
)
)
return stations
@app.post("/stations", response_model=StationResponse)
async def create_station(station: StationCreateModel):
"""Create a new monitoring station"""
@@ -332,12 +355,12 @@ async def create_station(station: StationCreateModel):
# Add to station mapping
scraper.station_mapping[str(new_station_id)] = {
'code': station.station_code,
'thai_name': station.thai_name,
'english_name': station.english_name,
'latitude': station.latitude,
'longitude': station.longitude,
'geohash': station.geohash
"code": station.station_code,
"thai_name": station.thai_name,
"english_name": station.english_name,
"latitude": station.latitude,
"longitude": station.longitude,
"geohash": station.geohash,
}
logger.info(f"Created new station: {station.station_code} ({station.english_name})")
@@ -350,13 +373,14 @@ async def create_station(station: StationCreateModel):
latitude=station.latitude,
longitude=station.longitude,
geohash=station.geohash,
status=station.status
status=station.status,
)
except Exception as e:
logger.error(f"Error creating station: {e}")
raise HTTPException(status_code=500, detail=str(e))
@app.put("/stations/{station_id}", response_model=StationResponse)
async def update_station(station_id: int, updates: StationUpdateModel):
"""Update an existing monitoring station"""
@@ -375,33 +399,34 @@ async def update_station(station_id: int, updates: StationUpdateModel):
# Update fields if provided
if updates.thai_name is not None:
station_info['thai_name'] = updates.thai_name
station_info["thai_name"] = updates.thai_name
if updates.english_name is not None:
station_info['english_name'] = updates.english_name
station_info["english_name"] = updates.english_name
if updates.latitude is not None:
station_info['latitude'] = updates.latitude
station_info["latitude"] = updates.latitude
if updates.longitude is not None:
station_info['longitude'] = updates.longitude
station_info["longitude"] = updates.longitude
if updates.geohash is not None:
station_info['geohash'] = updates.geohash
station_info["geohash"] = updates.geohash
logger.info(f"Updated station {station_id}: {station_info['code']}")
return StationResponse(
station_id=station_id,
station_code=station_info['code'],
thai_name=station_info['thai_name'],
english_name=station_info['english_name'],
latitude=station_info.get('latitude'),
longitude=station_info.get('longitude'),
geohash=station_info.get('geohash'),
status=updates.status or "active"
station_code=station_info["code"],
thai_name=station_info["thai_name"],
english_name=station_info["english_name"],
latitude=station_info.get("latitude"),
longitude=station_info.get("longitude"),
geohash=station_info.get("geohash"),
status=updates.status or "active",
)
except Exception as e:
logger.error(f"Error updating station {station_id}: {e}")
raise HTTPException(status_code=500, detail=str(e))
@app.delete("/stations/{station_id}")
async def delete_station(station_id: int):
"""Delete a monitoring station"""
@@ -425,6 +450,7 @@ async def delete_station(station_id: int):
logger.error(f"Error deleting station {station_id}: {e}")
raise HTTPException(status_code=500, detail=str(e))
@app.get("/stations/{station_id}", response_model=StationResponse)
async def get_station(station_id: int):
"""Get details of a specific monitoring station"""
@@ -442,15 +468,16 @@ async def get_station(station_id: int):
return StationResponse(
station_id=station_id,
station_code=station_info['code'],
thai_name=station_info['thai_name'],
english_name=station_info['english_name'],
latitude=station_info.get('latitude'),
longitude=station_info.get('longitude'),
geohash=station_info.get('geohash'),
status="active"
station_code=station_info["code"],
thai_name=station_info["thai_name"],
english_name=station_info["english_name"],
latitude=station_info.get("latitude"),
longitude=station_info.get("longitude"),
geohash=station_info.get("geohash"),
status="active",
)
@app.get("/measurements/latest", response_model=List[MeasurementResponse])
async def get_latest_measurements(limit: int = 100):
"""Get latest measurements from all stations"""
@@ -465,16 +492,18 @@ async def get_latest_measurements(limit: int = 100):
response = []
for measurement in measurements:
response.append(MeasurementResponse(
timestamp=measurement["timestamp"],
station_code=measurement["station_code"],
station_name_en=measurement["station_name_en"],
station_name_th=measurement["station_name_th"],
water_level=measurement["water_level"],
discharge=measurement["discharge"],
discharge_percent=measurement.get("discharge_percent"),
status=measurement.get("status", "active")
))
response.append(
MeasurementResponse(
timestamp=measurement["timestamp"],
station_code=measurement["station_code"],
station_name_en=measurement["station_name_en"],
station_name_th=measurement["station_name_th"],
water_level=measurement["water_level"],
discharge=measurement["discharge"],
discharge_percent=measurement.get("discharge_percent"),
status=measurement.get("status", "active"),
)
)
return response
@@ -482,12 +511,9 @@ async def get_latest_measurements(limit: int = 100):
logger.error(f"Error fetching latest measurements: {e}")
raise HTTPException(status_code=500, detail=str(e))
@app.get("/measurements/station/{station_code}", response_model=List[MeasurementResponse])
async def get_station_measurements(
station_code: str,
hours: int = 24,
limit: int = 1000
):
async def get_station_measurements(station_code: str, hours: int = 24, limit: int = 1000):
"""Get measurements for a specific station"""
increment_counter("api_requests", labels={"endpoint": "measurements_station"})
@@ -509,16 +535,18 @@ async def get_station_measurements(
response = []
for measurement in measurements:
response.append(MeasurementResponse(
timestamp=measurement["timestamp"],
station_code=measurement["station_code"],
station_name_en=measurement["station_name_en"],
station_name_th=measurement["station_name_th"],
water_level=measurement["water_level"],
discharge=measurement["discharge"],
discharge_percent=measurement.get("discharge_percent"),
status=measurement.get("status", "active")
))
response.append(
MeasurementResponse(
timestamp=measurement["timestamp"],
station_code=measurement["station_code"],
station_name_en=measurement["station_name_en"],
station_name_th=measurement["station_name_th"],
water_level=measurement["water_level"],
discharge=measurement["discharge"],
discharge_percent=measurement.get("discharge_percent"),
status=measurement.get("status", "active"),
)
)
return response
@@ -526,6 +554,7 @@ async def get_station_measurements(
logger.error(f"Error fetching station measurements: {e}")
raise HTTPException(status_code=500, detail=str(e))
@app.post("/scrape/trigger")
async def trigger_scraping(background_tasks: BackgroundTasks):
"""Trigger manual data scraping"""
@@ -568,6 +597,7 @@ async def trigger_scraping(background_tasks: BackgroundTasks):
return {"message": "Scraping triggered", "status": "started"}
@app.get("/scraping/status", response_model=ScrapingStatusResponse)
async def get_scraping_status():
"""Get current scraping status"""
@@ -581,9 +611,10 @@ async def get_scraping_status():
next_run=stats["next_run"],
total_runs=stats["total_runs"],
successful_runs=stats["successful_runs"],
failed_runs=stats["failed_runs"]
failed_runs=stats["failed_runs"],
)
@app.get("/config")
async def get_config():
"""Get current configuration (sensitive data masked)"""
@@ -593,12 +624,13 @@ async def get_config():
# Mask sensitive information
for key in config:
if 'password' in key.lower() or 'secret' in key.lower():
if "password" in key.lower() or "secret" in key.lower():
if config[key]:
config[key] = '*' * 8
config[key] = "*" * 8
return config
if __name__ == "__main__":
import uvicorn
@@ -607,14 +639,8 @@ if __name__ == "__main__":
log_level=Config.LOG_LEVEL,
log_file=Config.LOG_FILE,
enable_console=True,
enable_colors=True
enable_colors=True,
)
# Run the API server
uvicorn.run(
"web_api:app",
host="0.0.0.0",
port=8000,
reload=False,
log_config=None # Use our custom logging
)
uvicorn.run("web_api:app", host="0.0.0.0", port=8000, reload=False, log_config=None) # Use our custom logging