B4L/Northern-Thailand-Ping-River-Monitor
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 15 Wiki Activity

Fix Matrix message formatting and harden security

Browse Source 
Matrix alerts:
- Send HTML formatted_body (org.matrix.custom.html) so **bold** and URLs
  render instead of showing literal Markdown; add plain-text body fallback.
  Add dependency-free markdown_to_matrix_html/strip_markdown helpers with
  HTML escaping of station/message data.

Security:
- InfluxDB: bind untrusted station_codes as query params and cast limit to
  int (was f-string interpolation / injection risk).
- VictoriaMetrics: escape Prometheus label values and coerce metric values
  to float, preventing exposition-format injection and None crashes.
- web_api: run blocking scrape cycle via run_in_executor so it no longer
  freezes the event loop; make CORS origins configurable and only allow
  credentials with explicit origins ("*" + credentials is invalid/unsafe).
- config: remove hardcoded root/postgres password fallbacks (raise instead)
  and stop defaulting VM_HOST to a real infrastructure hostname.

Also remove unused imports and wrap long lines to satisfy flake8.
This commit is contained in:
grabowski
2026-07-22 12:07:05 +07:00
parent d3ec5a77e6
commit f4c63cabef
4 changed files with 688 additions and 525 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/config.py
+121 -109
View File
@@ -1,9 +1,10 @@
import os
from typing import Dict, Any, Optional
from typing import Any, Dict
# Load environment variables from .env file
try:
from dotenv import load_dotenv
load_dotenv()
except ImportError:
# python-dotenv not installed, continue without it
@@ -11,14 +12,14 @@ except ImportError:
try:
from .exceptions import ConfigurationError
from .models import DatabaseType, DatabaseConfig
from .models import DatabaseType
except ImportError:
# Handle case when running as standalone script
class ConfigurationError(Exception):
pass
from enum import Enum
class DatabaseType(Enum):
SQLITE = "sqlite"
MYSQL = "mysql"
@@ -26,87 +27,100 @@ except ImportError:
INFLUXDB = "influxdb"
VICTORIAMETRICS = "victoriametrics"
class Config:
"""Configuration class for the Water Level Monitor"""
# Database settings
DATABASE_PATH = os.getenv('WATER_DB_PATH', 'water_levels.db')
DATABASE_PATH = os.getenv("WATER_DB_PATH", "water_levels.db")
# Website settings
TARGET_URL = "https://hyd-app-db.rid.go.th/hydro1h.html"
API_URL = "https://hyd-app-db.rid.go.th/webservice/getGroupHourlyWaterLevelReportAllHL.ashx"
REQUEST_TIMEOUT = int(os.getenv('REQUEST_TIMEOUT', '30'))
USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
REQUEST_TIMEOUT = int(os.getenv("REQUEST_TIMEOUT", "30"))
USER_AGENT = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
)
# Database configuration
DB_TYPE = os.getenv('DB_TYPE', 'sqlite').lower()
DB_TYPE = os.getenv("DB_TYPE", "sqlite").lower()
# VictoriaMetrics settings
VM_HOST = os.getenv('VM_HOST', 'vm.newedge.house')
VM_PORT = int(os.getenv('VM_PORT', '443'))
# Default to localhost; set VM_HOST in the environment for real deployments
# (avoids committing infrastructure hostnames to the repo).
VM_HOST = os.getenv("VM_HOST", "localhost")
VM_PORT = int(os.getenv("VM_PORT", "443"))
# Support for HTTPS URLs (e.g., behind reverse proxy)
VM_URL = os.getenv('VM_URL') # Full URL override (e.g., https://vm.example.com)
VM_URL = os.getenv("VM_URL") # Full URL override (e.g., https://vm.example.com)
# InfluxDB settings
INFLUX_HOST = os.getenv('INFLUX_HOST', 'localhost')
INFLUX_PORT = int(os.getenv('INFLUX_PORT', '8086'))
INFLUX_DATABASE = os.getenv('INFLUX_DATABASE', 'water_monitoring')
INFLUX_USERNAME = os.getenv('INFLUX_USERNAME')
INFLUX_PASSWORD = os.getenv('INFLUX_PASSWORD')
INFLUX_HOST = os.getenv("INFLUX_HOST", "localhost")
INFLUX_PORT = int(os.getenv("INFLUX_PORT", "8086"))
INFLUX_DATABASE = os.getenv("INFLUX_DATABASE", "water_monitoring")
INFLUX_USERNAME = os.getenv("INFLUX_USERNAME")
INFLUX_PASSWORD = os.getenv("INFLUX_PASSWORD")
# PostgreSQL settings
POSTGRES_CONNECTION_STRING = os.getenv('POSTGRES_CONNECTION_STRING')
POSTGRES_HOST = os.getenv('POSTGRES_HOST', 'localhost')
POSTGRES_PORT = int(os.getenv('POSTGRES_PORT', '5432'))
POSTGRES_DB = os.getenv('POSTGRES_DB', 'water_monitoring')
POSTGRES_USER = os.getenv('POSTGRES_USER', 'postgres')
POSTGRES_PASSWORD = os.getenv('POSTGRES_PASSWORD')
POSTGRES_CONNECTION_STRING = os.getenv("POSTGRES_CONNECTION_STRING")
POSTGRES_HOST = os.getenv("POSTGRES_HOST", "localhost")
POSTGRES_PORT = int(os.getenv("POSTGRES_PORT", "5432"))
POSTGRES_DB = os.getenv("POSTGRES_DB", "water_monitoring")
POSTGRES_USER = os.getenv("POSTGRES_USER", "postgres")
POSTGRES_PASSWORD = os.getenv("POSTGRES_PASSWORD")
# MySQL settings
MYSQL_CONNECTION_STRING = os.getenv('MYSQL_CONNECTION_STRING')
MYSQL_CONNECTION_STRING = os.getenv("MYSQL_CONNECTION_STRING")
# Scheduler settings
SCRAPING_INTERVAL_HOURS = int(os.getenv('SCRAPING_INTERVAL_HOURS', '1'))
SCRAPING_INTERVAL_HOURS = int(os.getenv("SCRAPING_INTERVAL_HOURS", "1"))
# Logging settings
LOG_LEVEL = os.getenv('LOG_LEVEL', 'INFO')
LOG_FILE = os.getenv('LOG_FILE', 'water_monitor.log')
LOG_FORMAT = '%(asctime)s - %(levelname)s - %(message)s'
LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO")
LOG_FILE = os.getenv("LOG_FILE", "water_monitor.log")
LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s"
# Data retention
DATA_RETENTION_DAYS = int(os.getenv('DATA_RETENTION_DAYS', '365'))
DATA_RETENTION_DAYS = int(os.getenv("DATA_RETENTION_DAYS", "365"))
# Retry settings
MAX_RETRIES = int(os.getenv('MAX_RETRIES', '3'))
RETRY_DELAY_SECONDS = int(os.getenv('RETRY_DELAY_SECONDS', '60'))
MAX_RETRIES = int(os.getenv("MAX_RETRIES", "3"))
RETRY_DELAY_SECONDS = int(os.getenv("RETRY_DELAY_SECONDS", "60"))
# Web API / CORS settings
# Comma-separated list of allowed origins. Defaults to none (same-origin only);
# set CORS_ALLOW_ORIGINS to a specific list of front-end origins in production.
# Credentials are only enabled when explicit (non-wildcard) origins are set,
# because "*" + credentials is rejected by browsers and unsafe.
CORS_ALLOW_ORIGINS = [origin.strip() for origin in os.getenv("CORS_ALLOW_ORIGINS", "").split(",") if origin.strip()]
@classmethod
def validate_config(cls) -> bool:
"""Validate configuration settings"""
errors = []
# Validate database type
try:
DatabaseType(cls.DB_TYPE)
except ValueError:
errors.append(f"Invalid DB_TYPE: {cls.DB_TYPE}")
# Validate database-specific settings
if cls.DB_TYPE == 'victoriametrics':
if cls.DB_TYPE == "victoriametrics":
if not cls.VM_HOST:
errors.append("VM_HOST is required for VictoriaMetrics")
if not isinstance(cls.VM_PORT, int) or cls.VM_PORT <= 0:
errors.append("VM_PORT must be a positive integer")
elif cls.DB_TYPE == 'influxdb':
elif cls.DB_TYPE == "influxdb":
if not cls.INFLUX_HOST:
errors.append("INFLUX_HOST is required for InfluxDB")
if not cls.INFLUX_DATABASE:
errors.append("INFLUX_DATABASE is required for InfluxDB")
elif cls.DB_TYPE in ['postgresql', 'mysql']:
if cls.DB_TYPE == 'postgresql':
elif cls.DB_TYPE in ["postgresql", "mysql"]:
if cls.DB_TYPE == "postgresql":
# Check if either connection string or individual components are provided
if not cls.POSTGRES_CONNECTION_STRING:
# If no connection string, check individual components
@@ -121,105 +135,103 @@ class Config:
else: # mysql
if not cls.MYSQL_CONNECTION_STRING:
errors.append("MYSQL_CONNECTION_STRING is required for MySQL")
# Validate numeric settings
if cls.SCRAPING_INTERVAL_HOURS <= 0:
errors.append("SCRAPING_INTERVAL_HOURS must be positive")
if cls.DATA_RETENTION_DAYS <= 0:
errors.append("DATA_RETENTION_DAYS must be positive")
if errors:
raise ConfigurationError(f"Configuration errors: {'; '.join(errors)}")
return True
@classmethod
def get_database_config(cls) -> Dict[str, Any]:
"""Returns database configuration based on DB_TYPE"""
if cls.DB_TYPE == 'victoriametrics':
if cls.DB_TYPE == "victoriametrics":
return {"type": "victoriametrics", "host": cls.VM_HOST, "port": cls.VM_PORT}
elif cls.DB_TYPE == "influxdb":
return {
'type': 'victoriametrics',
'host': cls.VM_HOST,
'port': cls.VM_PORT
"type": "influxdb",
"host": cls.INFLUX_HOST,
"port": cls.INFLUX_PORT,
"database": cls.INFLUX_DATABASE,
"username": cls.INFLUX_USERNAME,
"password": cls.INFLUX_PASSWORD,
}
elif cls.DB_TYPE == 'influxdb':
return {
'type': 'influxdb',
'host': cls.INFLUX_HOST,
'port': cls.INFLUX_PORT,
'database': cls.INFLUX_DATABASE,
'username': cls.INFLUX_USERNAME,
'password': cls.INFLUX_PASSWORD
}
elif cls.DB_TYPE == 'postgresql':
elif cls.DB_TYPE == "postgresql":
# Use individual components if POSTGRES_CONNECTION_STRING is not provided
if cls.POSTGRES_CONNECTION_STRING:
return {
'type': 'postgresql',
'connection_string': cls.POSTGRES_CONNECTION_STRING
"type": "postgresql",
"connection_string": cls.POSTGRES_CONNECTION_STRING,
}
else:
# Build connection string from components (automatically URL-encodes password)
import urllib.parse
password = urllib.parse.quote(cls.POSTGRES_PASSWORD or 'password', safe='')
connection_string = f'postgresql://{cls.POSTGRES_USER}:{password}@{cls.POSTGRES_HOST}:{cls.POSTGRES_PORT}/{cls.POSTGRES_DB}'
return {
'type': 'postgresql',
'connection_string': connection_string
}
elif cls.DB_TYPE == 'mysql':
return {
'type': 'mysql',
'connection_string': cls.MYSQL_CONNECTION_STRING or
'mysql://root:password@localhost:3306/water_monitoring'
}
if not cls.POSTGRES_PASSWORD:
raise ConfigurationError("POSTGRES_PASSWORD is required for PostgreSQL (no default is provided)")
password = urllib.parse.quote(cls.POSTGRES_PASSWORD, safe="")
connection_string = (
f"postgresql://{cls.POSTGRES_USER}:{password}"
f"@{cls.POSTGRES_HOST}:{cls.POSTGRES_PORT}/{cls.POSTGRES_DB}"
)
return {"type": "postgresql", "connection_string": connection_string}
elif cls.DB_TYPE == "mysql":
if not cls.MYSQL_CONNECTION_STRING:
raise ConfigurationError("MYSQL_CONNECTION_STRING is required for MySQL (no default is provided)")
return {"type": "mysql", "connection_string": cls.MYSQL_CONNECTION_STRING}
else: # sqlite
return {
'type': 'sqlite',
'connection_string': f'sqlite:///{cls.DATABASE_PATH}'
"type": "sqlite",
"connection_string": f"sqlite:///{cls.DATABASE_PATH}",
}
@classmethod
def get_all_settings(cls) -> Dict[str, Any]:
"""Returns all configuration settings"""
return {
'DB_TYPE': cls.DB_TYPE,
'DATABASE_PATH': cls.DATABASE_PATH,
'TARGET_URL': cls.TARGET_URL,
'API_URL': cls.API_URL,
'REQUEST_TIMEOUT': cls.REQUEST_TIMEOUT,
'SCRAPING_INTERVAL_HOURS': cls.SCRAPING_INTERVAL_HOURS,
'LOG_LEVEL': cls.LOG_LEVEL,
'LOG_FILE': cls.LOG_FILE,
'DATA_RETENTION_DAYS': cls.DATA_RETENTION_DAYS,
'MAX_RETRIES': cls.MAX_RETRIES,
'RETRY_DELAY_SECONDS': cls.RETRY_DELAY_SECONDS,
'VM_HOST': cls.VM_HOST,
'VM_PORT': cls.VM_PORT,
'INFLUX_HOST': cls.INFLUX_HOST,
'INFLUX_PORT': cls.INFLUX_PORT,
'INFLUX_DATABASE': cls.INFLUX_DATABASE
"DB_TYPE": cls.DB_TYPE,
"DATABASE_PATH": cls.DATABASE_PATH,
"TARGET_URL": cls.TARGET_URL,
"API_URL": cls.API_URL,
"REQUEST_TIMEOUT": cls.REQUEST_TIMEOUT,
"SCRAPING_INTERVAL_HOURS": cls.SCRAPING_INTERVAL_HOURS,
"LOG_LEVEL": cls.LOG_LEVEL,
"LOG_FILE": cls.LOG_FILE,
"DATA_RETENTION_DAYS": cls.DATA_RETENTION_DAYS,
"MAX_RETRIES": cls.MAX_RETRIES,
"RETRY_DELAY_SECONDS": cls.RETRY_DELAY_SECONDS,
"VM_HOST": cls.VM_HOST,
"VM_PORT": cls.VM_PORT,
"INFLUX_HOST": cls.INFLUX_HOST,
"INFLUX_PORT": cls.INFLUX_PORT,
"INFLUX_DATABASE": cls.INFLUX_DATABASE,
}
@classmethod
def print_settings(cls):
"""Prints all current settings"""
print("=== Water Level Monitor Configuration ===")
for key, value in cls.get_all_settings().items():
# Hide sensitive information
if 'PASSWORD' in key and value:
value = '*' * len(str(value))
if "PASSWORD" in key and value:
value = "*" * len(str(value))
print(f"{key}: {value}")
print("=" * 45)
print("\nDatabase Configuration:")
db_config = cls.get_database_config()
for key, value in db_config.items():
if 'password' in key and value:
value = '*' * len(str(value))
if "password" in key and value:
value = "*" * len(str(value))
print(f" {key}: {value}")
print("=" * 45)
if __name__ == "__main__":
Config.print_settings()