Initial commit: Northern Thailand Ping River Monitor v3.1.0
Security & Dependency Updates / Dependency Security Scan (push) Successful in 29s
Security & Dependency Updates / Docker Security Scan (push) Failing after 53s
Security & Dependency Updates / License Compliance (push) Successful in 13s
Security & Dependency Updates / Check for Dependency Updates (push) Successful in 19s
Security & Dependency Updates / Code Quality Metrics (push) Successful in 11s
Security & Dependency Updates / Security Summary (push) Successful in 7s

Features:
- Real-time water level monitoring for Ping River Basin (16 stations)
- Coverage from Chiang Dao to Nakhon Sawan in Northern Thailand
- FastAPI web interface with interactive dashboard and station management
- Multi-database support (SQLite, MySQL, PostgreSQL, InfluxDB, VictoriaMetrics)
- Comprehensive monitoring with health checks and metrics collection
- Docker deployment with Grafana integration
- Production-ready architecture with enterprise-grade observability

 CI/CD & Automation:
- Complete Gitea Actions workflows for CI/CD, security, and releases
- Multi-Python version testing (3.9-3.12)
- Multi-architecture Docker builds (amd64, arm64)
- Daily security scanning and dependency monitoring
- Automated documentation generation
- Performance testing and validation

 Production Ready:
- Type safety with Pydantic models and comprehensive type hints
- Data validation layer with range checking and error handling
- Rate limiting and request tracking for API protection
- Enhanced logging with rotation, colors, and performance metrics
- Station management API for dynamic CRUD operations
- Comprehensive documentation and deployment guides

 Technical Stack:
- Python 3.9+ with FastAPI and Pydantic
- Multi-database architecture with adapter pattern
- Docker containerization with multi-stage builds
- Grafana dashboards for visualization
- Gitea Actions for CI/CD automation
- Enterprise monitoring and alerting

 Ready for deployment to B4L infrastructure!
This commit is contained in:
2025-08-12 15:40:24 +07:00
commit af62cfef0b
60 changed files with 13267 additions and 0 deletions
+663
View File
@@ -0,0 +1,663 @@
#!/usr/bin/env python3
"""
Database adapters for different storage backends
"""
import datetime
import logging
from typing import List, Dict, Optional, Any
from abc import ABC, abstractmethod
# Base adapter interface
class DatabaseAdapter(ABC):
@abstractmethod
def connect(self):
pass
@abstractmethod
def save_measurements(self, measurements: List[Dict]) -> bool:
pass
@abstractmethod
def get_latest_measurements(self, limit: int = 100) -> List[Dict]:
pass
@abstractmethod
def get_measurements_by_timerange(self, start_time: datetime.datetime,
end_time: datetime.datetime,
station_codes: Optional[List[str]] = None) -> List[Dict]:
pass
# InfluxDB Adapter
class InfluxDBAdapter(DatabaseAdapter):
def __init__(self, host: str = "localhost", port: int = 8086,
database: str = "water_monitoring", username: str = None, password: str = None):
self.host = host
self.port = port
self.database = database
self.username = username
self.password = password
self.client = None
def connect(self):
try:
from influxdb import InfluxDBClient
self.client = InfluxDBClient(
host=self.host,
port=self.port,
username=self.username,
password=self.password,
database=self.database
)
# Create database if it doesn't exist
databases = self.client.get_list_database()
if not any(db['name'] == self.database for db in databases):
self.client.create_database(self.database)
logging.info(f"Created InfluxDB database: {self.database}")
# Create retention policy (keep data for 2 years, downsample after 30 days)
retention_policies = self.client.get_list_retention_policies(self.database)
if not any(rp['name'] == 'water_data_policy' for rp in retention_policies):
self.client.create_retention_policy(
'water_data_policy',
'730d', # 2 years
'1', # replication factor
database=self.database,
default=True
)
logging.info("Connected to InfluxDB successfully")
return True
except ImportError:
logging.error("InfluxDB client not installed. Run: pip install influxdb")
return False
except Exception as e:
logging.error(f"Failed to connect to InfluxDB: {e}")
return False
def save_measurements(self, measurements: List[Dict]) -> bool:
if not self.client:
logging.error("InfluxDB client not connected")
return False
try:
points = []
for measurement in measurements:
point = {
"measurement": "water_data",
"tags": {
"station_code": measurement['station_code'],
"station_name_en": measurement['station_name_en'],
"station_name_th": measurement['station_name_th']
},
"time": measurement['timestamp'].isoformat(),
"fields": {
"water_level": float(measurement['water_level']),
"discharge": float(measurement['discharge']),
"discharge_percent": float(measurement['discharge_percent']) if measurement['discharge_percent'] else None
}
}
points.append(point)
success = self.client.write_points(points)
if success:
logging.info(f"Successfully wrote {len(points)} points to InfluxDB")
return success
except Exception as e:
logging.error(f"Error writing to InfluxDB: {e}")
return False
def get_latest_measurements(self, limit: int = 100) -> List[Dict]:
if not self.client:
return []
try:
query = f"""
SELECT last("water_level") as water_level,
last("discharge") as discharge,
last("discharge_percent") as discharge_percent
FROM "water_data"
GROUP BY "station_code", "station_name_en", "station_name_th"
LIMIT {limit}
"""
result = self.client.query(query)
measurements = []
for point in result.get_points():
measurements.append({
'timestamp': point['time'],
'station_code': point.get('station_code'),
'station_name_en': point.get('station_name_en'),
'station_name_th': point.get('station_name_th'),
'water_level': point.get('water_level'),
'discharge': point.get('discharge'),
'discharge_percent': point.get('discharge_percent')
})
return measurements
except Exception as e:
logging.error(f"Error querying InfluxDB: {e}")
return []
def get_measurements_by_timerange(self, start_time: datetime.datetime,
end_time: datetime.datetime,
station_codes: Optional[List[str]] = None) -> List[Dict]:
if not self.client:
return []
try:
where_clause = f"time >= '{start_time.isoformat()}' AND time <= '{end_time.isoformat()}'"
if station_codes:
station_filter = "'" + "','".join(station_codes) + "'"
where_clause += f" AND station_code IN ({station_filter})"
query = f"""
SELECT "water_level", "discharge", "discharge_percent", "station_code", "station_name_en", "station_name_th"
FROM "water_data"
WHERE {where_clause}
ORDER BY time DESC
"""
result = self.client.query(query)
measurements = []
for point in result.get_points():
measurements.append({
'timestamp': point['time'],
'station_code': point.get('station_code'),
'station_name_en': point.get('station_name_en'),
'station_name_th': point.get('station_name_th'),
'water_level': point.get('water_level'),
'discharge': point.get('discharge'),
'discharge_percent': point.get('discharge_percent')
})
return measurements
except Exception as e:
logging.error(f"Error querying InfluxDB: {e}")
return []
# MySQL/PostgreSQL Adapter
class SQLAdapter(DatabaseAdapter):
def __init__(self, connection_string: str, db_type: str = "mysql"):
self.connection_string = connection_string
self.db_type = db_type.lower()
self.engine = None
# Add SQLite-specific connection parameters for better concurrency
if self.db_type == "sqlite":
if "?" not in connection_string:
self.connection_string += "?timeout=30&check_same_thread=False"
else:
self.connection_string += "&timeout=30&check_same_thread=False"
def connect(self):
try:
from sqlalchemy import create_engine, text
from sqlalchemy.orm import sessionmaker
self.engine = create_engine(self.connection_string, pool_pre_ping=True)
# Create tables
self._create_tables()
logging.info(f"Connected to {self.db_type.upper()} successfully")
return True
except ImportError:
logging.error("SQLAlchemy not installed. Run: pip install sqlalchemy pymysql")
return False
except Exception as e:
logging.error(f"Failed to connect to {self.db_type.upper()}: {e}")
return False
def _create_tables(self):
from sqlalchemy import text
# Stations table - adjust for different databases
if self.db_type == "sqlite":
stations_sql = """
CREATE TABLE IF NOT EXISTS stations (
id INTEGER PRIMARY KEY,
station_code TEXT UNIQUE NOT NULL,
thai_name TEXT NOT NULL,
english_name TEXT NOT NULL,
latitude REAL,
longitude REAL,
geohash TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
"""
measurements_sql = """
CREATE TABLE IF NOT EXISTS water_measurements (
id INTEGER PRIMARY KEY AUTOINCREMENT,
timestamp DATETIME NOT NULL,
station_id INTEGER NOT NULL,
water_level REAL,
discharge REAL,
discharge_percent REAL,
status TEXT DEFAULT 'active',
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (station_id) REFERENCES stations(id),
UNIQUE(timestamp, station_id)
)
"""
# Create indexes separately for SQLite
index_sql = [
"CREATE INDEX IF NOT EXISTS idx_timestamp ON water_measurements(timestamp)",
"CREATE INDEX IF NOT EXISTS idx_station_timestamp ON water_measurements(station_id, timestamp)"
]
elif self.db_type == "postgresql":
stations_sql = """
CREATE TABLE IF NOT EXISTS stations (
id SERIAL PRIMARY KEY,
station_code VARCHAR(10) UNIQUE NOT NULL,
thai_name VARCHAR(255) NOT NULL,
english_name VARCHAR(255) NOT NULL,
latitude DECIMAL(10,8),
longitude DECIMAL(11,8),
geohash VARCHAR(20),
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
"""
measurements_sql = """
CREATE TABLE IF NOT EXISTS water_measurements (
id BIGSERIAL PRIMARY KEY,
timestamp TIMESTAMP NOT NULL,
station_id INTEGER NOT NULL,
water_level NUMERIC(10,3),
discharge NUMERIC(10,2),
discharge_percent NUMERIC(5,2),
status VARCHAR(20) DEFAULT 'active',
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (station_id) REFERENCES stations(id),
UNIQUE(timestamp, station_id)
)
"""
index_sql = [
"CREATE INDEX IF NOT EXISTS idx_timestamp ON water_measurements(timestamp)",
"CREATE INDEX IF NOT EXISTS idx_station_timestamp ON water_measurements(station_id, timestamp DESC)"
]
else: # MySQL
stations_sql = """
CREATE TABLE IF NOT EXISTS stations (
id INT PRIMARY KEY,
station_code VARCHAR(10) UNIQUE NOT NULL,
thai_name VARCHAR(255) NOT NULL,
english_name VARCHAR(255) NOT NULL,
latitude DECIMAL(10,8),
longitude DECIMAL(11,8),
geohash VARCHAR(20),
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
)
"""
measurements_sql = """
CREATE TABLE IF NOT EXISTS water_measurements (
id BIGINT AUTO_INCREMENT PRIMARY KEY,
timestamp DATETIME NOT NULL,
station_id INT NOT NULL,
water_level DECIMAL(10,3),
discharge DECIMAL(10,2),
discharge_percent DECIMAL(5,2),
status VARCHAR(20) DEFAULT 'active',
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (station_id) REFERENCES stations(id),
UNIQUE KEY unique_measurement (timestamp, station_id),
INDEX idx_timestamp (timestamp),
INDEX idx_station_timestamp (station_id, timestamp)
)
"""
index_sql = []
with self.engine.begin() as conn:
conn.execute(text(stations_sql))
conn.execute(text(measurements_sql))
# Create indexes for SQLite and PostgreSQL
for index in index_sql:
conn.execute(text(index))
# Transaction is automatically committed when context manager exits
def save_measurements(self, measurements: List[Dict]) -> bool:
if not self.engine:
return False
try:
from sqlalchemy import text
with self.engine.begin() as conn:
# Insert/update stations
for measurement in measurements:
if self.db_type == "sqlite":
station_sql = """
INSERT OR REPLACE INTO stations (id, station_code, thai_name, english_name, latitude, longitude, geohash, updated_at)
VALUES (:station_id, :station_code, :thai_name, :english_name, :latitude, :longitude, :geohash, CURRENT_TIMESTAMP)
"""
elif self.db_type == "postgresql":
station_sql = """
INSERT INTO stations (id, station_code, thai_name, english_name, latitude, longitude, geohash, updated_at)
VALUES (:station_id, :station_code, :thai_name, :english_name, :latitude, :longitude, :geohash, NOW())
ON CONFLICT (id) DO UPDATE SET
thai_name = EXCLUDED.thai_name,
english_name = EXCLUDED.english_name,
latitude = EXCLUDED.latitude,
longitude = EXCLUDED.longitude,
geohash = EXCLUDED.geohash,
updated_at = NOW()
"""
else: # MySQL
station_sql = """
INSERT INTO stations (id, station_code, thai_name, english_name, latitude, longitude, geohash, updated_at)
VALUES (:station_id, :station_code, :thai_name, :english_name, :latitude, :longitude, :geohash, NOW())
ON DUPLICATE KEY UPDATE
thai_name = VALUES(thai_name),
english_name = VALUES(english_name),
latitude = VALUES(latitude),
longitude = VALUES(longitude),
geohash = VALUES(geohash),
updated_at = NOW()
"""
conn.execute(text(station_sql), {
'station_id': measurement['station_id'],
'station_code': measurement['station_code'],
'thai_name': measurement['station_name_th'],
'english_name': measurement['station_name_en'],
'latitude': measurement.get('latitude'),
'longitude': measurement.get('longitude'),
'geohash': measurement.get('geohash')
})
# Insert measurements
for measurement in measurements:
if self.db_type == "sqlite":
measurement_sql = """
INSERT OR REPLACE INTO water_measurements
(timestamp, station_id, water_level, discharge, discharge_percent, status)
VALUES (:timestamp, :station_id, :water_level, :discharge, :discharge_percent, :status)
"""
elif self.db_type == "postgresql":
measurement_sql = """
INSERT INTO water_measurements
(timestamp, station_id, water_level, discharge, discharge_percent, status)
VALUES (:timestamp, :station_id, :water_level, :discharge, :discharge_percent, :status)
ON CONFLICT (timestamp, station_id) DO UPDATE SET
water_level = EXCLUDED.water_level,
discharge = EXCLUDED.discharge,
discharge_percent = EXCLUDED.discharge_percent,
status = EXCLUDED.status
"""
else: # MySQL
measurement_sql = """
INSERT INTO water_measurements
(timestamp, station_id, water_level, discharge, discharge_percent, status)
VALUES (:timestamp, :station_id, :water_level, :discharge, :discharge_percent, :status)
ON DUPLICATE KEY UPDATE
water_level = VALUES(water_level),
discharge = VALUES(discharge),
discharge_percent = VALUES(discharge_percent),
status = VALUES(status)
"""
conn.execute(text(measurement_sql), {
'timestamp': measurement['timestamp'],
'station_id': measurement['station_id'],
'water_level': measurement['water_level'],
'discharge': measurement['discharge'],
'discharge_percent': measurement['discharge_percent'],
'status': measurement['status']
})
# Transaction is automatically committed when context manager exits
logging.info(f"Successfully saved {len(measurements)} measurements to {self.db_type.upper()}")
return True
except Exception as e:
logging.error(f"Error saving to {self.db_type.upper()}: {e}")
return False
def get_latest_measurements(self, limit: int = 100) -> List[Dict]:
if not self.engine:
return []
try:
from sqlalchemy import text
query = """
SELECT m.timestamp, s.station_code, s.english_name, s.thai_name,
m.water_level, m.discharge, m.discharge_percent, m.status
FROM water_measurements m
JOIN stations s ON m.station_id = s.id
INNER JOIN (
SELECT station_id, MAX(timestamp) as max_timestamp
FROM water_measurements
GROUP BY station_id
) latest ON m.station_id = latest.station_id AND m.timestamp = latest.max_timestamp
ORDER BY s.station_code
LIMIT :limit
"""
with self.engine.connect() as conn:
result = conn.execute(text(query), {'limit': limit})
measurements = []
for row in result:
measurements.append({
'timestamp': row[0],
'station_code': row[1],
'station_name_en': row[2],
'station_name_th': row[3],
'water_level': float(row[4]) if row[4] else None,
'discharge': float(row[5]) if row[5] else None,
'discharge_percent': float(row[6]) if row[6] else None,
'status': row[7]
})
return measurements
except Exception as e:
logging.error(f"Error querying {self.db_type.upper()}: {e}")
return []
def get_measurements_by_timerange(self, start_time: datetime.datetime,
end_time: datetime.datetime,
station_codes: Optional[List[str]] = None) -> List[Dict]:
if not self.engine:
return []
try:
from sqlalchemy import text
where_clause = "m.timestamp BETWEEN :start_time AND :end_time"
params = {'start_time': start_time, 'end_time': end_time}
if station_codes:
placeholders = ','.join([f':station_{i}' for i in range(len(station_codes))])
where_clause += f" AND s.station_code IN ({placeholders})"
for i, code in enumerate(station_codes):
params[f'station_{i}'] = code
query = f"""
SELECT m.timestamp, s.station_code, s.english_name, s.thai_name,
m.water_level, m.discharge, m.discharge_percent, m.status
FROM water_measurements m
JOIN stations s ON m.station_id = s.id
WHERE {where_clause}
ORDER BY m.timestamp DESC, s.station_code
"""
with self.engine.connect() as conn:
result = conn.execute(text(query), params)
measurements = []
for row in result:
measurements.append({
'timestamp': row[0],
'station_code': row[1],
'station_name_en': row[2],
'station_name_th': row[3],
'water_level': float(row[4]) if row[4] else None,
'discharge': float(row[5]) if row[5] else None,
'discharge_percent': float(row[6]) if row[6] else None,
'status': row[7]
})
return measurements
except Exception as e:
logging.error(f"Error querying {self.db_type.upper()}: {e}")
return []
# VictoriaMetrics Adapter (using Prometheus format)
class VictoriaMetricsAdapter(DatabaseAdapter):
def __init__(self, host: str = "localhost", port: int = 8428):
self.host = host
self.port = port
# Handle HTTPS URLs and reverse proxy configurations
if host.startswith(('http://', 'https://')):
self.base_url = host
if port != 80 and port != 443 and not host.endswith(f':{port}'):
# Only add port if it's not standard and not already in URL
if '://' in host and ':' not in host.split('://')[1]:
self.base_url = f"{host}:{port}"
else:
# Default to HTTP for localhost, HTTPS for remote hosts
protocol = "https" if host != "localhost" and not host.startswith("127.") else "http"
if (protocol == "https" and port == 443) or (protocol == "http" and port == 80):
self.base_url = f"{protocol}://{host}"
else:
self.base_url = f"{protocol}://{host}:{port}"
def connect(self):
try:
import requests
# Test connection with SSL verification and timeout
response = requests.get(
f"{self.base_url}/api/v1/status/config",
timeout=10,
verify=True # Enable SSL verification for HTTPS
)
if response.status_code == 200:
logging.info(f"Connected to VictoriaMetrics successfully at {self.base_url}")
return True
else:
logging.error(f"VictoriaMetrics connection failed: {response.status_code}")
return False
except requests.exceptions.SSLError as e:
logging.error(f"SSL error connecting to VictoriaMetrics: {e}")
return False
except requests.exceptions.ConnectionError as e:
logging.error(f"Connection error to VictoriaMetrics: {e}")
return False
except Exception as e:
logging.error(f"Failed to connect to VictoriaMetrics: {e}")
return False
def save_measurements(self, measurements: List[Dict]) -> bool:
try:
import requests
# Convert to Prometheus format
metrics_data = []
timestamp_ms = int(datetime.datetime.now().timestamp() * 1000)
for measurement in measurements:
# Water level metric
metrics_data.append(
f'water_level{{station_code="{measurement["station_code"]}",'
f'station_name_en="{measurement["station_name_en"]}",'
f'station_name_th="{measurement["station_name_th"]}"}} '
f'{measurement["water_level"]} {timestamp_ms}'
)
# Discharge metric
metrics_data.append(
f'water_discharge{{station_code="{measurement["station_code"]}",'
f'station_name_en="{measurement["station_name_en"]}",'
f'station_name_th="{measurement["station_name_th"]}"}} '
f'{measurement["discharge"]} {timestamp_ms}'
)
# Discharge percentage metric
if measurement["discharge_percent"]:
metrics_data.append(
f'water_discharge_percent{{station_code="{measurement["station_code"]}",'
f'station_name_en="{measurement["station_name_en"]}",'
f'station_name_th="{measurement["station_name_th"]}"}} '
f'{measurement["discharge_percent"]} {timestamp_ms}'
)
# Send to VictoriaMetrics
data = '\n'.join(metrics_data)
response = requests.post(
f"{self.base_url}/api/v1/import/prometheus",
data=data,
headers={'Content-Type': 'text/plain'},
timeout=30
)
if response.status_code == 204:
logging.info(f"Successfully sent {len(measurements)} measurements to VictoriaMetrics")
return True
else:
logging.error(f"VictoriaMetrics import failed: {response.status_code} - {response.text}")
return False
except Exception as e:
logging.error(f"Error sending to VictoriaMetrics: {e}")
return False
def get_latest_measurements(self, limit: int = 100) -> List[Dict]:
# VictoriaMetrics queries would be implemented here
# This is a simplified version
logging.warning("get_latest_measurements not fully implemented for VictoriaMetrics")
return []
def get_measurements_by_timerange(self, start_time: datetime.datetime,
end_time: datetime.datetime,
station_codes: Optional[List[str]] = None) -> List[Dict]:
# VictoriaMetrics range queries would be implemented here
logging.warning("get_measurements_by_timerange not fully implemented for VictoriaMetrics")
return []
# Factory function to create appropriate adapter
def create_database_adapter(db_type: str, **kwargs) -> DatabaseAdapter:
"""
Factory function to create database adapter
Args:
db_type: 'influxdb', 'mysql', 'postgresql', 'sqlite', or 'victoriametrics'
**kwargs: Database-specific connection parameters
"""
db_type = db_type.lower()
if db_type == 'influxdb':
return InfluxDBAdapter(**kwargs)
elif db_type == 'mysql':
return SQLAdapter(db_type='mysql', **kwargs)
elif db_type == 'postgresql':
return SQLAdapter(db_type='postgresql', **kwargs)
elif db_type == 'sqlite':
return SQLAdapter(db_type='sqlite', **kwargs)
elif db_type == 'victoriametrics':
return VictoriaMetricsAdapter(**kwargs)
else:
raise ValueError(f"Unsupported database type: {db_type}")