Initial commit: Northern Thailand Ping River Monitor v3.1.0
Security & Dependency Updates / Dependency Security Scan (push) Successful in 29s
Security & Dependency Updates / Docker Security Scan (push) Failing after 53s
Security & Dependency Updates / License Compliance (push) Successful in 13s
Security & Dependency Updates / Check for Dependency Updates (push) Successful in 19s
Security & Dependency Updates / Code Quality Metrics (push) Successful in 11s
Security & Dependency Updates / Security Summary (push) Successful in 7s

Features:
- Real-time water level monitoring for Ping River Basin (16 stations)
- Coverage from Chiang Dao to Nakhon Sawan in Northern Thailand
- FastAPI web interface with interactive dashboard and station management
- Multi-database support (SQLite, MySQL, PostgreSQL, InfluxDB, VictoriaMetrics)
- Comprehensive monitoring with health checks and metrics collection
- Docker deployment with Grafana integration
- Production-ready architecture with enterprise-grade observability

 CI/CD & Automation:
- Complete Gitea Actions workflows for CI/CD, security, and releases
- Multi-Python version testing (3.9-3.12)
- Multi-architecture Docker builds (amd64, arm64)
- Daily security scanning and dependency monitoring
- Automated documentation generation
- Performance testing and validation

 Production Ready:
- Type safety with Pydantic models and comprehensive type hints
- Data validation layer with range checking and error handling
- Rate limiting and request tracking for API protection
- Enhanced logging with rotation, colors, and performance metrics
- Station management API for dynamic CRUD operations
- Comprehensive documentation and deployment guides

 Technical Stack:
- Python 3.9+ with FastAPI and Pydantic
- Multi-database architecture with adapter pattern
- Docker containerization with multi-stage builds
- Grafana dashboards for visualization
- Gitea Actions for CI/CD automation
- Enterprise monitoring and alerting

 Ready for deployment to B4L infrastructure!
This commit is contained in:
2025-08-12 15:40:24 +07:00
commit af62cfef0b
60 changed files with 13267 additions and 0 deletions
+171
View File
@@ -0,0 +1,171 @@
#!/usr/bin/env python3
"""
Metrics collection and monitoring for water monitoring system
"""
import time
import threading
from datetime import datetime, timedelta
from typing import Dict, Any, Optional, List
from dataclasses import dataclass, field
from collections import defaultdict, deque
import logging
logger = logging.getLogger(__name__)
@dataclass
class MetricPoint:
"""Single metric data point"""
timestamp: datetime
value: float
labels: Dict[str, str] = field(default_factory=dict)
class MetricsCollector:
"""Collects and manages application metrics"""
def __init__(self, retention_hours: int = 24):
self.retention_hours = retention_hours
self.metrics: Dict[str, deque] = defaultdict(lambda: deque(maxlen=1000))
self.counters: Dict[str, float] = defaultdict(float)
self.gauges: Dict[str, float] = defaultdict(float)
self.histograms: Dict[str, List[float]] = defaultdict(list)
self._lock = threading.Lock()
# Start cleanup thread
self._cleanup_thread = threading.Thread(target=self._cleanup_old_metrics, daemon=True)
self._cleanup_thread.start()
def increment_counter(self, name: str, value: float = 1.0, labels: Optional[Dict[str, str]] = None):
"""Increment a counter metric"""
with self._lock:
key = self._make_key(name, labels)
self.counters[key] += value
self.metrics[key].append(MetricPoint(datetime.now(), self.counters[key], labels or {}))
def set_gauge(self, name: str, value: float, labels: Optional[Dict[str, str]] = None):
"""Set a gauge metric"""
with self._lock:
key = self._make_key(name, labels)
self.gauges[key] = value
self.metrics[key].append(MetricPoint(datetime.now(), value, labels or {}))
def record_histogram(self, name: str, value: float, labels: Optional[Dict[str, str]] = None):
"""Record a histogram value"""
with self._lock:
key = self._make_key(name, labels)
self.histograms[key].append(value)
# Keep only recent values
if len(self.histograms[key]) > 1000:
self.histograms[key] = self.histograms[key][-1000:]
self.metrics[key].append(MetricPoint(datetime.now(), value, labels or {}))
def get_counter(self, name: str, labels: Optional[Dict[str, str]] = None) -> float:
"""Get current counter value"""
key = self._make_key(name, labels)
return self.counters.get(key, 0.0)
def get_gauge(self, name: str, labels: Optional[Dict[str, str]] = None) -> float:
"""Get current gauge value"""
key = self._make_key(name, labels)
return self.gauges.get(key, 0.0)
def get_histogram_stats(self, name: str, labels: Optional[Dict[str, str]] = None) -> Dict[str, float]:
"""Get histogram statistics"""
key = self._make_key(name, labels)
values = self.histograms.get(key, [])
if not values:
return {'count': 0, 'sum': 0, 'avg': 0, 'min': 0, 'max': 0}
return {
'count': len(values),
'sum': sum(values),
'avg': sum(values) / len(values),
'min': min(values),
'max': max(values)
}
def get_all_metrics(self) -> Dict[str, Any]:
"""Get all current metrics"""
with self._lock:
return {
'counters': dict(self.counters),
'gauges': dict(self.gauges),
'histograms': {k: self.get_histogram_stats(k) for k in self.histograms}
}
def _make_key(self, name: str, labels: Optional[Dict[str, str]]) -> str:
"""Create a unique key for metric with labels"""
if not labels:
return name
label_str = ','.join(f"{k}={v}" for k, v in sorted(labels.items()))
return f"{name}{{{label_str}}}"
def _cleanup_old_metrics(self):
"""Clean up old metric data points"""
while True:
try:
cutoff_time = datetime.now() - timedelta(hours=self.retention_hours)
with self._lock:
for metric_name, points in self.metrics.items():
# Remove old points
while points and points[0].timestamp < cutoff_time:
points.popleft()
time.sleep(3600) # Run cleanup every hour
except Exception as e:
logger.error(f"Error in metrics cleanup: {e}")
time.sleep(60) # Wait a minute before retrying
# Global metrics collector instance
_metrics_collector = None
def get_metrics_collector() -> MetricsCollector:
"""Get the global metrics collector instance"""
global _metrics_collector
if _metrics_collector is None:
_metrics_collector = MetricsCollector()
return _metrics_collector
# Convenience functions
def increment_counter(name: str, value: float = 1.0, labels: Optional[Dict[str, str]] = None):
"""Increment a counter metric"""
get_metrics_collector().increment_counter(name, value, labels)
def set_gauge(name: str, value: float, labels: Optional[Dict[str, str]] = None):
"""Set a gauge metric"""
get_metrics_collector().set_gauge(name, value, labels)
def record_histogram(name: str, value: float, labels: Optional[Dict[str, str]] = None):
"""Record a histogram value"""
get_metrics_collector().record_histogram(name, value, labels)
class Timer:
"""Context manager for timing operations"""
def __init__(self, metric_name: str, labels: Optional[Dict[str, str]] = None):
self.metric_name = metric_name
self.labels = labels
self.start_time = None
def __enter__(self):
self.start_time = time.time()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
if self.start_time:
duration = time.time() - self.start_time
record_histogram(self.metric_name, duration, self.labels)
def timer(metric_name: str, labels: Optional[Dict[str, str]] = None):
"""Decorator for timing function execution"""
def decorator(func):
def wrapper(*args, **kwargs):
with Timer(metric_name, labels):
return func(*args, **kwargs)
return wrapper
return decorator