B4L/Northern-Thailand-Ping-River-Monitor
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 15 Wiki Activity

Initial commit: Northern Thailand Ping River Monitor v3.1.0
Security & Dependency Updates / Dependency Security Scan (push) Successful in 29s
Details
Security & Dependency Updates / Docker Security Scan (push) Failing after 53s
Details
Security & Dependency Updates / License Compliance (push) Successful in 13s
Details
Security & Dependency Updates / Check for Dependency Updates (push) Successful in 19s
Details
Security & Dependency Updates / Code Quality Metrics (push) Successful in 11s
Details
Security & Dependency Updates / Security Summary (push) Successful in 7s
Details

Browse Source 
Features:
- Real-time water level monitoring for Ping River Basin (16 stations)
- Coverage from Chiang Dao to Nakhon Sawan in Northern Thailand
- FastAPI web interface with interactive dashboard and station management
- Multi-database support (SQLite, MySQL, PostgreSQL, InfluxDB, VictoriaMetrics)
- Comprehensive monitoring with health checks and metrics collection
- Docker deployment with Grafana integration
- Production-ready architecture with enterprise-grade observability

 CI/CD & Automation:
- Complete Gitea Actions workflows for CI/CD, security, and releases
- Multi-Python version testing (3.9-3.12)
- Multi-architecture Docker builds (amd64, arm64)
- Daily security scanning and dependency monitoring
- Automated documentation generation
- Performance testing and validation

 Production Ready:
- Type safety with Pydantic models and comprehensive type hints
- Data validation layer with range checking and error handling
- Rate limiting and request tracking for API protection
- Enhanced logging with rotation, colors, and performance metrics
- Station management API for dynamic CRUD operations
- Comprehensive documentation and deployment guides

 Technical Stack:
- Python 3.9+ with FastAPI and Pydantic
- Multi-database architecture with adapter pattern
- Docker containerization with multi-stage builds
- Grafana dashboards for visualization
- Gitea Actions for CI/CD automation
- Enterprise monitoring and alerting

 Ready for deployment to B4L infrastructure!
This commit is contained in:
grabowski
2025-08-12 15:40:24 +07:00
commit af62cfef0b
60 changed files with 13267 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/demo_databases.py
+331
View File
@@ -0,0 +1,331 @@
#!/usr/bin/env python3
"""
Demo script showing different database backend options for water monitoring
"""
import os
import sys
import datetime
from water_scraper_v3 import EnhancedWaterMonitorScraper
def demo_sqlite():
"""Demo with SQLite (local development)"""
print("=" * 60)
print("🗄️ SQLite Demo (Local Development)")
print("=" * 60)
config = {
'type': 'sqlite',
'connection_string': 'sqlite:///demo_water_sqlite.db'
}
try:
scraper = EnhancedWaterMonitorScraper(config)
# Fetch and save data
print("Fetching data from API...")
data = scraper.fetch_water_data()
if data:
print(f"✓ Fetched {len(data)} data points")
success = scraper.save_to_database(data)
if success:
print("✓ Data saved to SQLite database")
# Show latest data
latest = scraper.get_latest_data(5)
print(f"\nLatest 5 measurements:")
for measurement in latest:
print(f" • {measurement['station_code']} ({measurement['station_name_en']}): "
f"{measurement['water_level']:.2f}m, {measurement['discharge']:.1f} cms")
else:
print("✗ Failed to save data")
else:
print("✗ No data fetched")
except Exception as e:
print(f"Error: {e}")
def demo_influxdb():
"""Demo with InfluxDB (requires InfluxDB running)"""
print("\n" + "=" * 60)
print("📊 InfluxDB Demo (Time-Series Database)")
print("=" * 60)
config = {
'type': 'influxdb',
'host': 'localhost',
'port': 8086,
'database': 'water_monitoring_demo',
'username': None, # Set if authentication is enabled
'password': None
}
try:
scraper = EnhancedWaterMonitorScraper(config)
if scraper.db_adapter and scraper.db_adapter.client:
print("✓ Connected to InfluxDB")
# Fetch and save data
print("Fetching data from API...")
data = scraper.fetch_water_data()
if data:
print(f"✓ Fetched {len(data)} data points")
success = scraper.save_to_database(data)
if success:
print("✓ Data saved to InfluxDB")
print("💡 You can now query this data in Grafana or InfluxDB CLI")
print(" Example query: SELECT * FROM water_data ORDER BY time DESC LIMIT 10")
else:
print("✗ Failed to save data")
else:
print("✗ No data fetched")
else:
print("✗ Could not connect to InfluxDB")
print("💡 Make sure InfluxDB is running: docker run -p 8086:8086 influxdb:1.8")
except Exception as e:
print(f"Error: {e}")
print("💡 InfluxDB might not be running or accessible")
def demo_postgresql():
"""Demo with PostgreSQL (requires PostgreSQL running)"""
print("\n" + "=" * 60)
print("🐘 PostgreSQL Demo (Relational Database)")
print("=" * 60)
config = {
'type': 'postgresql',
'connection_string': 'postgresql://postgres:password@localhost:5432/water_monitoring'
}
try:
scraper = EnhancedWaterMonitorScraper(config)
if scraper.db_adapter and scraper.db_adapter.engine:
print("✓ Connected to PostgreSQL")
# Fetch and save data
print("Fetching data from API...")
data = scraper.fetch_water_data()
if data:
print(f"✓ Fetched {len(data)} data points")
success = scraper.save_to_database(data)
if success:
print("✓ Data saved to PostgreSQL")
print("💡 You can now query this data with SQL")
print(" Example: SELECT * FROM water_measurements ORDER BY timestamp DESC LIMIT 10;")
else:
print("✗ Failed to save data")
else:
print("✗ No data fetched")
else:
print("✗ Could not connect to PostgreSQL")
print("💡 Make sure PostgreSQL is running with correct credentials")
except Exception as e:
print(f"Error: {e}")
print("💡 PostgreSQL might not be running or credentials might be wrong")
def demo_mysql():
"""Demo with MySQL (requires MySQL running)"""
print("\n" + "=" * 60)
print("🐬 MySQL Demo (Relational Database)")
print("=" * 60)
config = {
'type': 'mysql',
'connection_string': 'mysql://root:password@localhost:3306/water_monitoring'
}
try:
scraper = EnhancedWaterMonitorScraper(config)
if scraper.db_adapter and scraper.db_adapter.engine:
print("✓ Connected to MySQL")
# Fetch and save data
print("Fetching data from API...")
data = scraper.fetch_water_data()
if data:
print(f"✓ Fetched {len(data)} data points")
success = scraper.save_to_database(data)
if success:
print("✓ Data saved to MySQL")
print("💡 You can now query this data with SQL")
print(" Example: SELECT * FROM water_measurements ORDER BY timestamp DESC LIMIT 10;")
else:
print("✗ Failed to save data")
else:
print("✗ No data fetched")
else:
print("✗ Could not connect to MySQL")
print("💡 Make sure MySQL is running with correct credentials")
except Exception as e:
print(f"Error: {e}")
print("💡 MySQL might not be running or credentials might be wrong")
def demo_victoriametrics():
"""Demo with VictoriaMetrics (supports both local and HTTPS configurations)"""
print("\n" + "=" * 60)
print("⚡ VictoriaMetrics Demo (High-Performance Metrics)")
print("=" * 60)
# Use configuration from environment or config.py
from config import Config
db_config = Config.get_database_config()
if db_config['type'] != 'victoriametrics':
# Fallback to default local configuration
config = {
'type': 'victoriametrics',
'host': 'vm.newedge.house',
'port': 443
}
else:
config = db_config
print(f"Connecting to: {config['host']}:{config['port']}")
try:
scraper = EnhancedWaterMonitorScraper(config)
if scraper.db_adapter:
# Test connection using the adapter's connect method
if scraper.db_adapter.connect():
print("✓ Connected to VictoriaMetrics")
# Fetch and save data
print("Fetching data from API...")
data = scraper.fetch_water_data()
if data:
print(f"✓ Fetched {len(data)} data points")
success = scraper.save_to_database(data)
if success:
print("✓ Data saved to VictoriaMetrics")
print("💡 You can now query this data via Prometheus API")
# Show appropriate query URL based on configuration
base_url = scraper.db_adapter.base_url
print(f" Example: {base_url}/api/v1/query?query=water_level")
print(f" Health check: {base_url}/health")
else:
print("✗ Failed to save data")
else:
print("✗ No data fetched")
else:
print("✗ Could not connect to VictoriaMetrics")
if config['host'] == 'localhost':
print("💡 Make sure VictoriaMetrics is running locally:")
print(" docker run -p 8428:8428 victoriametrics/victoria-metrics")
else:
print(f"💡 Check if VictoriaMetrics is accessible at {config['host']}:{config['port']}")
print("💡 Verify HTTPS configuration and network connectivity")
else:
print("✗ Failed to initialize VictoriaMetrics adapter")
except Exception as e:
print(f"Error: {e}")
print("💡 Check your VictoriaMetrics configuration and network connectivity")
def show_recommendations():
"""Show database recommendations"""
print("\n" + "=" * 60)
print("🏆 Database Recommendations")
print("=" * 60)
recommendations = [
{
'name': 'InfluxDB',
'best_for': 'Time-series data, Grafana dashboards',
'pros': ['Purpose-built for time-series', 'Great compression', 'Built-in retention'],
'cons': ['Learning curve', 'Less flexible for complex queries'],
'use_case': 'Recommended for most water monitoring deployments'
},
{
'name': 'PostgreSQL + TimescaleDB',
'best_for': 'Complex queries, existing PostgreSQL infrastructure',
'pros': ['Mature ecosystem', 'SQL compatibility', 'ACID compliance'],
'cons': ['More complex setup', 'Higher resource usage'],
'use_case': 'Best for organizations already using PostgreSQL'
},
{
'name': 'VictoriaMetrics',
'best_for': 'High-performance metrics, Prometheus compatibility',
'pros': ['Extremely fast', 'Low resource usage', 'Better compression'],
'cons': ['Newer ecosystem', 'Less tooling'],
'use_case': 'Best for high-volume, performance-critical deployments'
},
{
'name': 'MySQL',
'best_for': 'Existing MySQL infrastructure, familiar SQL',
'pros': ['Familiar', 'Mature', 'Wide support'],
'cons': ['Not optimized for time-series', 'Manual optimization needed'],
'use_case': 'Good for organizations with existing MySQL expertise'
}
]
for rec in recommendations:
print(f"\n📊 {rec['name']}")
print(f" Best for: {rec['best_for']}")
print(f" Pros: {', '.join(rec['pros'])}")
print(f" Cons: {', '.join(rec['cons'])}")
print(f" 💡 {rec['use_case']}")
def main():
"""Main demo function"""
print("🌊 Thailand Water Monitor - Database Backend Demo")
print("This demo shows how to use different database backends")
# Always run SQLite demo (no external dependencies)
demo_sqlite()
# Check for command line arguments to run specific demos
if len(sys.argv) > 1:
db_type = sys.argv[1].lower()
if db_type == 'influxdb':
demo_influxdb()
elif db_type == 'postgresql':
demo_postgresql()
elif db_type == 'mysql':
demo_mysql()
elif db_type == 'victoriametrics':
demo_victoriametrics()
elif db_type == 'all':
demo_influxdb()
demo_postgresql()
demo_mysql()
demo_victoriametrics()
else:
print(f"\nUnknown database type: {db_type}")
print("Available options: influxdb, postgresql, mysql, victoriametrics, all")
else:
print("\n💡 To test other databases, run:")
print(" python demo_databases.py influxdb")
print(" python demo_databases.py postgresql")
print(" python demo_databases.py mysql")
print(" python demo_databases.py victoriametrics")
print(" python demo_databases.py all")
# Show recommendations
show_recommendations()
print("\n" + "=" * 60)
print("✅ Demo completed!")
print("📖 See DATABASE_DEPLOYMENT_GUIDE.md for production setup instructions")
print("=" * 60)
if __name__ == "__main__":
main()