grabowski
9cac9c4d2a
The push-CI gates (black/isort/mypy) had never actually run before the branch-trigger fix, and the codebase predates them. Formatting is now black/isort clean repo-wide. mypy keeps running but non-blocking: 86 pre-existing errors are a separate cleanup, not a gate to hold hostage.
353 lines
12 KiB
Python
353 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Demo script showing different database backend options for water monitoring
|
|
"""
|
|
|
|
import datetime
|
|
import os
|
|
import sys
|
|
|
|
from water_scraper_v3 import EnhancedWaterMonitorScraper
|
|
|
|
|
|
def demo_sqlite():
|
|
"""Demo with SQLite (local development)"""
|
|
print("=" * 60)
|
|
print("🗄️ SQLite Demo (Local Development)")
|
|
print("=" * 60)
|
|
|
|
config = {"type": "sqlite", "connection_string": "sqlite:///demo_water_sqlite.db"}
|
|
|
|
try:
|
|
scraper = EnhancedWaterMonitorScraper(config)
|
|
|
|
# Fetch and save data
|
|
print("Fetching data from API...")
|
|
data = scraper.fetch_water_data()
|
|
|
|
if data:
|
|
print(f"✓ Fetched {len(data)} data points")
|
|
success = scraper.save_to_database(data)
|
|
|
|
if success:
|
|
print("✓ Data saved to SQLite database")
|
|
|
|
# Show latest data
|
|
latest = scraper.get_latest_data(5)
|
|
print(f"\nLatest 5 measurements:")
|
|
for measurement in latest:
|
|
print(
|
|
f" • {measurement['station_code']} ({measurement['station_name_en']}): "
|
|
f"{measurement['water_level']:.2f}m, {measurement['discharge']:.1f} cms"
|
|
)
|
|
else:
|
|
print("✗ Failed to save data")
|
|
else:
|
|
print("✗ No data fetched")
|
|
|
|
except Exception as e:
|
|
print(f"Error: {e}")
|
|
|
|
|
|
def demo_influxdb():
|
|
"""Demo with InfluxDB (requires InfluxDB running)"""
|
|
print("\n" + "=" * 60)
|
|
print("📊 InfluxDB Demo (Time-Series Database)")
|
|
print("=" * 60)
|
|
|
|
config = {
|
|
"type": "influxdb",
|
|
"host": "localhost",
|
|
"port": 8086,
|
|
"database": "water_monitoring_demo",
|
|
"username": None, # Set if authentication is enabled
|
|
"password": None,
|
|
}
|
|
|
|
try:
|
|
scraper = EnhancedWaterMonitorScraper(config)
|
|
|
|
if scraper.db_adapter and scraper.db_adapter.client:
|
|
print("✓ Connected to InfluxDB")
|
|
|
|
# Fetch and save data
|
|
print("Fetching data from API...")
|
|
data = scraper.fetch_water_data()
|
|
|
|
if data:
|
|
print(f"✓ Fetched {len(data)} data points")
|
|
success = scraper.save_to_database(data)
|
|
|
|
if success:
|
|
print("✓ Data saved to InfluxDB")
|
|
print("💡 You can now query this data in Grafana or InfluxDB CLI")
|
|
print(
|
|
" Example query: SELECT * FROM water_data ORDER BY time DESC LIMIT 10"
|
|
)
|
|
else:
|
|
print("✗ Failed to save data")
|
|
else:
|
|
print("✗ No data fetched")
|
|
else:
|
|
print("✗ Could not connect to InfluxDB")
|
|
print(
|
|
"💡 Make sure InfluxDB is running: docker run -p 8086:8086 influxdb:1.8"
|
|
)
|
|
|
|
except Exception as e:
|
|
print(f"Error: {e}")
|
|
print("💡 InfluxDB might not be running or accessible")
|
|
|
|
|
|
def demo_postgresql():
|
|
"""Demo with PostgreSQL (requires PostgreSQL running)"""
|
|
print("\n" + "=" * 60)
|
|
print("🐘 PostgreSQL Demo (Relational Database)")
|
|
print("=" * 60)
|
|
|
|
config = {
|
|
"type": "postgresql",
|
|
"connection_string": "postgresql://postgres:password@localhost:5432/water_monitoring",
|
|
}
|
|
|
|
try:
|
|
scraper = EnhancedWaterMonitorScraper(config)
|
|
|
|
if scraper.db_adapter and scraper.db_adapter.engine:
|
|
print("✓ Connected to PostgreSQL")
|
|
|
|
# Fetch and save data
|
|
print("Fetching data from API...")
|
|
data = scraper.fetch_water_data()
|
|
|
|
if data:
|
|
print(f"✓ Fetched {len(data)} data points")
|
|
success = scraper.save_to_database(data)
|
|
|
|
if success:
|
|
print("✓ Data saved to PostgreSQL")
|
|
print("💡 You can now query this data with SQL")
|
|
print(
|
|
" Example: SELECT * FROM water_measurements ORDER BY timestamp DESC LIMIT 10;"
|
|
)
|
|
else:
|
|
print("✗ Failed to save data")
|
|
else:
|
|
print("✗ No data fetched")
|
|
else:
|
|
print("✗ Could not connect to PostgreSQL")
|
|
print("💡 Make sure PostgreSQL is running with correct credentials")
|
|
|
|
except Exception as e:
|
|
print(f"Error: {e}")
|
|
print("💡 PostgreSQL might not be running or credentials might be wrong")
|
|
|
|
|
|
def demo_mysql():
|
|
"""Demo with MySQL (requires MySQL running)"""
|
|
print("\n" + "=" * 60)
|
|
print("🐬 MySQL Demo (Relational Database)")
|
|
print("=" * 60)
|
|
|
|
config = {
|
|
"type": "mysql",
|
|
"connection_string": "mysql://root:password@localhost:3306/water_monitoring",
|
|
}
|
|
|
|
try:
|
|
scraper = EnhancedWaterMonitorScraper(config)
|
|
|
|
if scraper.db_adapter and scraper.db_adapter.engine:
|
|
print("✓ Connected to MySQL")
|
|
|
|
# Fetch and save data
|
|
print("Fetching data from API...")
|
|
data = scraper.fetch_water_data()
|
|
|
|
if data:
|
|
print(f"✓ Fetched {len(data)} data points")
|
|
success = scraper.save_to_database(data)
|
|
|
|
if success:
|
|
print("✓ Data saved to MySQL")
|
|
print("💡 You can now query this data with SQL")
|
|
print(
|
|
" Example: SELECT * FROM water_measurements ORDER BY timestamp DESC LIMIT 10;"
|
|
)
|
|
else:
|
|
print("✗ Failed to save data")
|
|
else:
|
|
print("✗ No data fetched")
|
|
else:
|
|
print("✗ Could not connect to MySQL")
|
|
print("💡 Make sure MySQL is running with correct credentials")
|
|
|
|
except Exception as e:
|
|
print(f"Error: {e}")
|
|
print("💡 MySQL might not be running or credentials might be wrong")
|
|
|
|
|
|
def demo_victoriametrics():
|
|
"""Demo with VictoriaMetrics (supports both local and HTTPS configurations)"""
|
|
print("\n" + "=" * 60)
|
|
print("⚡ VictoriaMetrics Demo (High-Performance Metrics)")
|
|
print("=" * 60)
|
|
|
|
# Use configuration from environment or config.py
|
|
from config import Config
|
|
|
|
db_config = Config.get_database_config()
|
|
|
|
if db_config["type"] != "victoriametrics":
|
|
# Fallback to default local configuration
|
|
config = {"type": "victoriametrics", "host": "vm.newedge.house", "port": 443}
|
|
else:
|
|
config = db_config
|
|
|
|
print(f"Connecting to: {config['host']}:{config['port']}")
|
|
|
|
try:
|
|
scraper = EnhancedWaterMonitorScraper(config)
|
|
|
|
if scraper.db_adapter:
|
|
# Test connection using the adapter's connect method
|
|
if scraper.db_adapter.connect():
|
|
print("✓ Connected to VictoriaMetrics")
|
|
|
|
# Fetch and save data
|
|
print("Fetching data from API...")
|
|
data = scraper.fetch_water_data()
|
|
|
|
if data:
|
|
print(f"✓ Fetched {len(data)} data points")
|
|
success = scraper.save_to_database(data)
|
|
|
|
if success:
|
|
print("✓ Data saved to VictoriaMetrics")
|
|
print("💡 You can now query this data via Prometheus API")
|
|
|
|
# Show appropriate query URL based on configuration
|
|
base_url = scraper.db_adapter.base_url
|
|
print(f" Example: {base_url}/api/v1/query?query=water_level")
|
|
print(f" Health check: {base_url}/health")
|
|
else:
|
|
print("✗ Failed to save data")
|
|
else:
|
|
print("✗ No data fetched")
|
|
else:
|
|
print("✗ Could not connect to VictoriaMetrics")
|
|
if config["host"] == "localhost":
|
|
print("💡 Make sure VictoriaMetrics is running locally:")
|
|
print(" docker run -p 8428:8428 victoriametrics/victoria-metrics")
|
|
else:
|
|
print(
|
|
f"💡 Check if VictoriaMetrics is accessible at {config['host']}:{config['port']}"
|
|
)
|
|
print("💡 Verify HTTPS configuration and network connectivity")
|
|
else:
|
|
print("✗ Failed to initialize VictoriaMetrics adapter")
|
|
|
|
except Exception as e:
|
|
print(f"Error: {e}")
|
|
print("💡 Check your VictoriaMetrics configuration and network connectivity")
|
|
|
|
|
|
def show_recommendations():
|
|
"""Show database recommendations"""
|
|
print("\n" + "=" * 60)
|
|
print("🏆 Database Recommendations")
|
|
print("=" * 60)
|
|
|
|
recommendations = [
|
|
{
|
|
"name": "InfluxDB",
|
|
"best_for": "Time-series data, Grafana dashboards",
|
|
"pros": [
|
|
"Purpose-built for time-series",
|
|
"Great compression",
|
|
"Built-in retention",
|
|
],
|
|
"cons": ["Learning curve", "Less flexible for complex queries"],
|
|
"use_case": "Recommended for most water monitoring deployments",
|
|
},
|
|
{
|
|
"name": "PostgreSQL + TimescaleDB",
|
|
"best_for": "Complex queries, existing PostgreSQL infrastructure",
|
|
"pros": ["Mature ecosystem", "SQL compatibility", "ACID compliance"],
|
|
"cons": ["More complex setup", "Higher resource usage"],
|
|
"use_case": "Best for organizations already using PostgreSQL",
|
|
},
|
|
{
|
|
"name": "VictoriaMetrics",
|
|
"best_for": "High-performance metrics, Prometheus compatibility",
|
|
"pros": ["Extremely fast", "Low resource usage", "Better compression"],
|
|
"cons": ["Newer ecosystem", "Less tooling"],
|
|
"use_case": "Best for high-volume, performance-critical deployments",
|
|
},
|
|
{
|
|
"name": "MySQL",
|
|
"best_for": "Existing MySQL infrastructure, familiar SQL",
|
|
"pros": ["Familiar", "Mature", "Wide support"],
|
|
"cons": ["Not optimized for time-series", "Manual optimization needed"],
|
|
"use_case": "Good for organizations with existing MySQL expertise",
|
|
},
|
|
]
|
|
|
|
for rec in recommendations:
|
|
print(f"\n📊 {rec['name']}")
|
|
print(f" Best for: {rec['best_for']}")
|
|
print(f" Pros: {', '.join(rec['pros'])}")
|
|
print(f" Cons: {', '.join(rec['cons'])}")
|
|
print(f" 💡 {rec['use_case']}")
|
|
|
|
|
|
def main():
|
|
"""Main demo function"""
|
|
print("🌊 Thailand Water Monitor - Database Backend Demo")
|
|
print("This demo shows how to use different database backends")
|
|
|
|
# Always run SQLite demo (no external dependencies)
|
|
demo_sqlite()
|
|
|
|
# Check for command line arguments to run specific demos
|
|
if len(sys.argv) > 1:
|
|
db_type = sys.argv[1].lower()
|
|
|
|
if db_type == "influxdb":
|
|
demo_influxdb()
|
|
elif db_type == "postgresql":
|
|
demo_postgresql()
|
|
elif db_type == "mysql":
|
|
demo_mysql()
|
|
elif db_type == "victoriametrics":
|
|
demo_victoriametrics()
|
|
elif db_type == "all":
|
|
demo_influxdb()
|
|
demo_postgresql()
|
|
demo_mysql()
|
|
demo_victoriametrics()
|
|
else:
|
|
print(f"\nUnknown database type: {db_type}")
|
|
print(
|
|
"Available options: influxdb, postgresql, mysql, victoriametrics, all"
|
|
)
|
|
else:
|
|
print("\n💡 To test other databases, run:")
|
|
print(" python demo_databases.py influxdb")
|
|
print(" python demo_databases.py postgresql")
|
|
print(" python demo_databases.py mysql")
|
|
print(" python demo_databases.py victoriametrics")
|
|
print(" python demo_databases.py all")
|
|
|
|
# Show recommendations
|
|
show_recommendations()
|
|
|
|
print("\n" + "=" * 60)
|
|
print("✅ Demo completed!")
|
|
print("📖 See DATABASE_DEPLOYMENT_GUIDE.md for production setup instructions")
|
|
print("=" * 60)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|