Initial commit: Northern Thailand Ping River Monitor v3.1.0

Features:
- Real-time water level monitoring for Ping River Basin (16 stations)
- Coverage from Chiang Dao to Nakhon Sawan in Northern Thailand
- FastAPI web interface with interactive dashboard and station management
- Multi-database support (SQLite, MySQL, PostgreSQL, InfluxDB, VictoriaMetrics)
- Comprehensive monitoring with health checks and metrics collection
- Docker deployment with Grafana integration
- Production-ready architecture with enterprise-grade observability

 CI/CD & Automation:
- Complete Gitea Actions workflows for CI/CD, security, and releases
- Multi-Python version testing (3.9-3.12)
- Multi-architecture Docker builds (amd64, arm64)
- Daily security scanning and dependency monitoring
- Automated documentation generation
- Performance testing and validation

 Production Ready:
- Type safety with Pydantic models and comprehensive type hints
- Data validation layer with range checking and error handling
- Rate limiting and request tracking for API protection
- Enhanced logging with rotation, colors, and performance metrics
- Station management API for dynamic CRUD operations
- Comprehensive documentation and deployment guides

 Technical Stack:
- Python 3.9+ with FastAPI and Pydantic
- Multi-database architecture with adapter pattern
- Docker containerization with multi-stage builds
- Grafana dashboards for visualization
- Gitea Actions for CI/CD automation
- Enterprise monitoring and alerting

 Ready for deployment to B4L infrastructure!

tests/test_api.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+"""
+Simple API test script
+"""
+
+import requests
+import time
+import json
+
+def test_api_endpoints():
+    """Test the main API endpoints"""
+    base_url = "http://localhost:8000"
+    
+    endpoints = [
+        "/health",
+        "/metrics", 
+        "/stations",
+        "/measurements/latest?limit=5",
+        "/scraping/status",
+        "/config"
+    ]
+    
+    print("🧪 Testing API endpoints...")
+    print("Make sure the API server is running: python run.py --web-api")
+    print()
+    
+    for endpoint in endpoints:
+        try:
+            print(f"Testing {endpoint}...")
+            response = requests.get(f"{base_url}{endpoint}", timeout=5)
+            
+            if response.status_code == 200:
+                data = response.json()
+                print(f"✅ {endpoint} - Status: {response.status_code}")
+                
+                # Show some sample data
+                if endpoint == "/stations":
+                    print(f"   Found {len(data)} stations")
+                elif endpoint == "/measurements/latest?limit=5":
+                    print(f"   Found {len(data)} measurements")
+                    if data:
+                        latest = data[0]
+                        print(f"   Latest: {latest['station_code']} - {latest['water_level']}m")
+                elif endpoint == "/health":
+                    print(f"   Overall status: {data.get('overall_status', 'unknown')}")
+                elif endpoint == "/scraping/status":
+                    print(f"   Is running: {data.get('is_running', False)}")
+                    print(f"   Total runs: {data.get('total_runs', 0)}")
+                
+            else:
+                print(f"❌ {endpoint} - Status: {response.status_code}")
+                
+        except requests.exceptions.ConnectionError:
+            print(f"❌ {endpoint} - Connection failed (is the server running?)")
+        except Exception as e:
+            print(f"❌ {endpoint} - Error: {e}")
+        
+        print()
+
+if __name__ == "__main__":
+    test_api_endpoints()

tests/test_integration.py

@@ -0,0 +1,210 @@
+#!/usr/bin/env python3
+"""
+Integration test script for the enhanced water monitoring system
+"""
+
+import sys
+import os
+import time
+from datetime import datetime
+
+# Add project root to Python path
+project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, project_root)
+
+def test_imports():
+    """Test that all modules can be imported"""
+    print("Testing imports...")
+    
+    try:
+        from src.config import Config
+        from src.models import WaterMeasurement, StationInfo
+        from src.exceptions import WaterMonitorException
+        from src.validators import DataValidator
+        from src.metrics import get_metrics_collector
+        from src.health_check import HealthCheckManager
+        from src.rate_limiter import RateLimiter
+        from src.logging_config import setup_logging
+        print("✅ All imports successful")
+        return True
+    except ImportError as e:
+        print(f"❌ Import failed: {e}")
+        return False
+
+def test_configuration():
+    """Test configuration validation"""
+    print("Testing configuration...")
+    
+    try:
+        from src.config import Config
+        
+        # Test configuration loading
+        settings = Config.get_all_settings()
+        print(f"✅ Configuration loaded: {len(settings)} settings")
+        
+        # Test database config
+        db_config = Config.get_database_config()
+        print(f"✅ Database config: {db_config['type']}")
+        
+        return True
+    except Exception as e:
+        print(f"❌ Configuration test failed: {e}")
+        return False
+
+def test_metrics():
+    """Test metrics collection"""
+    print("Testing metrics...")
+    
+    try:
+        from src.metrics import increment_counter, set_gauge, record_histogram, get_metrics_collector
+        
+        # Test metrics
+        increment_counter("test_counter", 5)
+        set_gauge("test_gauge", 42.0)
+        record_histogram("test_histogram", 1.5)
+        
+        # Get metrics
+        collector = get_metrics_collector()
+        metrics = collector.get_all_metrics()
+        
+        print(f"✅ Metrics collected: {len(metrics['counters'])} counters, {len(metrics['gauges'])} gauges")
+        return True
+    except Exception as e:
+        print(f"❌ Metrics test failed: {e}")
+        return False
+
+def test_validation():
+    """Test data validation"""
+    print("Testing data validation...")
+    
+    try:
+        from src.validators import DataValidator
+        
+        # Test valid measurement
+        valid_measurement = {
+            'timestamp': datetime.now(),
+            'station_id': 1,
+            'water_level': 5.5,
+            'discharge': 100.0,
+            'discharge_percent': 75.0
+        }
+        
+        result = DataValidator.validate_measurement(valid_measurement)
+        if result:
+            print("✅ Valid measurement passed validation")
+        else:
+            print("❌ Valid measurement failed validation")
+            return False
+        
+        # Test invalid measurement
+        invalid_measurement = {
+            'timestamp': datetime.now(),
+            'station_id': 999,  # Invalid station ID
+            'water_level': -999.0,  # Invalid water level
+            'discharge': -50.0  # Invalid discharge
+        }
+        
+        result = DataValidator.validate_measurement(invalid_measurement)
+        if not result:
+            print("✅ Invalid measurement correctly rejected")
+        else:
+            print("❌ Invalid measurement incorrectly accepted")
+            return False
+        
+        return True
+    except Exception as e:
+        print(f"❌ Validation test failed: {e}")
+        return False
+
+def test_rate_limiter():
+    """Test rate limiting"""
+    print("Testing rate limiter...")
+    
+    try:
+        from src.rate_limiter import RateLimiter
+        
+        # Create rate limiter (2 requests per second)
+        limiter = RateLimiter(max_requests=2, time_window_seconds=1)
+        
+        # Test allowed requests
+        for i in range(2):
+            if not limiter.is_allowed():
+                print(f"❌ Request {i+1} should be allowed")
+                return False
+        
+        # Test blocked request
+        if limiter.is_allowed():
+            print("❌ Third request should be blocked")
+            return False
+        
+        print("✅ Rate limiter working correctly")
+        return True
+    except Exception as e:
+        print(f"❌ Rate limiter test failed: {e}")
+        return False
+
+def test_logging():
+    """Test logging configuration"""
+    print("Testing logging...")
+    
+    try:
+        from src.logging_config import setup_logging, get_logger
+        
+        # Setup logging
+        logger = setup_logging(log_level="INFO", enable_console=False)
+        
+        # Get a logger
+        test_logger = get_logger("test")
+        test_logger.info("Test log message")
+        
+        print("✅ Logging setup successful")
+        return True
+    except Exception as e:
+        print(f"❌ Logging test failed: {e}")
+        return False
+
+def main():
+    """Run all tests"""
+    print("🧪 Running integration tests for Northern Thailand Ping River Monitor v3.1.0")
+    print("=" * 60)
+    
+    tests = [
+        test_imports,
+        test_configuration,
+        test_metrics,
+        test_validation,
+        test_rate_limiter,
+        test_logging
+    ]
+    
+    passed = 0
+    failed = 0
+    
+    for test in tests:
+        try:
+            if test():
+                passed += 1
+            else:
+                failed += 1
+        except Exception as e:
+            print(f"❌ Test {test.__name__} crashed: {e}")
+            failed += 1
+        print()
+    
+    print("=" * 60)
+    print(f"Test Results: {passed} passed, {failed} failed")
+    
+    if failed == 0:
+        print("🎉 All tests passed! The system is ready to use.")
+        print("\nNext steps:")
+        print("1. Run 'python run.py --test' to test data collection")
+        print("2. Run 'python run.py --web-api' to start the web interface")
+        print("3. Visit http://localhost:8000 for the dashboard")
+        return True
+    else:
+        print("❌ Some tests failed. Please check the errors above.")
+        return False
+
+if __name__ == "__main__":
+    success = main()
+    sys.exit(0 if success else 1)

tests/test_station_management.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+"""
+Test script for station management API endpoints
+"""
+
+import requests
+import json
+
+def test_station_management():
+    """Test the station management endpoints"""
+    base_url = "http://localhost:8000"
+    
+    print("🧪 Testing Station Management API")
+    print("Make sure the API server is running: python run.py --web-api")
+    print()
+    
+    # Test data for new station
+    new_station = {
+        "station_code": "P.TEST",
+        "thai_name": "สถานีทดสอบ",
+        "english_name": "Test Station",
+        "latitude": 18.7875,
+        "longitude": 99.0045,
+        "geohash": "w5q6uuhvfcfp25",
+        "status": "active"
+    }
+    
+    try:
+        # 1. List existing stations
+        print("1. Listing existing stations...")
+        response = requests.get(f"{base_url}/stations")
+        if response.status_code == 200:
+            stations = response.json()
+            print(f"✅ Found {len(stations)} existing stations")
+            initial_count = len(stations)
+        else:
+            print(f"❌ Failed to list stations: {response.status_code}")
+            return
+        
+        # 2. Create new station
+        print("\n2. Creating new test station...")
+        response = requests.post(
+            f"{base_url}/stations",
+            json=new_station,
+            headers={"Content-Type": "application/json"}
+        )
+        
+        if response.status_code == 200:
+            created_station = response.json()
+            station_id = created_station["station_id"]
+            print(f"✅ Created station with ID: {station_id}")
+            print(f"   Code: {created_station['station_code']}")
+            print(f"   Name: {created_station['english_name']}")
+        else:
+            print(f"❌ Failed to create station: {response.status_code}")
+            print(f"   Response: {response.text}")
+            return
+        
+        # 3. Get specific station
+        print(f"\n3. Getting station details for ID {station_id}...")
+        response = requests.get(f"{base_url}/stations/{station_id}")
+        if response.status_code == 200:
+            station_details = response.json()
+            print(f"✅ Retrieved station details")
+            print(f"   Thai name: {station_details['thai_name']}")
+            print(f"   Coordinates: {station_details['latitude']}, {station_details['longitude']}")
+        else:
+            print(f"❌ Failed to get station: {response.status_code}")
+        
+        # 4. Update station
+        print(f"\n4. Updating station {station_id}...")
+        update_data = {
+            "english_name": "Updated Test Station",
+            "thai_name": "สถานีทดสอบที่อัปเดต"
+        }
+        
+        response = requests.put(
+            f"{base_url}/stations/{station_id}",
+            json=update_data,
+            headers={"Content-Type": "application/json"}
+        )
+        
+        if response.status_code == 200:
+            updated_station = response.json()
+            print(f"✅ Updated station successfully")
+            print(f"   New name: {updated_station['english_name']}")
+        else:
+            print(f"❌ Failed to update station: {response.status_code}")
+            print(f"   Response: {response.text}")
+        
+        # 5. Verify station count increased
+        print("\n5. Verifying station count...")
+        response = requests.get(f"{base_url}/stations")
+        if response.status_code == 200:
+            stations = response.json()
+            new_count = len(stations)
+            if new_count == initial_count + 1:
+                print(f"✅ Station count increased from {initial_count} to {new_count}")
+            else:
+                print(f"⚠️ Unexpected station count: {new_count} (expected {initial_count + 1})")
+        
+        # 6. Delete test station
+        print(f"\n6. Deleting test station {station_id}...")
+        response = requests.delete(f"{base_url}/stations/{station_id}")
+        if response.status_code == 200:
+            result = response.json()
+            print(f"✅ Deleted station: {result['message']}")
+        else:
+            print(f"❌ Failed to delete station: {response.status_code}")
+        
+        # 7. Verify station was deleted
+        print("\n7. Verifying station deletion...")
+        response = requests.get(f"{base_url}/stations/{station_id}")
+        if response.status_code == 404:
+            print("✅ Station successfully deleted (404 as expected)")
+        else:
+            print(f"⚠️ Station still exists: {response.status_code}")
+        
+        # 8. Final station count check
+        response = requests.get(f"{base_url}/stations")
+        if response.status_code == 200:
+            stations = response.json()
+            final_count = len(stations)
+            if final_count == initial_count:
+                print(f"✅ Station count restored to original: {final_count}")
+            else:
+                print(f"⚠️ Station count mismatch: {final_count} (expected {initial_count})")
+        
+        print("\n🎉 Station management tests completed!")
+        
+    except requests.exceptions.ConnectionError:
+        print("❌ Connection failed - is the API server running?")
+        print("Start it with: python run.py --web-api")
+    except Exception as e:
+        print(f"❌ Test failed with error: {e}")
+
+if __name__ == "__main__":
+    test_station_management()