fix: trigger CI on master (repo default branch), drop unsupported Python matrix entries

Workflows listened on 'main'/'develop' but the repo's default branch is
master, so push events never started a job (every historical run is a
schedule event). Point push/PR triggers and the deploy-gate refs at
master, and trim the test matrix to 3.11/3.12 to match
requires-python >=3.11.

.env.back

@@ -0,0 +1,87 @@
+# Northern Thailand Ping River Monitor Configuration
+# Copy this file to .env and customize for your environment
+
+# Database Configuration
+DB_TYPE=postgresql
+# Options: sqlite, mysql, postgresql, influxdb, victoriametrics
+
+# SQLite Configuration (default)
+WATER_DB_PATH=water_levels.db
+
+# VictoriaMetrics Configuration
+VM_HOST=localhost
+VM_PORT=8428
+VM_URL=
+
+# InfluxDB Configuration
+INFLUX_HOST=localhost
+INFLUX_PORT=8086
+INFLUX_DATABASE=ping_river_monitoring
+INFLUX_USERNAME=
+INFLUX_PASSWORD=
+
+# PostgreSQL Configuration (Remote Server)
+# Option 1: Full connection string (URL encode special characters in password)
+#POSTGRES_CONNECTION_STRING=postgresql://username:url_encoded_password@your-postgres-host:5432/water_monitoring
+
+# Option 2: Individual components (password will be automatically URL encoded)
+POSTGRES_HOST=10.0.10.201
+POSTGRES_PORT=5432
+POSTGRES_DB=ping_river
+POSTGRES_USER=ping_river
+POSTGRES_PASSWORD=3_%m]k:+16"rx?M#`swIA
+
+# Examples for connection string:
+# - Local: postgresql://postgres:password@localhost:5432/water_monitoring
+# - Remote: postgresql://user:pass@192.168.1.100:5432/water_monitoring
+# - With special chars: postgresql://user:my%3Apass%40word@host:5432/db
+# - With SSL: postgresql://user:pass@host:port/db?sslmode=require
+# - Connection pooling: postgresql://user:pass@host:port/db?pool_size=20&max_overflow=0
+
+# Special character URL encoding:
+# : → %3A    @ → %40    # → %23    ? → %3F    & → %26    / → %2F    % → %25
+
+# MySQL Configuration
+MYSQL_CONNECTION_STRING=mysql://user:password@localhost:3306/ping_river_monitoring
+
+# API Configuration
+API_HOST=0.0.0.0
+API_PORT=8000
+API_WORKERS=1
+
+# Data Collection Settings
+SCRAPING_INTERVAL_HOURS=1
+REQUEST_TIMEOUT=30
+MAX_RETRIES=3
+RETRY_DELAY_SECONDS=60
+
+# Data Retention
+DATA_RETENTION_DAYS=365
+
+# Logging Configuration
+LOG_LEVEL=INFO
+LOG_FILE=water_monitor.log
+
+# Security (for production)
+SECRET_KEY=your-secret-key-here
+API_KEY=your-api-key-here
+
+# Monitoring
+ENABLE_METRICS=true
+ENABLE_HEALTH_CHECKS=true
+
+# Geographic Settings
+TIMEZONE=Asia/Bangkok
+DEFAULT_LATITUDE=18.7875
+DEFAULT_LONGITUDE=99.0045
+
+# External Services
+NOTIFICATION_EMAIL=
+SMTP_SERVER=
+SMTP_PORT=587
+SMTP_USERNAME=
+SMTP_PASSWORD=
+
+# Development Settings
+DEBUG=false
+DEVELOPMENT_MODE=false

.env.example

@@ -2,7 +2,7 @@
 # Copy this file to .env and customize for your environment
 
 # Database Configuration
-DB_TYPE=sqlite
+DB_TYPE=postgresql
 # Options: sqlite, mysql, postgresql, influxdb, victoriametrics
 
 # SQLite Configuration (default)
@@ -20,8 +20,26 @@ INFLUX_DATABASE=ping_river_monitoring
 INFLUX_USERNAME=
 INFLUX_PASSWORD=
 
-# PostgreSQL Configuration
-POSTGRES_CONNECTION_STRING=postgresql://user:password@localhost:5432/ping_river_monitoring
+# PostgreSQL Configuration (Remote Server)
+# Option 1: Full connection string (URL encode special characters in password)
+POSTGRES_CONNECTION_STRING=postgresql://username:url_encoded_password@your-postgres-host:5432/water_monitoring
+
+# Option 2: Individual components (password will be automatically URL encoded)
+POSTGRES_HOST=your-postgres-host
+POSTGRES_PORT=5432
+POSTGRES_DB=water_monitoring
+POSTGRES_USER=username
+POSTGRES_PASSWORD=your:password@with!special#chars
+
+# Examples for connection string:
+# - Local: postgresql://postgres:password@localhost:5432/water_monitoring
+# - Remote: postgresql://user:pass@192.168.1.100:5432/water_monitoring
+# - With special chars: postgresql://user:my%3Apass%40word@host:5432/db
+# - With SSL: postgresql://user:pass@host:port/db?sslmode=require
+# - Connection pooling: postgresql://user:pass@host:port/db?pool_size=20&max_overflow=0
+
+# Special character URL encoding:
+# : → %3A    @ → %40    # → %23    ? → %3F    & → %26    / → %2F    % → %25
 
 # MySQL Configuration
 MYSQL_CONNECTION_STRING=mysql://user:password@localhost:3306/ping_river_monitoring
@@ -30,6 +48,8 @@ MYSQL_CONNECTION_STRING=mysql://user:password@localhost:3306/ping_river_monitori
 API_HOST=0.0.0.0
 API_PORT=8000
 API_WORKERS=1
+# Public ThaiWater API key used to add Ping-basin water-level sensors.
+THAIWATER_API_KEY=
 
 # Data Collection Settings
 SCRAPING_INTERVAL_HOURS=1
@@ -64,6 +84,18 @@ SMTP_PORT=587
 SMTP_USERNAME=
 SMTP_PASSWORD=
 
+# Matrix Alerting Configuration
+MATRIX_HOMESERVER=https://matrix.org
+MATRIX_ACCESS_TOKEN=
+MATRIX_ROOM_ID=
+
+# Grafana Integration
+GRAFANA_URL=http://localhost:3000
+
+# Alert Configuration
+ALERT_MAX_AGE_HOURS=2
+ALERT_CHECK_INTERVAL_MINUTES=15
+
 # Development Settings
 DEBUG=false
 DEVELOPMENT_MODE=false

.env.postgres

@@ -0,0 +1,2 @@
+DB_TYPE=postgresql
+POSTGRES_CONNECTION_STRING=postgresql://postgres:password@localhost:5432/water_monitoring

.gitea/workflows/ci.yml

@@ -2,9 +2,9 @@ name: CI/CD Pipeline - Northern Thailand Ping River Monitor
 
 on:
   push:
-    branches: [ main, develop ]
+    branches: [ master, develop ]
   pull_request:
-    branches: [ main ]
+    branches: [ master ]
   schedule:
     # Run tests daily at 2 AM UTC
     - cron: '0 2 * * *'
@@ -23,11 +23,13 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.11', '3.12']
     
     steps:
     - name: Checkout code
-      uses: https://gitea.com/actions/checkout
+      uses: actions/checkout@v4
+      with:
+        token: ${{ secrets.GITEA_TOKEN }}
       
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4
@@ -97,7 +99,9 @@ jobs:
     
     steps:
     - name: Checkout code
-      uses: https://gitea.com/actions/checkout
+      uses: actions/checkout@v4
+      with:
+        token: ${{ secrets.GITEA_TOKEN }}
       
     - name: Set up Python
       uses: actions/setup-python@v4
@@ -133,7 +137,9 @@ jobs:
     
     steps:
     - name: Checkout code
-      uses: https://gitea.com/actions/checkout
+      uses: actions/checkout@v4
+      with:
+        token: ${{ secrets.GITEA_TOKEN }}
       
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v3
@@ -192,7 +198,9 @@ jobs:
           
     steps:
     - name: Checkout code
-      uses: https://gitea.com/actions/checkout
+      uses: actions/checkout@v4
+      with:
+        token: ${{ secrets.GITEA_TOKEN }}
       
     - name: Wait for VictoriaMetrics
       run: |
@@ -243,7 +251,9 @@ jobs:
       
     steps:
     - name: Checkout code
-      uses: https://gitea.com/actions/checkout
+      uses: actions/checkout@v4
+      with:
+        token: ${{ secrets.GITEA_TOKEN }}
       
     - name: Deploy to staging
       run: |
@@ -261,14 +271,16 @@ jobs:
     name: Deploy to Production
     runs-on: ubuntu-latest
     needs: [test, build, integration-test]
-    if: github.ref == 'refs/heads/main'
+    if: github.ref == 'refs/heads/master'
     environment:
       name: production
       url: https://ping-river-monitor.b4l.co.th
       
     steps:
     - name: Checkout code
-      uses: https://gitea.com/actions/checkout
+      uses: actions/checkout@v4
+      with:
+        token: ${{ secrets.GITEA_TOKEN }}
       
     - name: Deploy to production
       run: |
@@ -291,11 +303,13 @@ jobs:
     name: Performance Test
     runs-on: ubuntu-latest
     needs: deploy-production
-    if: github.ref == 'refs/heads/main'
+    if: github.ref == 'refs/heads/master'
     
     steps:
     - name: Checkout code
-      uses: https://gitea.com/actions/checkout
+      uses: actions/checkout@v4
+      with:
+        token: ${{ secrets.GITEA_TOKEN }}
       
     - name: Install Apache Bench
       run: |

.gitea/workflows/docs.yml

@@ -2,7 +2,7 @@ name: Documentation
 
 on:
   push:
-    branches: [ main, develop ]
+    branches: [ master, develop ]
     paths:
       - 'docs/**'
       - 'README.md'
@@ -26,7 +26,9 @@ jobs:
     
     steps:
     - name: Checkout code
-      uses: https://gitea.com/actions/checkout
+      uses: actions/checkout@v4
+      with:
+        token: ${{ secrets.GITEA_TOKEN }}
       
     - name: Set up Python
       uses: actions/setup-python@v4
@@ -126,7 +128,9 @@ jobs:
     
     steps:
     - name: Checkout code
-      uses: https://gitea.com/actions/checkout
+      uses: actions/checkout@v4
+      with:
+        token: ${{ secrets.GITEA_TOKEN }}
       
     - name: Set up Python
       uses: actions/setup-python@v4
@@ -223,7 +227,9 @@ jobs:
     
     steps:
     - name: Checkout code
-      uses: https://gitea.com/actions/checkout
+      uses: actions/checkout@v4
+      with:
+        token: ${{ secrets.GITEA_TOKEN }}
       
     - name: Set up Python
       uses: actions/setup-python@v4
@@ -248,8 +254,8 @@ jobs:
         project = 'Northern Thailand Ping River Monitor'
         copyright = '2025, Ping River Monitor Team'
         author = 'Ping River Monitor Team'
-        version = '3.1.1'
-        release = '3.1.1'
+        version = '3.1.3'
+        release = '3.1.3'
         
         extensions = [
             'sphinx.ext.autodoc',

.gitea/workflows/release.yml

@@ -3,16 +3,16 @@ name: Release - Northern Thailand Ping River Monitor
 on:
   push:
     tags:
-      - 'v*.*.*'
+      - "v*.*.*"
   workflow_dispatch:
     inputs:
       version:
-        description: 'Release version (e.g., v3.1.1)'
+        description: "Release version (e.g., v3.1.3)"
         required: true
         type: string
 
 env:
-  PYTHON_VERSION: '3.11'
+  PYTHON_VERSION: "3.11"
   REGISTRY: git.b4l.co.th
   IMAGE_NAME: b4l/northern-thailand-ping-river-monitor
   # GitHub token for better rate limits and authentication
@@ -28,8 +28,9 @@ jobs:
 
     steps:
       - name: Checkout code
-      uses: https://gitea.com/actions/checkout
+        uses: actions/checkout@v4
         with:
+          token: ${{ secrets.GITEA_TOKEN }}
           fetch-depth: 0
 
       - name: Get version
@@ -70,11 +71,13 @@ jobs:
     needs: create-release
     strategy:
       matrix:
-        python-version: ['3.9', '3.10', '3.11', '3.12']
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     steps:
       - name: Checkout code
-      uses: https://gitea.com/actions/checkout
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GITEA_TOKEN }}
 
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
@@ -112,7 +115,9 @@ jobs:
 
     steps:
       - name: Checkout code
-      uses: https://gitea.com/actions/checkout
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GITEA_TOKEN }}
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
@@ -121,8 +126,8 @@ jobs:
         uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
-        username: ${{ github.actor }}
-        password: ${{ secrets.GITEA_TOKEN }}
+          username: ${{ vars.WORKER_USERNAME}}
+          password: ${{ secrets.CI_BOT_TOKEN }}
 
       - name: Build and push release images
         uses: docker/build-push-action@v5
@@ -142,7 +147,7 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
         env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITEA_TOKEN }}
 
   # Security scan for release
   security-scan:
@@ -152,145 +157,158 @@ jobs:
 
     steps:
       - name: Checkout code
-      uses: https://gitea.com/actions/checkout
-      
-    - name: Run Trivy vulnerability scanner
-      uses: aquasecurity/trivy-action@master
+        uses: actions/checkout@v4
         with:
-        image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.create-release.outputs.version }}
-        format: 'sarif'
-        output: 'trivy-results.sarif'
-        github-token: ${{ secrets.GH_TOKEN }}
-      env:
-        GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
+          token: ${{ secrets.GITEA_TOKEN}}
 
-    - name: Upload Trivy scan results
-      uses: actions/upload-artifact@v3
-      with:
-        name: security-scan-results
-        path: trivy-results.sarif
-
-  # Deploy release to production
+  # Test release deployment locally
   deploy-release:
-    name: Deploy Release
+    name: Test Release Deployment
     runs-on: ubuntu-latest
     needs: [create-release, build-release, security-scan]
     environment:
-      name: production
-      url: https://ping-river-monitor.b4l.co.th
+      name: testing
+      url: http://localhost:8080
 
     steps:
       - name: Checkout code
-      uses: https://gitea.com/actions/checkout
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GITEA_TOKEN }}
 
-    - name: Deploy to production
+      - name: Log in to Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ vars.WORKER_USERNAME}}
+          password: ${{ secrets.CI_BOT_TOKEN }}
+
+      - name: Deploy to production (Local Test)
         run: |
-        echo "🚀 Deploying ${{ needs.create-release.outputs.version }} to production..."
+          set -euo pipefail
+          echo "🚀 Testing ${{ needs.create-release.outputs.version }} deployment locally..."
 
-        # Example deployment commands (customize for your infrastructure)
-        # kubectl set image deployment/ping-river-monitor app=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.create-release.outputs.version }}
-        # docker-compose pull && docker-compose up -d
-        # Or webhook call to your deployment system
+          # Create a dedicated network so we can resolve by container name
+          docker network create ci_net || true
 
-        echo "✅ Deployment initiated"
+          # Pull the built image
+          docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.create-release.outputs.version }}
+
+          # Stop & remove any existing container
+          docker rm -f ping-river-monitor-test 2>/dev/null || true
+
+          # Start the container on the user-defined network
+          docker run -d \
+            --name ping-river-monitor-test \
+            --network ci_net \
+            -p 8080:8000 \
+            -e LOG_LEVEL=INFO \
+            -e DB_TYPE=sqlite \
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.create-release.outputs.version }}
+
+          echo "✅ Container started for testing"
 
       - name: Health check after deployment
         run: |
-        echo "⏳ Waiting for deployment to stabilize..."
-        sleep 60
+          set -euo pipefail
+          echo "⏳ Waiting for application to start..."
 
-        echo "🔍 Running health checks..."
-        curl -f https://ping-river-monitor.b4l.co.th/health
-        curl -f https://ping-river-monitor.b4l.co.th/stations
+          # Pull a curl-only image for probing (keeps your app image slim)
+          docker pull curlimages/curl:8.10.1
 
-        echo "✅ Health checks passed!"
+          # Helper: curl via a sibling container on the SAME Docker network
+          probe() {
+            local url="$1"
+            docker run --rm --network ci_net curlimages/curl:8.10.1 \
+              -sS --max-time 5 --connect-timeout 3 -w "HTTP_CODE:%{http_code}" "$url" || true
+          }
 
-    - name: Update deployment status
-      run: |
-        echo "📊 Deployment Summary:"
-        echo "Version: ${{ needs.create-release.outputs.version }}"
-        echo "Image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.create-release.outputs.version }}"
-        echo "URL: https://ping-river-monitor.b4l.co.th"
-        echo "Grafana: https://grafana.ping-river-monitor.b4l.co.th"
-        echo "API Docs: https://ping-river-monitor.b4l.co.th/docs"
+          # Wait for /health (up to ~3m 45s)
+          for i in {1..15}; do
+            echo "🔍 Attempt $i/15: checking http://ping-river-monitor-test:8000/health"
+            resp="$(probe http://ping-river-monitor-test:8000/health)"
+            code="$(echo "$resp" | sed -n 's/.*HTTP_CODE:\([0-9]\+\).*/\1/p')"
+            body="$(echo "$resp" | sed 's/HTTP_CODE:[0-9]*$//')"
             
-  # Post-release validation
-  validate-release:
-    name: Validate Release
-    runs-on: ubuntu-latest
-    needs: deploy-release
+            echo "HTTP: ${code:-<none>} | Body: ${body:-<empty>}"
             
-    steps:
-    - name: Comprehensive API test
-      run: |
-        echo "🧪 Running comprehensive API tests..."
+            if [ "${code:-}" = "200" ] && [ -n "${body:-}" ]; then
+              echo "✅ Health endpoint responding successfully"
+              break
+            fi
             
-        # Test all major endpoints
-        curl -f https://ping-river-monitor.b4l.co.th/health
-        curl -f https://ping-river-monitor.b4l.co.th/metrics
-        curl -f https://ping-river-monitor.b4l.co.th/stations
-        curl -f https://ping-river-monitor.b4l.co.th/measurements/latest?limit=5
-        curl -f https://ping-river-monitor.b4l.co.th/scraping/status
+            echo "❌ Not ready yet. Showing recent logs…"
+            docker logs --tail 20 ping-river-monitor-test || true
+            sleep 15
             
-        echo "✅ All API endpoints responding correctly"
-        
-    - name: Performance validation
-      run: |
-        echo "⚡ Running performance validation..."
-        
-        # Install Apache Bench
-        sudo apt-get update && sudo apt-get install -y apache2-utils
-        
-        # Test response times
-        ab -n 10 -c 2 https://ping-river-monitor.b4l.co.th/health
-        ab -n 10 -c 2 https://ping-river-monitor.b4l.co.th/stations
-        
-        echo "✅ Performance validation completed"
-        
-    - name: Data validation
-      run: |
-        echo "📊 Validating data collection..."
-        
-        # Check if recent data is available
-        response=$(curl -s https://ping-river-monitor.b4l.co.th/measurements/latest?limit=1)
-        echo "Latest measurement: $response"
-        
-        # Validate data structure (basic check)
-        if echo "$response" | grep -q "water_level"; then
-          echo "✅ Data structure validation passed"
-        else
-          echo "❌ Data structure validation failed"
+            if [ "$i" -eq 15 ]; then
+              echo "❌ Health never reached 200. Failing."
               exit 1
             fi
+          done
+
+          echo "🧪 Testing API endpoints…"
+          endpoints=("health" "docs" "stations" "metrics")
+          for ep in "${endpoints[@]}"; do
+            url="http://ping-river-monitor-test:8000/$ep"
+            resp="$(probe "$url")"
+            code="$(echo "$resp" | sed -n 's/.*HTTP_CODE:\([0-9]\+\).*/\1/p')"
+            
+            if [ "${code:-}" = "200" ]; then
+              echo "✅ /$ep: OK"
+            else
+              echo "❌ /$ep: FAILED (HTTP ${code:-<none>})"
+              echo "Response: $(echo "$resp" | sed 's/HTTP_CODE:[0-9]*$//')"
+              exit 1
+            fi
+          done
+
+          echo "✅ All health checks passed!"
+
+      - name: Container logs and cleanup
+        if: always()
+        run: |
+          echo "📋 Container logs:"
+          docker logs ping-river-monitor-test || true
+
+          echo "🧹 Cleaning up test container..."
+          docker stop ping-river-monitor-test || true
+          docker rm ping-river-monitor-test || true
+
+          echo "📊 Deployment Test Summary:"
+          echo "Version: ${{ needs.create-release.outputs.version }}"
+          echo "Image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.create-release.outputs.version }}"
+          echo "Status: Container tested successfully"
+          echo "Ready for production deployment"
 
   # Notify stakeholders
   notify:
     name: Notify Release
     runs-on: ubuntu-latest
-    needs: [create-release, validate-release]
+    needs: [create-release, deploy-release]
     if: always()
 
     steps:
       - name: Notify success
-      if: needs.validate-release.result == 'success'
+        if: needs.deploy-release.result == 'success'
         run: |
-        echo "🎉 Release ${{ needs.create-release.outputs.version }} deployed successfully!"
-        echo "🌐 Production URL: https://ping-river-monitor.b4l.co.th"
-        echo "📊 Grafana: https://grafana.ping-river-monitor.b4l.co.th"
-        echo "📚 API Docs: https://ping-river-monitor.b4l.co.th/docs"
+          echo "🎉 Release ${{ needs.create-release.outputs.version }} tested successfully!"
+          echo "🧪 Local Test: Passed all health checks"
+          echo "� GDocker Image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.create-release.outputs.version }}"
+          echo "✅ Ready for production deployment"
 
           # Add notification to Slack, Discord, email, etc.
           # curl -X POST -H 'Content-type: application/json' \
-        #   --data '{"text":"🎉 Northern Thailand Ping River Monitor ${{ needs.create-release.outputs.version }} deployed successfully!"}' \
+          #   --data '{"text":"🎉 Northern Thailand Ping River Monitor ${{ needs.create-release.outputs.version }} tested and ready for deployment!"}' \
           #   ${{ secrets.SLACK_WEBHOOK_URL }}
 
       - name: Notify failure
-      if: needs.validate-release.result == 'failure'
+        if: needs.deploy-release.result == 'failure'
         run: |
-        echo "❌ Release ${{ needs.create-release.outputs.version }} deployment failed!"
-        echo "Please check the logs and take corrective action."
+          echo "❌ Release ${{ needs.create-release.outputs.version }} testing failed!"
+          echo "Please check the logs and fix issues before production deployment."
 
           # Add failure notification
           # curl -X POST -H 'Content-type: application/json' \
-        #   --data '{"text":"❌ Northern Thailand Ping River Monitor ${{ needs.create-release.outputs.version }} deployment failed!"}' \
+          #   --data '{"text":"❌ Northern Thailand Ping River Monitor ${{ needs.create-release.outputs.version }} testing failed!"}' \
           #   ${{ secrets.SLACK_WEBHOOK_URL }}

.gitea/workflows/security.yml

@@ -24,7 +24,9 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: https://gitea.com/actions/checkout
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GITEA_TOKEN }}
 
       - name: Set up Python
         uses: actions/setup-python@v4
@@ -61,16 +63,16 @@ jobs:
 
       - name: Check for critical vulnerabilities
         run: |
-          echo "🔍 Checking for critical vulnerabilities..."
+          echo "Checking for critical vulnerabilities..."
 
           # Check Safety results
           if [ -f safety-report.json ]; then
             critical_count=$(jq '.vulnerabilities | length' safety-report.json 2>/dev/null || echo "0")
             if [ "$critical_count" -gt 0 ]; then
-              echo "⚠️ Found $critical_count dependency vulnerabilities"
+              echo "Found $critical_count dependency vulnerabilities"
               jq '.vulnerabilities[] | "- \(.package_name) \(.installed_version): \(.vulnerability_id)"' safety-report.json
             else
-              echo "✅ No dependency vulnerabilities found"
+              echo "No dependency vulnerabilities found"
             fi
           fi
 
@@ -78,86 +80,9 @@ jobs:
           if [ -f bandit-report.json ]; then
             high_severity=$(jq '.results[] | select(.issue_severity == "HIGH") | length' bandit-report.json 2>/dev/null | wc -l)
             if [ "$high_severity" -gt 0 ]; then
-              echo "⚠️ Found $high_severity high-severity security issues"
+              echo "Found $high_severity high-severity security issues"
             else
-              echo "✅ No high-severity security issues found"
-            fi
-          fi
-
-  # Docker image security scan
-  docker-security-scan:
-    name: Docker Security Scan
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout code
-        uses: https://gitea.com/actions/checkout
-
-      - name: Check GitHub token availability
-        run: |
-          if [ -z "${{ secrets.GH_TOKEN }}" ]; then
-            echo "⚠️ GH_TOKEN not configured. Trivy scans may fail due to rate limits."
-            echo "💡 To fix: Add GH_TOKEN secret in repository settings"
-          else
-            echo "✅ GH_TOKEN is configured"
-          fi
-
-      - name: Build Docker image for scanning
-        run: |
-          docker build -t ping-river-monitor:scan .
-        env:
-          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
-
-      - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@master
-        with:
-          image-ref: "ping-river-monitor:scan"
-          format: "json"
-          output: "trivy-report.json"
-          github-token: ${{ secrets.GH_TOKEN }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
-        continue-on-error: true
-
-      - name: Run Trivy filesystem scan
-        uses: aquasecurity/trivy-action@master
-        with:
-          scan-type: "fs"
-          scan-ref: "."
-          format: "json"
-          output: "trivy-fs-report.json"
-          github-token: ${{ secrets.GH_TOKEN }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
-        continue-on-error: true
-
-      - name: Upload Trivy reports
-        uses: actions/upload-artifact@v3
-        if: always()
-        with:
-          name: trivy-reports-${{ github.run_number }}
-          path: |
-            trivy-report.json
-            trivy-fs-report.json
-
-      - name: Check Trivy results
-        run: |
-          echo "🔍 Analyzing Docker security scan results..."
-
-          if [ -f trivy-report.json ]; then
-            critical_vulns=$(jq '.Results[]?.Vulnerabilities[]? | select(.Severity == "CRITICAL") | length' trivy-report.json 2>/dev/null | wc -l)
-            high_vulns=$(jq '.Results[]?.Vulnerabilities[]? | select(.Severity == "HIGH") | length' trivy-report.json 2>/dev/null | wc -l)
-            
-            echo "Critical vulnerabilities: $critical_vulns"
-            echo "High vulnerabilities: $high_vulns"
-            
-            if [ "$critical_vulns" -gt 0 ]; then
-              echo "❌ Critical vulnerabilities found in Docker image!"
-              exit 1
-            elif [ "$high_vulns" -gt 5 ]; then
-              echo "⚠️ Many high-severity vulnerabilities found"
-            else
-              echo "✅ Docker image security scan passed"
+              echo "No high-severity security issues found"
             fi
           fi
 
@@ -168,7 +93,9 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: https://gitea.com/actions/checkout
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GITEA_TOKEN }}
 
       - name: Set up Python
         uses: actions/setup-python@v4
@@ -183,7 +110,7 @@ jobs:
 
       - name: Check licenses
         run: |
-          echo "📄 Checking dependency licenses..."
+          echo "Checking dependency licenses..."
           pip-licenses --format=json --output-file=licenses.json
           pip-licenses --format=markdown --output-file=licenses.md
 
@@ -192,11 +119,11 @@ jobs:
 
           for license in "${problematic_licenses[@]}"; do
             if grep -i "$license" licenses.json; then
-              echo "⚠️ Found potentially problematic license: $license"
+              echo "Found potentially problematic license: $license"
             fi
           done
 
-          echo "✅ License check completed"
+          echo "License check completed"
 
       - name: Upload license report
         uses: actions/upload-artifact@v3
@@ -213,7 +140,9 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: https://gitea.com/actions/checkout
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GITEA_TOKEN }}
 
       - name: Set up Python
         uses: actions/setup-python@v4
@@ -227,56 +156,15 @@ jobs:
 
       - name: Check for outdated packages
         run: |
-          echo "📦 Checking for outdated packages..."
+          echo "Checking for outdated packages..."
           pip install --root-user-action=ignore -r requirements.txt
           pip list --outdated --format=json > outdated-packages.json || true
 
           if [ -s outdated-packages.json ]; then
-            echo "📋 Outdated packages found:"
+            echo "Outdated packages found:"
             cat outdated-packages.json | jq -r '.[] | "- \(.name): \(.version) -> \(.latest_version)"'
           else
-            echo "✅ All packages are up to date"
-          fi
-
-      - name: Create dependency update issue
-        if: github.event_name == 'schedule'
-        run: |
-          if [ -s outdated-packages.json ] && [ "$(cat outdated-packages.json)" != "[]" ]; then
-            echo "📝 Creating dependency update issue..."
-            
-            # Create issue body
-            cat > issue-body.md << 'EOF'
-          ## 📦 Dependency Updates Available
-
-          The following packages have updates available:
-
-          EOF
-            
-            cat outdated-packages.json | jq -r '.[] | "- **\(.name)**: \(.version) → \(.latest_version)"' >> issue-body.md
-            
-            cat >> issue-body.md << 'EOF'
-
-          ## 🔍 Security Impact
-
-          Please review each update for:
-          - Security fixes
-          - Breaking changes
-          - Compatibility issues
-
-          ## ✅ Action Items
-
-          - [ ] Review changelog for each package
-          - [ ] Test updates in development environment
-          - [ ] Update requirements.txt
-          - [ ] Run full test suite
-          - [ ] Deploy to staging for validation
-
-          ---
-          *This issue was automatically created by the security workflow.*
-          EOF
-            
-            echo "Issue body created. In a real implementation, you would create a Gitea issue here."
-            cat issue-body.md
+            echo "All packages are up to date"
           fi
 
       - name: Upload dependency reports
@@ -285,7 +173,6 @@ jobs:
           name: dependency-reports-${{ github.run_number }}
           path: |
             outdated-packages.json
-            issue-body.md
 
   # Code quality metrics
   code-quality:
@@ -294,7 +181,9 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: https://gitea.com/actions/checkout
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GITEA_TOKEN }}
 
       - name: Set up Python
         uses: actions/setup-python@v4
@@ -309,24 +198,24 @@ jobs:
 
       - name: Calculate code complexity
         run: |
-          echo "📊 Calculating code complexity..."
+          echo "Calculating code complexity..."
           radon cc src/ --json > complexity-report.json
           radon mi src/ --json > maintainability-report.json
 
-          echo "🔍 Complexity Summary:"
+          echo "Complexity Summary:"
           radon cc src/ --average
 
-          echo "🔧 Maintainability Summary:"
+          echo "Maintainability Summary:"
           radon mi src/
 
       - name: Find dead code
         run: |
-          echo "🧹 Checking for dead code..."
+          echo "Checking for dead code..."
           vulture src/ --json > dead-code-report.json || true
 
       - name: Check for code smells
         run: |
-          echo "👃 Checking for code smells..."
+          echo "Checking for code smells..."
           xenon --max-absolute B --max-modules A --max-average A src/ || true
 
       - name: Upload quality reports
@@ -342,7 +231,7 @@ jobs:
   security-summary:
     name: Security Summary
     runs-on: ubuntu-latest
-    needs: [dependency-scan, docker-security-scan, license-check, code-quality]
+    needs: [dependency-scan, license-check, code-quality]
     if: always()
 
     steps:
@@ -351,51 +240,47 @@ jobs:
 
       - name: Generate security summary
         run: |
-          echo "# 🔒 Security Scan Summary" > security-summary.md
+          echo "# Security Scan Summary" > security-summary.md
           echo "" >> security-summary.md
           echo "**Scan Date:** $(date -u)" >> security-summary.md
           echo "**Repository:** ${{ github.repository }}" >> security-summary.md
           echo "**Commit:** ${{ github.sha }}" >> security-summary.md
           echo "" >> security-summary.md
 
-          echo "## 📊 Results" >> security-summary.md
+          echo "## Results" >> security-summary.md
           echo "" >> security-summary.md
 
           # Dependency scan results
           if [ -f security-reports-*/safety-report.json ]; then
             vuln_count=$(jq '.vulnerabilities | length' security-reports-*/safety-report.json 2>/dev/null || echo "0")
             if [ "$vuln_count" -eq 0 ]; then
-              echo "- ✅ **Dependency Scan**: No vulnerabilities found" >> security-summary.md
+              echo "- Dependency Scan: No vulnerabilities found" >> security-summary.md
             else
-              echo "- ⚠️ **Dependency Scan**: $vuln_count vulnerabilities found" >> security-summary.md
+              echo "- Dependency Scan: $vuln_count vulnerabilities found" >> security-summary.md
             fi
           else
-            echo "- ❓ **Dependency Scan**: Results not available" >> security-summary.md
+            echo "- Dependency Scan: Results not available" >> security-summary.md
           fi
 
-          # Docker scan results
-          if [ -f trivy-reports-*/trivy-report.json ]; then
-            echo "- ✅ **Docker Scan**: Completed" >> security-summary.md
-          else
-            echo "- ❓ **Docker Scan**: Results not available" >> security-summary.md
-          fi
+          # Docker scan results (removed Trivy)
+          echo "- Docker Scan: Skipped (Trivy removed)" >> security-summary.md
 
           # License check results
           if [ -f license-report-*/licenses.json ]; then
-            echo "- ✅ **License Check**: Completed" >> security-summary.md
+            echo "- License Check: Completed" >> security-summary.md
           else
-            echo "- ❓ **License Check**: Results not available" >> security-summary.md
+            echo "- License Check: Results not available" >> security-summary.md
           fi
 
           # Code quality results
           if [ -f code-quality-reports-*/complexity-report.json ]; then
-            echo "- ✅ **Code Quality**: Analyzed" >> security-summary.md
+            echo "- Code Quality: Analyzed" >> security-summary.md
           else
-            echo "- ❓ **Code Quality**: Results not available" >> security-summary.md
+            echo "- Code Quality: Results not available" >> security-summary.md
           fi
 
           echo "" >> security-summary.md
-          echo "## 🔗 Detailed Reports" >> security-summary.md
+          echo "## Detailed Reports" >> security-summary.md
           echo "" >> security-summary.md
           echo "Detailed reports are available in the workflow artifacts." >> security-summary.md
 

.gitignore

@@ -135,3 +135,16 @@ cython_debug/
 # Docker volumes
 vm_data/
 grafana_data/
+# Runtime station config (persisted CRUD); bundled default lives in src/data/
+/stations.json
+
+# Ruflo local secrets and runtime data
+.env.*.local
+.claude-flow/data/
+.claude-flow/logs/
+.claude-flow/sessions/
+
+# Trained flood-forecast model artifacts (produced on the server, ~100 MB; see docs/FLOOD_FORECASTING.md)
+models/*.joblib
+models/cache/
+models/metrics.json

.pre-commit-config.yaml

@@ -0,0 +1,40 @@
+# Pre-commit hooks for Northern Thailand Ping River Monitor
+# See https://pre-commit.com for more information
+
+repos:
+  # General file checks
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-json
+      - id: check-toml
+      - id: check-added-large-files
+        args: ['--maxkb=1000']
+      - id: check-merge-conflict
+      - id: check-case-conflict
+      - id: mixed-line-ending
+
+  # Python code formatting with Black
+  - repo: https://github.com/psf/black
+    rev: 23.11.0
+    hooks:
+      - id: black
+        language_version: python3
+        args: ['--line-length=120']
+
+  # Import sorting with isort
+  - repo: https://github.com/pycqa/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+        args: ['--profile', 'black', '--line-length', '120']
+
+  # Linting with flake8
+  - repo: https://github.com/pycqa/flake8
+    rev: 6.1.0
+    hooks:
+      - id: flake8
+        args: ['--max-line-length=120', '--extend-ignore=E203,W503']

DEPLOYMENT_CHECKLIST.md

@@ -259,7 +259,7 @@ make health-check
 
 **Deployment Date**: ___________  
 **Deployed By**: ___________  
-**Version**: v3.1.2  
+**Version**: v3.1.3  
 **Environment**: ___________  
 
 **Sign-off**:

Dockerfile

@@ -22,26 +22,27 @@ FROM python:3.11-slim
 # Set working directory
 WORKDIR /app
 
-# Install runtime dependencies
+# Install runtime dependencies and create user
 RUN apt-get update && apt-get install -y \
     wget \
     curl \
     && rm -rf /var/lib/apt/lists/* \
-    && groupadd -r appuser && useradd -r -g appuser appuser
+    && groupadd -r appuser && useradd -r -g appuser appuser \
+    && mkdir -p /home/appuser/.local
 
 # Copy Python packages from builder stage
-COPY --from=builder /root/.local /root/.local
+COPY --from=builder /root/.local /home/appuser/.local
 
 # Copy application code
 COPY . .
 
 # Create logs directory and set permissions
-RUN mkdir -p logs && chown -R appuser:appuser /app
+RUN mkdir -p logs && chown -R appuser:appuser /app /home/appuser/.local
 
 # Set environment variables
 ENV PYTHONUNBUFFERED=1
 ENV TZ=Asia/Bangkok
-ENV PATH=/root/.local/bin:$PATH
+ENV PATH=/home/appuser/.local/bin:$PATH
 
 # Switch to non-root user
 USER appuser

GITEA_SETUP_SUMMARY.md

@@ -222,12 +222,12 @@ Your repository is now equipped with:
 2. **Configure deployment environments** (staging/production)
 3. **Set up monitoring dashboards** for workflow metrics
 4. **Configure notifications** for team collaboration
-5. **Create your first release** with `git tag v3.1.2`
+5. **Create your first release** with `git tag v3.1.3`
 
 Your **Northern Thailand Ping River Monitor** is now ready for professional development and deployment! 🎊
 
 ---
 
-**Workflow Version**: v3.1.2  
+**Workflow Version**: v3.1.3  
 **Setup Date**: 2025-08-12  
 **Repository**: https://git.b4l.co.th/grabowski/Northern-Thailand-Ping-River-Monitor

MIGRATION_TO_UV.md

@@ -0,0 +1,165 @@
+# Migration to uv
+
+This document describes the migration from traditional Python package management (pip + requirements.txt) to [uv](https://docs.astral.sh/uv/), a fast Python package installer and resolver.
+
+## What Changed
+
+### Files Added
+- `pyproject.toml` - Modern Python project configuration combining dependencies and metadata
+- `.python-version` - Specifies Python version for uv
+- `scripts/setup_uv.sh` - Unix setup script for uv environment
+- `scripts/setup_uv.bat` - Windows setup script for uv environment
+- This migration guide
+
+### Files Modified
+- `Makefile` - Updated all commands to use `uv run` instead of direct Python execution
+
+### Files That Can Be Removed (Optional)
+- `requirements.txt` - Dependencies now in pyproject.toml
+- `requirements-dev.txt` - Dev dependencies now in pyproject.toml
+- `setup.py` - Configuration now in pyproject.toml
+
+## Installation
+
+### Install uv
+
+**Unix/macOS:**
+```bash
+curl -LsSf https://astral.sh/uv/install.sh | sh
+```
+
+**Windows (PowerShell):**
+```powershell
+powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"
+```
+
+### Setup Project
+
+**Unix/macOS:**
+```bash
+# Run the setup script
+chmod +x scripts/setup_uv.sh
+./scripts/setup_uv.sh
+
+# Or manually:
+uv sync
+uv run pre-commit install
+```
+
+**Windows:**
+```batch
+REM Run the setup script
+scripts\setup_uv.bat
+
+REM Or manually:
+uv sync
+uv run pre-commit install
+```
+
+## New Workflow
+
+### Common Commands
+
+| Old Command | New Command | Description |
+|-------------|-------------|-------------|
+| `pip install -r requirements.txt` | `uv sync --no-dev` | Install production dependencies |
+| `pip install -r requirements-dev.txt` | `uv sync` | Install all dependencies (including dev) |
+| `python run.py` | `uv run python run.py` | Run the application |
+| `pytest` | `uv run pytest` | Run tests |
+| `black src/` | `uv run black src/` | Format code |
+
+### Using the Makefile
+
+The Makefile has been updated to use uv, so all existing commands work the same:
+
+```bash
+make install-dev    # Install dev dependencies with uv
+make test          # Run tests with uv
+make run-api       # Start API server with uv
+make lint          # Lint code with uv
+make format        # Format code with uv
+```
+
+### Adding Dependencies
+
+**Production dependency:**
+```bash
+uv add requests
+```
+
+**Development dependency:**
+```bash
+uv add --dev pytest
+```
+
+**Specific version:**
+```bash
+uv add "fastapi==0.104.1"
+```
+
+### Managing Python Versions
+
+uv can automatically manage Python versions:
+
+```bash
+# Install and use Python 3.11
+uv python install 3.11
+uv sync
+
+# Use specific Python version
+uv sync --python 3.11
+```
+
+## Benefits of uv
+
+1. **Speed** - 10-100x faster than pip
+2. **Reliability** - Better dependency resolution
+3. **Simplicity** - Single tool for packages and Python versions
+4. **Reproducibility** - Lock file ensures consistent environments
+5. **Modern** - Built-in support for pyproject.toml
+
+## Troubleshooting
+
+### Command not found
+Make sure uv is in your PATH after installation. Restart your terminal or run:
+```bash
+source ~/.bashrc  # or ~/.zshrc
+```
+
+### Lock file conflicts
+If you encounter lock file issues:
+```bash
+rm uv.lock
+uv sync
+```
+
+### Python version issues
+Ensure the Python version in `.python-version` is available:
+```bash
+uv python list
+uv python install 3.11  # if needed
+```
+
+## Rollback (if needed)
+
+If you need to rollback to the old system:
+
+1. Use the original requirements files:
+   ```bash
+   pip install -r requirements.txt
+   pip install -r requirements-dev.txt
+   ```
+
+2. Revert the Makefile changes to use `python` instead of `uv run python`
+
+3. Remove uv-specific files:
+   ```bash
+   rm pyproject.toml .python-version uv.lock
+   rm -rf .venv  # if created by uv
+   ```
+
+## Additional Resources
+
+- [uv Documentation](https://docs.astral.sh/uv/)
+- [Migration Guide](https://docs.astral.sh/uv/guides/projects/)
+- [pyproject.toml Reference](https://packaging.python.org/en/latest/specifications/pyproject-toml/)

Makefile

@@ -21,39 +21,55 @@ help:
 	@echo "  run          Run the monitor in continuous mode"
 	@echo "  run-api      Run the web API server"
 	@echo "  run-test     Run a single test cycle"
+	@echo "  run-status   Show system status"
+	@echo ""
+	@echo "Alerting:"
+	@echo "  alert-check  Check water levels and send alerts"
+	@echo "  alert-test   Send test Matrix message"
+	@echo ""
+	@echo "Distribution:"
+	@echo "  build-exe    Build standalone executable"
+	@echo "  package      Build and create distribution package"
 	@echo ""
 	@echo "Docker:"
 	@echo "  docker-build Build Docker image"
 	@echo "  docker-run   Run with Docker Compose"
 	@echo "  docker-stop  Stop Docker services"
 	@echo ""
+	@echo "Database:"
+	@echo "  setup-postgres Setup PostgreSQL database"
+	@echo "  test-postgres  Test PostgreSQL connection"
+	@echo "  encode-password URL encode password for connection string"
+	@echo "  migrate-sqlite Migrate SQLite data to PostgreSQL"
+	@echo "  migrate-fast   Fast migration with 10K batch size"
+	@echo "  analyze-sqlite Analyze SQLite database structure (dry run)"
+	@echo ""
 	@echo "Documentation:"
 	@echo "  docs         Generate documentation"
 
 # Installation
 install:
-	pip install -r requirements.txt
+	uv sync --no-dev
 
 install-dev:
-	pip install -r requirements-dev.txt
-	pre-commit install
+	uv sync
+	uv run pre-commit install
 
 # Testing
 test:
-	python test_integration.py
-	python test_station_management.py
+	uv run pytest -q
 
 test-cov:
-	pytest --cov=src --cov-report=html --cov-report=term
+	uv run pytest --cov=src --cov-report=html --cov-report=term
 
 # Code quality
 lint:
-	flake8 src/ --max-line-length=100
-	mypy src/
+	uv run flake8 src/ --max-line-length=100
+	uv run mypy src/
 
 format:
-	black src/ *.py
-	isort src/ *.py
+	uv run black src/ *.py
+	uv run isort src/ *.py
 
 # Cleanup
 clean:
@@ -69,16 +85,23 @@ clean:
 
 # Running
 run:
-	python run.py
+	uv run python run.py
 
 run-api:
-	python run.py --web-api
+	uv run python run.py --web-api
 
 run-test:
-	python run.py --test
+	uv run python run.py --test
 
 run-status:
-	python run.py --status
+	uv run python run.py --status
+
+# Alerting
+alert-check:
+	uv run python run.py --alert-check
+
+alert-test:
+	uv run python run.py --alert-test
 
 # Docker
 docker-build:
@@ -99,7 +122,7 @@ docs:
 
 # Database management
 db-migrate:
-	python scripts/migrate_geolocation.py
+	uv run python scripts/migrate_geolocation.py
 
 # Monitoring
 health-check:
@@ -116,9 +139,38 @@ dev-setup: install-dev
 
 # Production deployment
 deploy-check:
-	python run.py --test
+	uv run python run.py --test
 	@echo "Deployment check passed!"
 
+# Database management
+setup-postgres:
+	uv run python scripts/setup_postgres.py
+
+test-postgres:
+	uv run python -c "from scripts.setup_postgres import test_postgres_connection; from src.config import Config; config = Config.get_database_config(); test_postgres_connection(config['connection_string'])"
+
+encode-password:
+	uv run python scripts/encode_password.py
+
+migrate-sqlite:
+	uv run python scripts/migrate_sqlite_to_postgres.py
+
+migrate-fast:
+	uv run python scripts/migrate_sqlite_to_postgres.py --fast
+
+analyze-sqlite:
+	uv run python scripts/migrate_sqlite_to_postgres.py --dry-run
+
+# Distribution
+build-exe:
+	uv run python build_simple.py
+
+package: build-exe
+	@echo "Creating distribution package..."
+	@if exist dist\ping-river-monitor-distribution.zip del dist\ping-river-monitor-distribution.zip
+	@cd dist && powershell -Command "Compress-Archive -Path * -DestinationPath ping-river-monitor-distribution.zip -Force"
+	@echo "✅ Distribution package created: dist/ping-river-monitor-distribution.zip"
+
 # Git helpers
 git-setup:
 	git remote add origin https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor.git
@@ -134,7 +186,7 @@ validate-workflows:
 	@echo "Validating Gitea Actions workflows..."
 	@for file in .gitea/workflows/*.yml; do \
 		echo "Checking $$file..."; \
-		python -c "import yaml; yaml.safe_load(open('$$file', encoding='utf-8'))" || exit 1; \
+		uv run python -c "import yaml; yaml.safe_load(open('$$file', encoding='utf-8'))" || exit 1; \
 	done
 	@echo "✅ All workflows are valid"
 

POSTGRESQL_SETUP.md

@@ -0,0 +1,287 @@
+# PostgreSQL Setup for Northern Thailand Ping River Monitor
+
+This guide helps you configure PostgreSQL as the database backend for the water monitoring system.
+
+## Prerequisites
+
+- PostgreSQL server running on a remote machine (already available)
+- Network connectivity to the PostgreSQL server
+- Database credentials (username, password, host, port)
+
+## Quick Setup
+
+### 1. Configure Environment
+
+Copy the example environment file and configure it:
+
+```bash
+cp .env.example .env
+```
+
+Edit `.env` and update the PostgreSQL configuration:
+
+```bash
+# Database Configuration
+DB_TYPE=postgresql
+
+# PostgreSQL Configuration (Remote Server)
+POSTGRES_CONNECTION_STRING=postgresql://username:password@your-postgres-host:5432/water_monitoring
+```
+
+### 2. Run Setup Script
+
+Use the interactive setup script:
+
+```bash
+# Using uv
+uv run python scripts/setup_postgres.py
+
+# Or using make
+make setup-postgres
+```
+
+The script will:
+- Test your database connection
+- Create the database if it doesn't exist
+- Initialize the required tables and indexes
+- Set up sample monitoring stations
+
+### 3. Test Connection
+
+Test your PostgreSQL connection:
+
+```bash
+make test-postgres
+```
+
+### 4. Run the Application
+
+Start collecting data:
+
+```bash
+# Run a test cycle
+make run-test
+
+# Start the web API
+make run-api
+```
+
+## Manual Configuration
+
+If you prefer manual setup, here's what you need:
+
+### Connection String Format
+
+```
+postgresql://username:password@host:port/database
+```
+
+**Examples:**
+- Basic: `postgresql://postgres:mypassword@192.168.1.100:5432/water_monitoring`
+- With SSL: `postgresql://user:pass@host:5432/db?sslmode=require`
+- With connection pooling: `postgresql://user:pass@host:5432/db?pool_size=20&max_overflow=0`
+
+### Environment Variables
+
+| Variable | Description | Example |
+|----------|-------------|---------|
+| `DB_TYPE` | Database type | `postgresql` |
+| `POSTGRES_CONNECTION_STRING` | Full connection string | See above |
+
+### Database Schema
+
+The application uses these main tables:
+
+1. **stations** - Monitoring station information
+2. **water_measurements** - Time series water level data
+3. **alert_thresholds** - Warning/danger level definitions
+4. **data_quality_log** - Data collection issue tracking
+
+See `sql/init_postgres.sql` for the complete schema.
+
+## Connection Options
+
+### SSL Connection
+
+For secure connections, add SSL parameters:
+
+```bash
+POSTGRES_CONNECTION_STRING=postgresql://user:pass@host:5432/db?sslmode=require
+```
+
+SSL modes:
+- `disable` - No SSL
+- `require` - Require SSL
+- `prefer` - Use SSL if available
+- `verify-ca` - Verify certificate authority
+- `verify-full` - Full certificate verification
+
+### Connection Pooling
+
+For high-performance applications, configure connection pooling:
+
+```bash
+POSTGRES_CONNECTION_STRING=postgresql://user:pass@host:5432/db?pool_size=20&max_overflow=0
+```
+
+Parameters:
+- `pool_size` - Number of connections to maintain
+- `max_overflow` - Additional connections allowed
+- `pool_timeout` - Seconds to wait for connection
+- `pool_recycle` - Seconds before connection refresh
+
+## Troubleshooting
+
+### Common Issues
+
+**1. Connection Refused**
+```
+psycopg2.OperationalError: could not connect to server
+```
+- Check if PostgreSQL server is running
+- Verify host/port in connection string
+- Check firewall settings
+
+**2. Authentication Failed**
+```
+psycopg2.OperationalError: FATAL: password authentication failed
+```
+- Verify username/password in connection string
+- Check PostgreSQL pg_hba.conf configuration
+- Ensure user has database access permissions
+
+**3. Database Does Not Exist**
+```
+psycopg2.OperationalError: FATAL: database "water_monitoring" does not exist
+```
+- Run the setup script to create the database
+- Or manually create: `CREATE DATABASE water_monitoring;`
+
+**4. Permission Denied**
+```
+psycopg2.ProgrammingError: permission denied for table
+```
+- Ensure user has appropriate permissions
+- Grant access: `GRANT ALL PRIVILEGES ON DATABASE water_monitoring TO username;`
+
+### Network Configuration
+
+For remote PostgreSQL servers, ensure:
+
+1. **PostgreSQL allows remote connections** (`postgresql.conf`):
+   ```
+   listen_addresses = '*'
+   port = 5432
+   ```
+
+2. **Client authentication is configured** (`pg_hba.conf`):
+   ```
+   # Allow connections from your application server
+   host    water_monitoring    username    your.app.ip/32    md5
+   ```
+
+3. **Firewall allows PostgreSQL port**:
+   ```bash
+   # On PostgreSQL server
+   sudo ufw allow 5432/tcp
+   ```
+
+### Performance Tuning
+
+For optimal performance with time series data:
+
+1. **Increase work_mem** for sorting operations
+2. **Tune shared_buffers** for caching
+3. **Configure maintenance_work_mem** for indexing
+4. **Set up regular VACUUM and ANALYZE** for statistics
+
+Example PostgreSQL configuration additions:
+```
+# postgresql.conf
+shared_buffers = 256MB
+work_mem = 16MB
+maintenance_work_mem = 256MB
+effective_cache_size = 1GB
+```
+
+## Monitoring
+
+### Check Application Status
+
+```bash
+# View current configuration
+uv run python -c "from src.config import Config; Config.print_settings()"
+
+# Test database connection
+make test-postgres
+
+# Check latest data
+psql "postgresql://user:pass@host:5432/water_monitoring" -c "SELECT COUNT(*) FROM water_measurements;"
+```
+
+### PostgreSQL Monitoring
+
+Connect directly to check database status:
+
+```bash
+# Connect to database
+psql "postgresql://username:password@host:5432/water_monitoring"
+
+# Check table sizes
+\dt+
+
+# View latest measurements
+SELECT * FROM latest_measurements LIMIT 10;
+
+# Check data quality
+SELECT issue_type, COUNT(*) FROM data_quality_log
+WHERE created_at > NOW() - INTERVAL '24 hours'
+GROUP BY issue_type;
+```
+
+## Backup and Maintenance
+
+### Backup Database
+
+```bash
+# Full backup
+pg_dump "postgresql://user:pass@host:5432/water_monitoring" > backup.sql
+
+# Data only
+pg_dump --data-only "postgresql://user:pass@host:5432/water_monitoring" > data_backup.sql
+```
+
+### Restore Database
+
+```bash
+# Restore full backup
+psql "postgresql://user:pass@host:5432/water_monitoring" < backup.sql
+
+# Restore data only
+psql "postgresql://user:pass@host:5432/water_monitoring" < data_backup.sql
+```
+
+### Regular Maintenance
+
+Set up regular maintenance tasks:
+
+```sql
+-- Update table statistics (run weekly)
+ANALYZE;
+
+-- Reclaim disk space (run monthly)
+VACUUM;
+
+-- Reindex tables (run quarterly)
+REINDEX DATABASE water_monitoring;
+```
+
+## Next Steps
+
+1. Set up monitoring and alerting
+2. Configure data retention policies
+3. Set up automated backups
+4. Implement connection pooling if needed
+5. Configure SSL for production use
+
+For more advanced configuration, see the [PostgreSQL documentation](https://www.postgresql.org/docs/).

README.md

@@ -2,7 +2,7 @@
 
 A comprehensive real-time water level monitoring system for the Ping River Basin in Northern Thailand, covering Royal Irrigation Department (RID) stations from Chiang Dao to Nakhon Sawan with advanced data collection, storage, and visualization capabilities.
 
-[![CI/CD](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/actions/workflows/ci.yml/badge.svg)](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/actions) [![Security](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/actions/workflows/security.yml/badge.svg)](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/actions) [![Documentation](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/actions/workflows/docs.yml/badge.svg)](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/actions) [![Python](https://img.shields.io/badge/Python-3.9+-blue.svg)](https://python.org) [![FastAPI](https://img.shields.io/badge/FastAPI-0.104+-green.svg)](https://fastapi.tiangolo.com) [![Docker](https://img.shields.io/badge/Docker-Ready-blue.svg)](https://docker.com) [![License](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE) [![Version](https://img.shields.io/badge/Version-v3.1.2-blue.svg)](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/releases)
+[![CI/CD](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/actions/workflows/ci.yml/badge.svg)](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/actions) [![Security](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/actions/workflows/security.yml/badge.svg)](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/actions) [![Documentation](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/actions/workflows/docs.yml/badge.svg)](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/actions) [![Python](https://img.shields.io/badge/Python-3.9+-blue.svg)](https://python.org) [![FastAPI](https://img.shields.io/badge/FastAPI-0.104+-green.svg)](https://fastapi.tiangolo.com) [![Docker](https://img.shields.io/badge/Docker-Ready-blue.svg)](https://docker.com) [![License](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE) [![Version](https://img.shields.io/badge/Version-v3.1.3-blue.svg)](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/releases)
 
 ## 🌟 Features
 
@@ -267,14 +267,32 @@ docker run -d \
 
 ### Systemd Service (Linux)
 
-```bash
-# Copy service file
-sudo cp scripts/water-monitor.service /etc/systemd/system/
+The install script sets everything up: a dedicated `water-monitor` system user,
+a deploy to `/opt/thailand-water-monitor`, a uv-managed virtualenv, and the
+enabled systemd unit.
 
-# Enable and start
+```bash
+# From a checkout of the repo, as root:
+sudo bash scripts/install.sh
+
+# Then start and check:
+sudo systemctl start water-monitor.service
+systemctl status water-monitor.service
+```
+
+Fill in `/opt/thailand-water-monitor/.env` (Matrix token/room, DB settings)
+before starting if the script reports it is missing.
+
+<details>
+<summary>Manual setup (if you prefer not to use the script)</summary>
+
+```bash
+sudo useradd --system --no-create-home --shell /usr/sbin/nologin water-monitor
+sudo cp scripts/water-monitor.service /etc/systemd/system/
 sudo systemctl enable water-monitor.service
 sudo systemctl start water-monitor.service
 ```
+</details>
 
 ### Migration for Existing Systems
 

SQLITE_MIGRATION.md

@@ -0,0 +1,278 @@
+# SQLite to PostgreSQL Migration Guide
+
+This guide helps you migrate your existing SQLite water monitoring data to PostgreSQL.
+
+## Quick Migration
+
+### 1. Analyze Your SQLite Database (Optional)
+
+First, check what's in your SQLite database:
+
+```bash
+# Analyze without migrating
+make analyze-sqlite
+
+# Or specify a specific SQLite file
+uv run python scripts/migrate_sqlite_to_postgres.py --dry-run /path/to/your/database.db
+```
+
+### 2. Run the Migration
+
+```bash
+# Auto-detect SQLite file and migrate
+make migrate-sqlite
+
+# Or specify a specific SQLite file
+uv run python scripts/migrate_sqlite_to_postgres.py /path/to/your/database.db
+```
+
+The migration tool will:
+- ✅ Connect to both databases
+- ✅ Analyze your SQLite schema automatically
+- ✅ Migrate station information
+- ✅ Migrate all measurement data in batches
+- ✅ Handle different SQLite table structures
+- ✅ Verify the migration results
+- ✅ Generate a detailed log file
+
+## What Gets Migrated
+
+### Station Data
+- Station IDs and codes
+- Thai and English names
+- Coordinates (latitude/longitude)
+- Geohash data (if available)
+- Creation/update timestamps
+
+### Measurement Data
+- Water level readings
+- Discharge measurements
+- Discharge percentages
+- Timestamps
+- Station associations
+- Data quality status
+
+## Supported SQLite Schemas
+
+The migration tool automatically detects and handles various SQLite table structures:
+
+### Modern Schema
+```sql
+-- Stations
+stations: id, station_code, station_name_th, station_name_en, latitude, longitude, geohash
+
+-- Measurements
+water_measurements: timestamp, station_id, water_level, discharge, discharge_percent, status
+```
+
+### Legacy Schema
+```sql
+-- Stations
+water_stations: station_id, station_code, station_name, lat, lon
+
+-- Measurements
+measurements: timestamp, station_id, water_level, discharge, discharge_percent
+```
+
+### Simple Schema
+```sql
+-- Any table with basic water level data
+-- The tool will adapt and map columns automatically
+```
+
+## Migration Process
+
+### Step 1: Database Connection
+- Connects to your SQLite database
+- Verifies PostgreSQL connection
+- Validates configuration
+
+### Step 2: Schema Analysis
+- Scans SQLite tables and columns
+- Reports data counts
+- Identifies table structures
+
+### Step 3: Station Migration
+- Extracts station metadata
+- Maps to PostgreSQL format
+- Handles missing data gracefully
+
+### Step 4: Measurement Migration
+- Processes data in batches (1000 records at a time)
+- Converts timestamps correctly
+- Preserves all measurement values
+- Shows progress during migration
+
+### Step 5: Verification
+- Compares record counts
+- Validates data integrity
+- Reports migration statistics
+
+## Command Options
+
+```bash
+# Basic migration (auto-detects SQLite file)
+uv run python scripts/migrate_sqlite_to_postgres.py
+
+# Specify SQLite database path
+uv run python scripts/migrate_sqlite_to_postgres.py /path/to/database.db
+
+# Dry run (analyze only, no migration)
+uv run python scripts/migrate_sqlite_to_postgres.py --dry-run
+
+# Custom batch size for large databases
+uv run python scripts/migrate_sqlite_to_postgres.py --batch-size 5000
+```
+
+## Auto-Detection
+
+The tool automatically searches for SQLite files in common locations:
+- `water_levels.db`
+- `water_monitoring.db`
+- `database.db`
+- `../water_levels.db`
+
+## Migration Output
+
+The tool provides detailed logging:
+
+```
+========================================
+SQLite to PostgreSQL Migration Tool
+========================================
+SQLite database: water_levels.db
+PostgreSQL: postgresql
+
+Step 1: Connecting to databases...
+Connected to SQLite database: water_levels.db
+Connected to PostgreSQL database
+
+Step 2: Analyzing SQLite database structure...
+Table 'stations': 8 columns, 25 rows
+Table 'water_measurements': 7 columns, 15420 rows
+
+Step 3: Migrating station data...
+Migrated 25 stations
+
+Step 4: Migrating measurement data...
+Found 15420 measurements to migrate
+Migrated 1000/15420 measurements
+Migrated 2000/15420 measurements
+...
+Successfully migrated 15420 measurements
+
+Step 5: Verifying migration...
+SQLite stations: 25
+SQLite measurements: 15420
+PostgreSQL measurements retrieved: 15420
+Migrated stations: 25
+Migrated measurements: 15420
+
+========================================
+MIGRATION COMPLETED
+========================================
+Duration: 0:02:15
+Stations migrated: 25
+Measurements migrated: 15420
+No errors encountered
+```
+
+## Error Handling
+
+The migration tool is robust and handles:
+- **Missing tables** - Tries alternative table names
+- **Different column names** - Maps common variations
+- **Missing data** - Uses sensible defaults
+- **Invalid timestamps** - Attempts multiple date formats
+- **Connection issues** - Provides clear error messages
+- **Large datasets** - Processes in batches to avoid memory issues
+
+## Log Files
+
+Migration creates a detailed log file:
+- `migration.log` - Complete migration log
+- Shows all operations, errors, and statistics
+- Useful for troubleshooting
+
+## Troubleshooting
+
+### Common Issues
+
+**1. SQLite file not found**
+```
+SQLite database file not found. Please specify the path:
+  python migrate_sqlite_to_postgres.py /path/to/database.db
+```
+**Solution**: Specify the correct path to your SQLite file
+
+**2. PostgreSQL not configured**
+```
+Error: PostgreSQL not configured. Set DB_TYPE=postgresql in your .env file
+```
+**Solution**: Ensure your .env file has `DB_TYPE=postgresql`
+
+**3. Connection failed**
+```
+Database connection error: connection refused
+```
+**Solution**: Check your PostgreSQL connection settings
+
+**4. No tables found**
+```
+Could not analyze SQLite database structure
+```
+**Solution**: Verify your SQLite file contains water monitoring data
+
+### Performance Tips
+
+- **Large databases**: Use `--batch-size 5000` for faster processing
+- **Slow networks**: Reduce batch size to `--batch-size 100`
+- **Memory issues**: Process smaller batches
+
+## After Migration
+
+Once migration is complete:
+
+1. **Verify data**:
+   ```bash
+   make run-test
+   make run-api
+   ```
+
+2. **Check the web interface**: Latest readings should show your migrated data
+
+3. **Backup your SQLite**: Keep the original file as backup
+
+4. **Update configurations**: Remove SQLite references from configs
+
+## Rollback
+
+If you need to rollback:
+
+1. **Clear PostgreSQL data**:
+   ```sql
+   DELETE FROM water_measurements;
+   DELETE FROM stations;
+   ```
+
+2. **Switch back to SQLite**:
+   ```bash
+   # In .env file
+   DB_TYPE=sqlite
+   WATER_DB_PATH=water_levels.db
+   ```
+
+3. **Test the rollback**:
+   ```bash
+   make run-test
+   ```
+
+The migration tool is designed to be safe and can be run multiple times - it handles duplicates appropriately.
+
+## Next Steps
+
+After successful migration:
+- Set up automated backups for PostgreSQL
+- Configure monitoring and alerting
+- Consider data retention policies
+- Update documentation references

build_executable.py

@@ -0,0 +1,301 @@
+#!/usr/bin/env python3
+"""
+Build script to create a standalone executable for Northern Thailand Ping River Monitor
+"""
+
+import os
+import sys
+import shutil
+from pathlib import Path
+
+def create_spec_file():
+    """Create PyInstaller spec file"""
+    spec_content = """
+# -*- mode: python ; coding: utf-8 -*-
+
+block_cipher = None
+
+# Data files to include
+data_files = [
+    ('.env', '.'),
+    ('sql/*.sql', 'sql'),
+    ('README.md', '.'),
+    ('POSTGRESQL_SETUP.md', '.'),
+    ('SQLITE_MIGRATION.md', '.'),
+]
+
+# Hidden imports that PyInstaller might miss
+hidden_imports = [
+    'psycopg2',
+    'psycopg2-binary',
+    'sqlalchemy.dialects.postgresql',
+    'sqlalchemy.dialects.sqlite',
+    'sqlalchemy.dialects.mysql',
+    'influxdb',
+    'pymysql',
+    'dotenv',
+    'pydantic',
+    'fastapi',
+    'uvicorn',
+    'schedule',
+    'pandas',
+    'requests',
+    'psutil',
+]
+
+a = Analysis(
+    ['run.py'],
+    pathex=['.'],
+    binaries=[],
+    datas=data_files,
+    hiddenimports=hidden_imports,
+    hookspath=[],
+    hooksconfig={},
+    runtime_hooks=[],
+    excludes=[
+        'tkinter',
+        'matplotlib',
+        'PIL',
+        'jupyter',
+        'notebook',
+        'IPython',
+    ],
+    win_no_prefer_redirects=False,
+    win_private_assemblies=False,
+    cipher=block_cipher,
+    noarchive=False,
+)
+
+pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher)
+
+exe = EXE(
+    pyz,
+    a.scripts,
+    a.binaries,
+    a.zipfiles,
+    a.datas,
+    [],
+    name='ping-river-monitor',
+    debug=False,
+    bootloader_ignore_signals=False,
+    strip=False,
+    upx=True,
+    upx_exclude=[],
+    runtime_tmpdir=None,
+    console=True,
+    disable_windowed_traceback=False,
+    argv_emulation=False,
+    target_arch=None,
+    codesign_identity=None,
+    entitlements_file=None,
+    icon='icon.ico' if os.path.exists('icon.ico') else None,
+)
+"""
+
+    with open('ping-river-monitor.spec', 'w') as f:
+        f.write(spec_content.strip())
+
+    print("[OK] Created ping-river-monitor.spec")
+
+def install_pyinstaller():
+    """Install PyInstaller if not present"""
+    try:
+        import PyInstaller
+        print("[OK] PyInstaller already installed")
+    except ImportError:
+        print("Installing PyInstaller...")
+        os.system("uv add --dev pyinstaller")
+        print("[OK] PyInstaller installed")
+
+def build_executable():
+    """Build the executable"""
+    print("🔨 Building executable...")
+
+    # Clean previous builds
+    if os.path.exists('dist'):
+        shutil.rmtree('dist')
+    if os.path.exists('build'):
+        shutil.rmtree('build')
+
+    # Build with PyInstaller using uv
+    result = os.system("uv run pyinstaller ping-river-monitor.spec --clean --noconfirm")
+
+    if result == 0:
+        print("✅ Executable built successfully!")
+
+        # Copy additional files to dist directory
+        dist_dir = Path('dist')
+        if dist_dir.exists():
+            # Copy .env file if it exists
+            if os.path.exists('.env'):
+                shutil.copy2('.env', dist_dir / '.env')
+                print("✅ Copied .env file")
+
+            # Copy documentation
+            for doc in ['README.md', 'POSTGRESQL_SETUP.md', 'SQLITE_MIGRATION.md']:
+                if os.path.exists(doc):
+                    shutil.copy2(doc, dist_dir / doc)
+                    print(f"✅ Copied {doc}")
+
+            # Copy SQL files
+            if os.path.exists('sql'):
+                shutil.copytree('sql', dist_dir / 'sql', dirs_exist_ok=True)
+                print("✅ Copied SQL files")
+
+            print(f"\n🎉 Executable created: {dist_dir / 'ping-river-monitor.exe'}")
+            print(f"📁 All files in: {dist_dir.absolute()}")
+
+    else:
+        print("❌ Build failed!")
+        return False
+
+    return True
+
+def create_batch_files():
+    """Create convenient batch files"""
+    batch_files = {
+        'start.bat': '''@echo off
+echo Starting Ping River Monitor...
+ping-river-monitor.exe
+pause
+''',
+        'start-api.bat': '''@echo off
+echo Starting Ping River Monitor Web API...
+ping-river-monitor.exe --web-api
+pause
+''',
+        'test.bat': '''@echo off
+echo Running Ping River Monitor test...
+ping-river-monitor.exe --test
+pause
+''',
+        'status.bat': '''@echo off
+echo Checking Ping River Monitor status...
+ping-river-monitor.exe --status
+pause
+'''
+    }
+
+    dist_dir = Path('dist')
+    for filename, content in batch_files.items():
+        batch_file = dist_dir / filename
+        with open(batch_file, 'w') as f:
+            f.write(content)
+        print(f"✅ Created {filename}")
+
+def create_readme():
+    """Create deployment README"""
+    readme_content = """# Ping River Monitor - Standalone Executable
+
+This is a standalone executable version of the Northern Thailand Ping River Monitor.
+
+## Quick Start
+
+1. **Configure Database**: Edit `.env` file with your PostgreSQL settings
+2. **Test Connection**: Double-click `test.bat`
+3. **Start Monitoring**: Double-click `start.bat`
+4. **Web Interface**: Double-click `start-api.bat`
+
+## Files Included
+
+- `ping-river-monitor.exe` - Main executable
+- `.env` - Configuration file (EDIT THIS!)
+- `start.bat` - Start continuous monitoring
+- `start-api.bat` - Start web API server
+- `test.bat` - Run a test cycle
+- `status.bat` - Check system status
+- `README.md`, `POSTGRESQL_SETUP.md` - Documentation
+- `sql/` - Database initialization scripts
+
+## Configuration
+
+Edit `.env` file:
+```
+DB_TYPE=postgresql
+POSTGRES_HOST=your-server-ip
+POSTGRES_PORT=5432
+POSTGRES_DB=water_monitoring
+POSTGRES_USER=your-username
+POSTGRES_PASSWORD=your-password
+```
+
+## Usage
+
+### Command Line
+```cmd
+# Continuous monitoring
+ping-river-monitor.exe
+
+# Single test run
+ping-river-monitor.exe --test
+
+# Web API server
+ping-river-monitor.exe --web-api
+
+# Check status
+ping-river-monitor.exe --status
+```
+
+### Batch Files
+- Just double-click the `.bat` files for easy operation
+
+## Troubleshooting
+
+1. **Database Connection Issues**
+   - Check `.env` file settings
+   - Verify PostgreSQL server is accessible
+   - Test with `test.bat`
+
+2. **Permission Issues**
+   - Run as administrator if needed
+   - Check firewall settings for API mode
+
+3. **Log Files**
+   - Check `water_monitor.log` for detailed logs
+   - Logs are created in the same directory as the executable
+
+## Support
+
+For issues or questions, check the documentation files included.
+"""
+
+    with open('dist/DEPLOYMENT_README.txt', 'w') as f:
+        f.write(readme_content)
+
+    print("✅ Created DEPLOYMENT_README.txt")
+
+def main():
+    """Main build process"""
+    print("Building Ping River Monitor Executable")
+    print("=" * 50)
+
+    # Check if we're in the right directory
+    if not os.path.exists('run.py'):
+        print("❌ Error: run.py not found. Please run this from the project root directory.")
+        return False
+
+    # Install PyInstaller
+    install_pyinstaller()
+
+    # Create spec file
+    create_spec_file()
+
+    # Build executable
+    if not build_executable():
+        return False
+
+    # Create convenience files
+    create_batch_files()
+    create_readme()
+
+    print("\n" + "=" * 50)
+    print("🎉 BUILD COMPLETE!")
+    print("📁 Check the 'dist' folder for your executable")
+    print("💡 Edit the .env file before distributing")
+    print("🚀 Ready for deployment!")
+
+    return True
+
+if __name__ == "__main__":
+    success = main()
+    sys.exit(0 if success else 1)

build_simple.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+"""
+Simple build script for standalone executable
+"""
+
+import os
+import sys
+import shutil
+from pathlib import Path
+
+def main():
+    print("Building Ping River Monitor Executable")
+    print("=" * 50)
+
+    # Check if PyInstaller is installed
+    try:
+        import PyInstaller
+        print("[OK] PyInstaller available")
+    except ImportError:
+        print("[INFO] Installing PyInstaller...")
+        os.system("uv add --dev pyinstaller")
+
+    # Clean previous builds
+    if os.path.exists('dist'):
+        shutil.rmtree('dist')
+        print("[CLEAN] Removed old dist directory")
+    if os.path.exists('build'):
+        shutil.rmtree('build')
+        print("[CLEAN] Removed old build directory")
+
+    # Build command with all necessary options
+    cmd = [
+        "uv", "run", "pyinstaller",
+        "--onefile",
+        "--console",
+        "--name=ping-river-monitor",
+        "--add-data=.env;.",
+        "--add-data=sql;sql",
+        "--add-data=README.md;.",
+        "--add-data=POSTGRESQL_SETUP.md;.",
+        "--add-data=SQLITE_MIGRATION.md;.",
+        "--hidden-import=psycopg2",
+        "--hidden-import=sqlalchemy.dialects.postgresql",
+        "--hidden-import=sqlalchemy.dialects.sqlite",
+        "--hidden-import=dotenv",
+        "--hidden-import=pydantic",
+        "--hidden-import=fastapi",
+        "--hidden-import=uvicorn",
+        "--hidden-import=schedule",
+        "--hidden-import=pandas",
+        "--clean",
+        "--noconfirm",
+        "run.py"
+    ]
+
+    print("[BUILD] Running PyInstaller...")
+    print("[CMD] " + " ".join(cmd))
+
+    result = os.system(" ".join(cmd))
+
+    if result == 0:
+        print("[SUCCESS] Executable built successfully!")
+
+        # Copy .env file to dist if it exists
+        if os.path.exists('.env') and os.path.exists('dist'):
+            shutil.copy2('.env', 'dist/.env')
+            print("[COPY] .env file copied to dist/")
+
+        # Create batch files for easy usage
+        batch_files = {
+            'start.bat': '''@echo off
+echo Starting Ping River Monitor...
+ping-river-monitor.exe
+pause
+''',
+            'start-api.bat': '''@echo off
+echo Starting Web API...
+ping-river-monitor.exe --web-api
+pause
+''',
+            'test.bat': '''@echo off
+echo Running test...
+ping-river-monitor.exe --test
+pause
+'''
+        }
+
+        for filename, content in batch_files.items():
+            if os.path.exists('dist'):
+                with open(f'dist/{filename}', 'w') as f:
+                    f.write(content)
+                print(f"[CREATE] {filename}")
+
+        print("\n" + "=" * 50)
+        print("BUILD COMPLETE!")
+        print(f"Executable: dist/ping-river-monitor.exe")
+        print("Batch files: start.bat, start-api.bat, test.bat")
+        print("Don't forget to edit .env file before using!")
+
+        return True
+    else:
+        print("[ERROR] Build failed!")
+        return False
+
+if __name__ == "__main__":
+    success = main()
+    sys.exit(0 if success else 1)

docs/FLOOD_FORECASTING.md

@@ -0,0 +1,625 @@
+# Flood forecasting
+
+Short-range flood-risk forecasts for the Ping River gauge network, trained on the
+monitor's own PostgreSQL history. This document covers what the system predicts,
+what it is built from, how well it actually performs, how to run it on the server,
+and when to retrain.
+
+Code lives in `src/ml/` (`data.py`, `features.py`, `train.py`, `predict.py`), the
+training entry point is `scripts/train_flood_model.py`, tests are in
+`tests/test_flood_forecast.py`, and trained artifacts land in `models/`.
+
+## 1. Overview
+
+For every station the system answers three questions at three lead times (6, 12
+and 24 hours):
+
+- **`p_warning`** — probability the water level reaches or exceeds the warning
+  threshold (3.0 m) at any point within the horizon.
+- **`p_danger`** — same for the danger threshold (4.5 m).
+- **`predicted_max_level`** — the expected peak level within the horizon, in
+  metres on the station's own datum.
+
+The window is open-ended forward: `exceed_warn_6` at 09:00 asks whether the level
+touches 3.0 m anywhere in (09:00, 15:00], not what it will be at 15:00 — the
+question an operator actually has.
+
+Fifteen of the sixteen stations have trained models. P.4A (Ban Mae Taeng) is
+excluded by `features.NOT_TRAINABLE` (17.2% hourly fill, effectively dead
+2019–2024: 289 rows in 2019, 769 in 2024) and is served by the persistence
+heuristic instead. It still feeds downstream stations as an *input*, where
+HistGradientBoosting's native NaN handling copes with the gaps.
+
+Every forecast row carries `source` (`model` or `heuristic`), `model_version`
+and `trained_at`, so a stale or degraded forecast is visible in the payload
+rather than silently indistinguishable from a good one.
+
+## 2. Data
+
+**Source of record.** PostgreSQL table `water_measurements` joined to `stations`.
+As verified on 2026-08-10 (`inventory.json`, `db_cross_check`): **592,240 rows,
+16 stations, 2018-08-01 through 2026-08-10**, zero mismatches against the HTTP
+API, and a `status` column that is uniformly `active` (there are no quality flags
+to filter on — bad readings must be caught by the feature pipeline, not by the
+database).
+
+**Coverage is the dominant data constraint.** Readings are nominally hourly
+(modal interval 1 h, ~95% of gaps), but only about **56% of hours on the complete
+hourly grid have a reading**: P.1, P.67, P.76 and P.84 at 56.1%, P.103 at 55.8%,
+P.21 at 55.4%, P.20 at 53.7%, P.87 at 53.3%, P.5 at 50.7%, P.4A at 17.2%.
+
+The missingness is **systematic, not random**. Measured over the 587 days from
+2025-01-01 in `models/cache/P.1.csv.gz`, the fraction of days with a reading at
+each hour is roughly 0.80 for 01:00–12:00, 0.68–0.69 for 13:00–16:00, 0.46–0.49
+for 17:00–21:00, 0.40–0.42 for 22:00–23:00, and 0.32 at midnight. That is a
+scrape-schedule fingerprint, not hydrology — and it is why hour-of-day is
+deliberately *not* a feature (see section 3).
+
+Long outages would poison training if used naively: P.87 lost 3,961 hours
+(165 days) in 2023, P.20 lost 2,681 hours in 2021, P.77 lost 2,522 hours in early
+2022, P.5 lost 2,429 hours over the 2020–21 turn. `features.TRAIN_START` excludes
+P.5 before 2022-01-01; the rest are handled by the per-row coverage gates.
+
+**How `src/ml/data.py` loads it.** `resolve_db_url()` picks a connection string in
+priority order: an explicit `--db-url` argument, then the `FLOOD_ML_DB_URL`
+environment variable, then `Config.get_database_config()` when `DB_TYPE` is
+`postgresql`, else `None`. `load_measurements()` then tries three tiers:
+
+1. **PostgreSQL** (`_fetch_from_db`) — the primary path. NULL discharge stays
+   NULL, which matters because the models must learn from the real missingness
+   pattern.
+2. **HTTP API** (`_fetch_from_api`, default `http://100.81.167.42:8000`) — a
+   fallback for running off-server. **Caveat:** the public history endpoint
+   backfills missing discharge with a synthetic rating-curve estimate, so this
+   path is not equivalent to the DB path. It is flagged as
+   `discharge_maybe_synthetic: true` in the cache metadata.
+3. **On-disk cache** (`models/cache/{station}.csv.gz` plus `meta.json`) — last
+   resort only. A successful DB or API fetch refreshes the cache; the cache is
+   never treated as a source of fresh data.
+
+`load_latest()` (used by the API) pulls the trailing 336 hours and never writes
+the cache.
+
+## 3. Physics and features
+
+### Upstream routing
+
+The Ping mainstem gives real forecast skill for free: a flood wave takes hours to
+travel downstream, so an upstream gauge reading *now* is information about a
+downstream gauge *later*. `data-scout` measured these travel times by
+cross-correlating water-level anomalies against the basin anchor P.1. The peak
+correlation lags, which are hard-coded in `features.UPSTREAM_LEADS`:
+
+| Station | Lead vs P.1 | Peak anomaly correlation | Distance to P.1 (km) |
+|---|---|---|---|
+| P.20 (Ban Chiang Dao) | 17 h | 0.59 | 84.5 |
+| P.92 (Ban Muang Aut) | 15 h | 0.66 | 63.8 |
+| P.75 (Ban Chai Lat) | 12 h | 0.61 | 45.0 |
+| P.4A (Ban Mae Taeng) | 12 h | 0.73 | 37.9 |
+| P.67 (Ban Tae) | 7 h | 0.74 | 25.3 |
+| P.21 (Ban Rim Tai) | 9 h | 0.56 | 15.0 |
+| P.103 (Ring Bridge 3) | 1 h | 0.88 | 9.3 |
+
+(Distances are cumulative straight-line gauge-to-gauge, from the inventory's
+`spatial_order_north_to_south`, not channel length — the real river is longer.)
+
+The lags are broadly consistent with distance, with one exception worth knowing
+about: P.21 is 10 km closer to P.1 than P.67 yet lags by 9 h rather than 7 h, and
+it has the weakest correlation of the mainstem set (0.56). Whatever the cause,
+the table encodes the measured lag rather than the one distance would predict —
+which is the point of measuring instead of assuming.
+
+Two stations are *downstream* of P.1 (P.5 at −12 h, P.81 at −4 h). For those,
+`UPSTREAM_LEADS` routes P.1 and P.103 forward as their inputs, which is the same
+physics running the other direction.
+
+Six western-tributary stations — P.82, P.84, P.87, P.77, P.85, P.76 — have empty
+`UPSTREAM_LEADS` and are **un-routed**. Their anomaly correlations with P.1 are
+0.22–0.32, low enough that routing them would inject noise rather than signal.
+They are forecast from their own history plus the P.1 basin-state features. This
+is a known gap: those catchments have no upstream gauge of their own in this
+network.
+
+### Feature set (`features.build_features`)
+
+Everything is computed on an hourly grid built by `make_hourly_grid()`, which
+keeps three aligned frames: `observed` (raw, NaN where nothing was recorded),
+`filled` (forward-filled with `FFILL_LIMIT_H = 3`), and `mask` (True where a real
+reading exists). Features read `filled`; labels read `observed` only.
+
+Per target station:
+
+- **Self level**: current level, lags at 1/2/3/6/12/24/48/72 h.
+- **Rate of rise**: level minus its own value 1/3/6/12/24 h ago — a river at
+  2.5 m and falling is a different situation from one at 2.5 m rising 30 cm/h.
+- **Rolling statistics**: 6/24 h means, 6/24/72 h maxima, 24 h minimum.
+- **Discharge**: current, lags at 6/24 h, 6 h rise (read from `observed`, so NULL
+  discharge stays NULL).
+- **Observation health**: `obs_age_h` (hours since the last real reading, capped
+  at `FFILL_LIMIT_H`) and `cov_24h` (fraction of the last 24 h actually observed),
+  so the model can learn to hedge when a gauge is going quiet.
+- **Routed upstream**, per `(upstream, lead)` pair: the upstream level at
+  `lead−3`, `lead`, and `lead+3` hours ago, its 6 h rise at `lead`, and its 24 h
+  rolling max at `lead−3`. The three-point bracket absorbs error in the measured
+  travel time rather than depending on it being exact.
+- **Basin state** (non-P.1 stations only): P.1 level, its 24 h rolling max, and
+  its 24 h rise.
+- **Seasonality**: `doy_sin`, `doy_cos` and an `is_monsoon` flag for June–October.
+
+**Hour-of-day is deliberately excluded.** Given the availability profile in
+section 2, an hour-of-day feature would let the model learn "readings at 03:00
+are more likely to exist" and route that through to the label — an artefact of
+when the scraper runs, with no hydrological content, that would evaporate the
+moment the scrape schedule changed.
+
+### No-leakage guarantees
+
+- Only `shift()`, backward `rolling()` and forward-fill are used — nothing
+  interpolates, and no row can read a value timestamped after itself.
+- `test_no_future_leakage` enforces this empirically: it adds +50 m to every
+  reading after time *t*, rebuilds the features, and asserts the rows at or
+  before *t* are bit-identical.
+- Labels come from `observed`, never `filled`, so a forward-filled value can
+  never become its own target.
+- The split is strictly temporal, and `early_stopping` is disabled in
+  `HGB_PARAMS` specifically because scikit-learn's internal validation split is
+  random and would leak across time.
+
+### Coverage gating
+
+A label is only trusted if enough of its forward window was actually observed.
+`build_labels` requires `MIN_WINDOW_COVERAGE = 0.5` — at least half the horizon's
+hours present — otherwise the label is NaN and the row is dropped from that head's
+training set. The one exception is deliberate: **an observed exceedance always
+produces a positive label regardless of coverage**, because a confirmed 3.5 m
+reading inside a sparse window is not ambiguous. Rows whose own features are
+stale (`obs_age_h` is NaN, i.e. the last real reading is more than 3 h old) are
+dropped entirely in `build_matrix`.
+
+## 4. Models
+
+### Architecture
+
+One `HistGradientBoosting` model per **station × horizon × head**:
+
+| Head | Type | Target |
+|---|---|---|
+| `max_{h}` | `HistGradientBoostingRegressor` (squared error) | max observed level in (t, t+h] |
+| `warn_{h}` | `HistGradientBoostingClassifier` | level ≥ 3.0 m anywhere in (t, t+h] |
+| `danger_{h}` | `HistGradientBoostingClassifier` | level ≥ 4.5 m anywhere in (t, t+h] |
+
+Nine heads per station, three horizons (6/12/24 h), fifteen trained stations.
+Hyperparameters are fixed (`HGB_PARAMS`: 300 iterations, learning rate 0.06, 31
+leaf nodes, minimum 50 samples per leaf, L2 1.0, `random_state=42`), chosen in an
+earlier sweep and not re-searched per run — training is deterministic and
+repeatable.
+
+HistGradientBoosting was chosen for three concrete reasons: it handles NaN
+natively (essential given ~44% missing hours), it needs no feature scaling, and
+it trains on CPU alone — no GPU anywhere in this pipeline (measured cost in
+section 6).
+
+### Head gating and fallbacks
+
+The system degrades in tiers rather than failing:
+
+1. **Classifier head**, when the training span contains at least
+   `MIN_POSITIVES_FOR_CLASSIFIER = 30` positive examples. Below that, a
+   classifier would be fitting noise, and the head is recorded in
+   `skipped_heads` with its reason.
+2. **Sigmoid on the regression head**, when the classifier is absent.
+   `p = 1/(1 + exp(−(predicted_max − threshold)/σ))`, where σ is the standard
+   deviation of the regressor's test residuals (floor `MIN_SIGMA = 0.15` m). This
+   turns the peak-level prediction into a calibrated-ish probability that widens
+   correctly when the regressor is less accurate at that horizon — at P.1, σ is
+   0.15 m at 6 and 12 h but 0.166 m at 24 h.
+3. **Persistence heuristic** (`predict._heuristic_forecast`), when there is no
+   model file at all, or the station's newest reading is more than
+   `STALE_AFTER_H = 6` hours old. It extrapolates the last 3 h rate of rise
+   forward with a 0.7 damping factor and a fixed σ of 0.3 m. It is not skilful; it
+   exists so the endpoint always returns something structurally valid.
+
+A station is skipped entirely if it has fewer than `MIN_ROWS_TO_TRAIN = 200`
+usable rows; an individual head is skipped below `MIN_ROWS_FOR_HEAD = 50` labeled
+rows. `_safe_fit` converts any fit failure (typically HistGradientBoosting's
+binning step rejecting an all-NaN or constant column) into a recorded skip rather
+than a station-killing exception.
+
+### Training procedure
+
+`train_station` runs two passes. First it evaluates on the strict temporal
+holdout (train ≤ 2024-12-31, test 2025-01-01 → 2026-08-10) to produce the metrics
+and the σ calibration. Then it **refits every head on the entire record** for the
+deployed artifact, so the shipped model has seen the most recent data. Because
+the full record has more labeled rows than the training half, the head-gating
+decisions can differ between the two passes — `skipped_heads` is therefore
+re-derived during the refit so it always describes what is actually in the saved
+bundle, not what the evaluation pass decided.
+
+### Bundle format
+
+`models/flood_{station}.joblib` contains: `station_code`, `model_version`
+(`hgb-v1+<git short SHA>`), `trained_at`, `sklearn_version`, `feature_names`,
+`horizons`, `thresholds`, `heads`, `sigma`, `skipped_heads`, `train_span`, and
+`n_train_rows`.
+
+`feature_names` is the important one. At prediction time `_model_forecast`
+rebuilds the feature row from live data and checks it against the bundle's stored
+list; if any expected column is missing it logs an error and falls back to the
+heuristic rather than feeding scikit-learn silently misaligned columns.
+`test_feature_name_stability` guards the same invariant at build time. Bundles are
+cached in memory keyed by `(path, mtime)`, so dropping in a retrained file
+invalidates the cache without a restart.
+
+## 5. Measured performance
+
+### Holdout metrics (`models/metrics.json`)
+
+Model version `hgb-v1+49a3de0`, generated 2026-08-10. Train ≤ 2024-12-31, test
+2025-01-01 → 2026-08-10 — the test span is entirely unseen future data relative
+to training.
+
+P.1 (Nawarat Bridge), the station that matters most:
+
+| Horizon | Warning PR-AUC | Recall @1% FAR | Recall @5% FAR | MAE | MAE above 2 m | Test rows | Base rate |
+|---|---|---|---|---|---|---|---|
+| 6 h | 0.974 | 98.3% | 100% | 6.1 cm | 9.2 cm | 8,536 | 1.36% |
+| 12 h | 0.904 | 93.8% | 97.7% | 9.0 cm | 15.0 cm | 7,932 | 1.61% |
+| 24 h | 0.900 | 90.1% | 93.4% | 11.3 cm | 24.5 cm | 8,572 | 1.77% |
+
+Read PR-AUC against the base rate — 0.974 versus a 1.36% positive rate is a wide
+margin over chance. "Recall at 1% false-alarm rate" is the operationally honest
+number: at a threshold that fires on 1% of quiet hours, the 6 h model still
+catches 98.3% of warning exceedances.
+
+P.103 (Ring Bridge 3) is the only station with enough danger-level events to
+evaluate a danger head on the 2025–26 span (base rate 5.7–7.4%): PR-AUC 0.979 /
+0.953 / 0.892 and recall at 1% FAR of 97.9% / 89.9% / 79.5% at 6 / 12 / 24 h.
+
+Across the other stations the 6 h warning PR-AUC spans 0.996 (P.5) down to 0.302
+(P.82), and tracks almost exactly with how many exceedances that station saw. The
+strong ones are the frequently-flooded gauges — P.5 0.996, P.81 0.992, P.77 0.968,
+P.85 0.953, P.75 0.927 — and the weak ones are un-routed western tributaries with
+almost no positives (P.84 0.570, P.82 0.302 on 0.22% of test hours). P.92 and P.20
+have no evaluable warning metric at all: neither crossed 3.0 m often enough in the
+test span (P.92 not once, P.20 in 0.09% of hours) to score.
+
+### Headline validation: the October 2024 record flood
+
+The holdout above never sees a true extreme, because the 2024 flood is in the
+training half. So the model was retrained on data **ending 2024-08-31** and asked
+to forecast September–November 2024 cold, with no knowledge of the event that
+followed. This is the closest thing to a real operational test available.
+
+- **25 September cold start.** P.1's first warning crossing of the episode was
+  alerted **24–26 hours ahead**. This is the genuinely impressive case: the river
+  was in normal state, and the alert came from upstream routing alone.
+- **5 October record peak** (P.1 5.30 m, P.103 9.93 m — the highest levels in the
+  eight-year record). Alerted **48 hours ahead**. Read this one carefully: the
+  river was already in sustained flood by then, so "48 hours" is the
+  `_first_alert_at` lookback window (`lookback_h = 48`) saturating, not a
+  measurement of true lead time. The model was correctly alarmed throughout;
+  the metric simply cannot express how much earlier than 48 h that started.
+- **P.103 danger head** over the same window: PR-AUC 0.98–0.99, recall at 1% FAR
+  87–95%. It called the danger-level crossings, not just the warning ones.
+- **8 November re-flood.** Caught **26–31 hours ahead** by the 12 and 24 h models
+  — a second, independent event in the same test window.
+- **P.103's 1 September "miss"** is a test-boundary artefact: the event begins in
+  the first hours of the test span, before the feature window has enough test-side
+  history to have produced a sustained alert. It is not a model failure, but it is
+  also not evidence of skill.
+
+### Honest limits
+
+**Genuine lead time is capped by gauge-only physics.** The longest upstream travel
+time into P.1 is 17 h (P.20), and the strongest predictors are much closer:
+P.103 at 1 h, P.67 at 7 h, P.21 at 9 h. Once a 24 h forecast reaches past roughly
+17 h, there is no observation that has "already happened" to inform it — the model
+is extrapolating basin state and season, not routing a wave. The 2025–26 test
+events bear this out: the 25 September 2025 cold-start crossing was called 7 h
+ahead by the 12 h model and 9 h ahead by the 24 h model. **Practical lead for P.1
+is ~7–17 h.** Extending it requires rainfall forecasts and Mae Ngat/Mae Kuang dam
+release data, neither of which this system currently ingests.
+
+**Danger-level skill at P.1 is unproven.** P.1 never crossed 4.5 m in the
+2025-01-01 → 2026-08-10 test span (`base_rate_danger` is 0.0, so every danger
+metric is `null`). The danger head exists and is trained on the full record — the
+river has spent 57 hours above 4.5 m historically, 0.144% of all hours — but no
+out-of-sample number backs it. Treat `p_danger` at P.1 as indicative, not
+validated.
+
+**Thresholds are per-station as of 2026-08-10.** `THRESHOLDS` now carries
+calibrated (warning, danger) pairs for all 16 stations, derived from the DB's
+`discharge_percent` (RID % of channel capacity): warning = median level at
+75–85% capacity, danger = median level at 95–105%. P.1 instead uses the official
+Chiang Mai inundation map (`P1_FLOOD_STAGES`): warning 3.70 m (city flooding
+begins, stage 1) and danger 4.20 m (stage 5). The prior single default of
+(3.0, 4.5) m made P.103 badly over-alert (its bank-full level is ~6.75 m) and
+P.67 under-alert (overflow at ~2.9 m, 1.6 m below the old danger line).
+**A retrain is required after any threshold change** — classifier labels depend
+on them; until then, model rows report the thresholds baked into their bundle.
+P.1 additionally reports `stages`: exceedance probability for each of the seven
+official inundation stages (3.70–4.60 m), computed from the regression head and
+its calibration sigma, so they need no retrain and no per-stage classifiers.
+
+## 6. Deployment
+
+### API
+
+`GET /forecast` (`src/web_api.py`) returns one JSON row per station × horizon with
+the fields listed in section 1. Results are cached in-process for
+`FORECAST_TTL = 900` seconds (15 minutes), which matches the data cadence — the
+underlying readings do not update faster than hourly. Inference runs in a thread
+via `asyncio.to_thread` so it never blocks the event loop.
+
+Failure modes: **503** if the `src.ml` package cannot be imported (missing
+scikit-learn, say), **502** on any other exception.
+
+One behaviour worth knowing, because the code comments suggest otherwise: the
+endpoint's `FileNotFoundError` ("No trained flood models found") and `RuntimeError`
+handlers are unreachable — nothing in `src/ml/` raises either, and
+`predict._forecast_station` checks `bundle_path.exists()` and falls back to the
+heuristic instead. So **before the first training run `/forecast` returns 200 with
+an all-heuristic payload**, not a 503, provided there is recent gauge data; you
+get an empty `200 []` only when there is no recent data at all. Judge deployment
+state by the `source` field, not the status code.
+
+### Dashboard
+
+The "Flood risk outlook" panel (`src/static/dashboard.html`, `loadForecasts()`)
+loads non-blocking after the map renders and **stays hidden unless `/forecast`
+returns a non-empty array** — a non-OK response, an empty array, or a thrown
+fetch all just leave the panel hidden, and the rest of the dashboard is
+unaffected. Per the note above, this means the panel appears with heuristic-only
+content once data is flowing but before any model is trained; the per-chip
+tooltip is what tells you so. Stations are sorted worst-risk first, each showing
+three chips (6/12/24 h) coloured by risk band, with the tooltip carrying the exact
+warning and danger percentages, the predicted peak level, and a "heuristic
+fallback" note when the row did not come from a model. The panel is labelled
+*experimental*.
+
+### Training on the server
+
+The server already has the PostgreSQL connection configured, so no host override
+is needed:
+
+```bash
+cd /path/to/Northern-Thailand-Ping-River-Monitor
+python scripts/train_flood_model.py --stations all
+```
+
+`resolve_db_url()` picks up `Config.get_database_config()` automatically when
+`DB_TYPE=postgresql`. The run writes fifteen `models/flood_{station}.joblib`
+bundles plus `models/metrics.json`.
+
+### Artifacts and dependencies
+
+The fifteen bundles total **101.4 MB** — mean 6.76 MB, from 4.04 MB (P.20) to
+9.35 MB (P.103 and P.87, with P.5 next at 8.99 MB) — plus `metrics.json` at
+0.34 MB and a 2.1 MB `models/cache/`. **These are not in
+git**, and they should stay that way — artifacts are produced on the server, not
+shipped. `.gitignore` excludes `models/*.joblib`, `models/cache/` and
+`models/metrics.json` for exactly this reason.
+
+Two pins matter and are already in `requirements.txt` / `pyproject.toml`:
+`scikit-learn==1.9.0` and `numpy>=1.24,<2` (pandas 2.0.3 wheels are ABI
+incompatible with numpy 2.x). Bundles record `sklearn_version`; unpickling a
+bundle under a different scikit-learn version is not guaranteed to work, so
+retrain after any scikit-learn upgrade rather than assuming the artifacts carry
+over.
+
+### Measured resource use
+
+All figures below were measured on 2026-08-10 on a development workstation —
+**24 physical / 32 logical cores at 2.20 GHz, 32 GiB RAM** (Python 3.11.9,
+scikit-learn 1.9.0, joblib 1.5.3, numpy 1.26.4, pandas 2.0.3) — **not** on the
+production server. They come from two independent benchmark runs on that same
+machine, which is why a couple of figures below are quoted as narrow ranges.
+Treat the CPU times as a floor and the memory figures as representative, since
+RSS barely depends on core count. Training read the `models/cache/` csv.gz files
+(592,240 rows load in 0.5 s); loading the same history from PostgreSQL was not
+measured and will be slower.
+
+**Training** (`train_all`, all 15 stations, evaluation pass plus full refit):
+
+| Measurement | Value |
+|---|---|
+| Full 15-station run, unrestricted threads | **199 s (3.3 min)** |
+| Peak RSS during the full run | **209 MB** |
+| Single station, unrestricted (P.1 / P.103) | 16.5 s / 18.7 s |
+
+HistGradientBoosting threads through OpenMP, and it scales only modestly. Timing
+P.1 alone under `OMP_NUM_THREADS`:
+
+| Threads | 1 | 2 | 4 | unrestricted (32) |
+|---|---|---|---|---|
+| P.1 train time | 36.5 s | 23.5 s | 14.7 s | 16.5 s |
+
+Two things follow. **Four threads is the sweet spot** — 32 threads was marginally
+*slower* than 4, so oversubscription costs you a little. And **even one core is
+enough**: at 36.5 s per station, a single-core box retrains all fifteen in roughly
+9 minutes (extrapolated, not measured end-to-end).
+
+Per station the fit costs **8–17 s**, and P.1 is the worst case at 16.9 s — it is
+the basin anchor, so it carries 64 features against 32 for stations with fewer
+upstream inputs (P.85 9.6 s, P.20 8.2 s). Two things are *not* the cost driver.
+Evaluation isn't: P.1 with `skip_eval=True` took 17.3 s, no faster than the full
+path. Nor is feature engineering — `build_matrix` over P.1's whole 8-year history
+is 256 ms against 8–17 s of fitting. **The fit is the cost.**
+
+One honest caveat about the run that produced the current artifacts. By file
+mtime it wrote all fifteen models between 11:55:29 and 12:01:15 — **5 min 46 s**,
+averaging 25 s/station including joblib serialization, which lines up with the
+measured fits. But `models/cache/meta.json` records the data fetch finishing at
+11:45:49, so end to end that run spanned about 15.5 minutes, and the 9 min 40 s
+gap between fetch and first model could not be reconstructed from the surviving
+artifacts. Do not attribute it to per-station training cost. Either way the
+conclusion holds: **retraining is minutes, not tens of minutes.**
+
+**Inference** (15 bundles, 16 stations × 3 horizons = 48 rows):
+
+| Measurement | Value |
+|---|---|
+| Cold call — every bundle unpickled from disk | **6.7 s** |
+| Warm call — bundles in `_MODEL_CACHE` | **0.72 s** median (0.63–0.84 s) |
+| RSS after imports, before any model | 71 MB |
+| RSS with all 15 bundles resident | **288 MB** |
+
+The 101.4 MB of on-disk pickles expand to roughly **203–211 MB resident** — about
+2× — and they stay there: `_MODEL_CACHE` replaces an entry when the file's mtime
+changes but never drops one to reclaim memory. That is the single largest memory
+cost of the whole feature.
+
+Where the time goes: cold start is 5.30 s, of which 0.46 s is the import and
+4.84 s is unpickling, and 2.6 s of *that* is the first bundle alone paying a
+one-time lazy `sklearn.ensemble` import — the remaining fourteen average 159 ms.
+Of the ~640 ms warm compute, model prediction is ~525 ms, the hourly grid 47 ms,
+and feature building 71 ms across all sixteen stations.
+
+**Live-endpoint measurements** (a second, independent benchmark run against a real
+uvicorn instance of the app, same day, same workstation, RSS summed over the
+process tree):
+
+| Measurement | Value |
+|---|---|
+| `/forecast` cache hit (15-min TTL) | **2.4 ms** median |
+| `/forecast` cache miss, default threads | 15.2 s (≈7 s of that was the HTTP data fallback; a local DB replaces it) |
+| `/forecast` cache miss, `OMP_NUM_THREADS=1` | 10.9 s |
+| API process RSS, idle → models resident | 76 MB → **335 MB** |
+
+The endpoint-level RSS (335 MB) is higher than the models-only figure above
+because the live process also retains the pandas frames from the data pull and
+the HTTP/JSON machinery — use 335 MB as the sizing number.
+
+One threading subtlety cuts the other way in serving: inference is ~135
+single-row predicts, and at one row OpenMP thread dispatch costs more than the
+math — `OMP_NUM_THREADS=1` makes the warm compute 2.6× faster (642 ms → 252 ms).
+Training shows the opposite (2.3× slower single-threaded), so set the variable
+per process, never globally.
+
+**Server sizing, in plain terms:** this is a small workload and almost any server
+runs it. **RAM is the binding constraint, not CPU.** Budget about **1 GB for the
+API process** so the ~335 MB steady state has headroom on top of the rest of the
+app; training peaks at only ~210–315 MB and can share the same box. No GPU
+anywhere. Pin thread counts per process — `OMP_NUM_THREADS=1` in the serving
+unit, `OMP_NUM_THREADS=4` for retraining so it cannot monopolise every core while
+the API is serving. And since a cold call costs seconds against a 2.4 ms cache
+hit, consider warming `/forecast` once at startup rather than letting a user
+absorb it.
+
+## 7. Retraining policy
+
+**Why it matters here specifically.** This is not a generic "models go stale"
+argument:
+
+- **Channel geometry changes after every major flood.** Scour, deposition and
+  bank failure shift the level-to-discharge relationship at a gauge, and RID
+  revises rating curves after big events. A model trained on the pre-2024 channel
+  is predicting levels for a cross-section that no longer exists.
+- **Extreme events extend the label range.** The highest P.103 reading before the
+  2024 season was 7.54 m (October 2022); the 2024 event pushed it to 8.27 m on
+  26 September and 9.93 m on 5 October. Gradient boosting cannot extrapolate past
+  its training range — predictions saturate at the largest value it has seen — so
+  every new record is what makes the next one predictable.
+- **Station outages change feature availability.** P.87's 165-day gap in 2023 and
+  P.4A's five dead years mean the set of populated features drifts over time.
+  Retraining lets head gating and NaN handling re-adapt to the current sensors.
+
+**Recommended schedule:**
+
+| When | Why |
+|---|---|
+| **Every year, May–June (pre-monsoon)** | The minimum. Ensures the model entering the flood season has seen last season in full. |
+| **Monthly, July–November** | Cheap insurance during the season — a full retrain costs minutes, not hours (section 6), so `nice` it and forget it. |
+| **After any major flood event** | Non-negotiable. Channel geometry and rating curves have changed, and the new extreme extends the trainable label range. |
+
+Staleness is auditable without guesswork: `model_version` embeds the git short SHA
+of the code that trained the bundle (`hgb-v1+49a3de0`), and `trained_at` is a
+timestamp in every bundle. Both are echoed in every `/forecast` row, so you can
+tell from the API response alone which code produced a forecast and how old the
+model is.
+
+## 8. Operations runbook
+
+All commands assume the project virtualenv is active (`.venv` locally).
+
+**Train (all stations, with evaluation):**
+
+```bash
+python scripts/train_flood_model.py --stations all
+```
+
+**Train a subset, refit-only (skips the holdout evaluation — much faster, but
+produces no metrics and leaves σ at the `MIN_SIGMA` floor):**
+
+```bash
+python scripts/train_flood_model.py --stations P.1,P.103 --skip-eval
+```
+
+**Train from a workstation against the server's database:**
+
+```bash
+export FLOOD_ML_DB_URL='postgresql://user:pass@host:5432/dbname'
+python scripts/train_flood_model.py --stations all
+```
+
+Do not commit that URL anywhere. If the DB is unreachable the loader silently
+falls back to the HTTP API, whose discharge values are partly synthetic — check
+the log line `PostgreSQL fetch failed, falling back to HTTP API` before trusting a
+run.
+
+**Verify before promoting.** Training writes `models/metrics.json` alongside the
+bundles. Check it before treating a run as good:
+
+```bash
+python -c "import json; m=json.load(open('models/metrics.json')); \
+print(m['model_version'], m['split']); \
+print({s: v['status'] for s, v in m['stations'].items()}); \
+print({h: (d.get('pr_auc_warn'), d.get('mae')) for h, d in m['stations']['P.1']['per_horizon'].items()})"
+```
+
+Expect fifteen `trained` and one `heuristic` (P.4A). A station that reports
+`failed` names its reason in the same payload. If P.1's 6 h warning PR-AUC has
+dropped materially below ~0.97 or its MAE has risen well above ~6 cm, investigate
+before deploying — that usually means a data problem (a gauge that went quiet, or
+a bad backfill) rather than a modelling one.
+
+**Run the tests** (synthetic data only, no database or network required):
+
+```bash
+python -m pytest tests/test_flood_forecast.py -v
+```
+
+Seven tests covering leakage, label alignment, the coverage gate, forward-fill and
+staleness, a train/predict round trip, the heuristic fallback, and feature-name
+stability. The whole suite runs in about 8 seconds, so there is no excuse for
+skipping it before a deploy.
+
+**Understanding graceful degradation.** Three things can make a forecast row
+non-model-backed, and all of them are visible in the payload:
+
+- `source: "heuristic"`, `model_version: "heuristic-v1"` — either no bundle exists
+  for that station (P.4A always, every station before the first training run), or
+  the station's newest reading is more than 6 hours old.
+- A single horizon coming back heuristic while others are model-backed — that
+  horizon's head is in the bundle's `skipped_heads`, almost always because the
+  station had fewer than 30 positive examples for that threshold.
+- A whole station flipping to heuristic after a code change — the feature-name
+  check in `_model_forecast` caught a mismatch between the live feature builder
+  and the stored `feature_names`. The fix is to retrain; the log line names the
+  missing columns.
+
+A live example from the 2026-08-10 cache: of 48 forecast rows, 42 came from
+models and 6 were heuristic — three for P.4A, which has no bundle by design, and
+three for P.92, whose newest reading was 02:00 while the basin's newest was 09:00.
+That 7 hours of staleness crossed `STALE_AFTER_H = 6`, so P.92 correctly dropped
+to persistence. Both fallback triggers, working as intended, in one ordinary call.
+
+Inspect a bundle's skipped heads directly (`joblib.load` unpickles, so only ever
+point it at a bundle this pipeline's own `train.py` wrote — never a file from
+elsewhere):
+
+```bash
+python -c "import joblib; b=joblib.load('models/flood_P.1.joblib'); \
+print(b['model_version'], b['trained_at'], b['n_train_rows']); print(b['skipped_heads'])"
+```

docs/GITEA_WORKFLOWS.md

@@ -297,6 +297,6 @@ make validate-workflows
 
 ---
 
-**Workflow Version**: v3.1.2  
+**Workflow Version**: v3.1.3  
 **Last Updated**: 2025-08-12  
 **Maintained By**: Ping River Monitor Team

docs/GRAFANA_MATRIX_ALERTING.md

@@ -0,0 +1,168 @@
+# Grafana Matrix Alerting Setup
+
+## Overview
+Configure Grafana to send water level alerts directly to Matrix channels when thresholds are exceeded.
+
+## Prerequisites
+- Grafana instance with your PostgreSQL data source
+- Matrix account and access token
+- Matrix room for alerts
+
+## Step 1: Configure Matrix Contact Point
+
+1. **In Grafana, go to Alerting → Contact Points**
+2. **Add new contact point:**
+   ```
+   Name: matrix-water-alerts
+   Integration: Webhook
+   URL: https://matrix.org/_matrix/client/v3/rooms/!ROOM_ID:matrix.org/send/m.room.message
+   HTTP Method: POST
+   ```
+
+3. **Add Headers:**
+   ```
+   Authorization: Bearer YOUR_MATRIX_ACCESS_TOKEN
+   Content-Type: application/json
+   ```
+
+4. **Message Template:**
+   ```json
+   {
+     "msgtype": "m.text",
+     "body": "🌊 WATER ALERT: {{ .CommonLabels.alertname }}\n\nStation: {{ .CommonLabels.station_code }}\nLevel: {{ .CommonAnnotations.water_level }}m\nStatus: {{ .CommonLabels.severity }}\n\nTime: {{ .CommonAnnotations.time }}"
+   }
+   ```
+
+## Step 2: Create Alert Rules
+
+### High Water Level Alert
+```yaml
+Rule Name: high-water-level
+Query: water_level > 6.0
+Condition: IS ABOVE 6.0 FOR 5m
+Labels:
+  - severity: critical
+  - station_code: {{ .station_code }}
+Annotations:
+  - water_level: {{ .water_level }}
+  - summary: "Critical water level at {{ .station_code }}"
+```
+
+### Low Water Level Alert
+```yaml
+Rule Name: low-water-level
+Query: water_level < 1.0
+Condition: IS BELOW 1.0 FOR 10m
+Labels:
+  - severity: warning
+  - station_code: {{ .station_code }}
+```
+
+### Data Gap Alert
+```yaml
+Rule Name: data-gap
+Query: increase(measurements_total[1h]) == 0
+Condition: IS EQUAL TO 0 FOR 30m
+Labels:
+  - severity: warning
+  - issue: data-gap
+```
+
+## Step 3: Matrix Setup
+
+### Get Matrix Access Token
+```bash
+curl -X POST https://matrix.org/_matrix/client/v3/login \
+  -H "Content-Type: application/json" \
+  -d '{
+    "type": "m.login.password",
+    "user": "your_username",
+    "password": "your_password"
+  }'
+```
+
+### Create Alert Room
+```bash
+curl -X POST "https://matrix.org/_matrix/client/v3/createRoom" \
+  -H "Authorization: Bearer YOUR_ACCESS_TOKEN" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "name": "Water Level Alerts - Northern Thailand",
+    "topic": "Automated alerts for Ping River water monitoring",
+    "preset": "trusted_private_chat"
+  }'
+```
+
+## Example Alert Queries
+
+### Critical Water Levels
+```promql
+# High water alert
+water_level{station_code=~"P.1|P.4A|P.20"} > 6.0
+
+# Dangerous discharge
+discharge{station_code=~".*"} > 500
+
+# Rapid level change
+increase(water_level[15m]) > 0.5
+```
+
+### System Health
+```promql
+# No data received
+up{job="water-monitor"} == 0
+
+# Old data
+(time() - timestamp) > 7200
+```
+
+## Alert Notification Format
+
+Your Matrix messages will look like:
+```
+🌊 WATER ALERT: High Water Level
+
+Station: P.1 (Chiang Mai)
+Level: 6.2m (CRITICAL)
+Discharge: 450 cms
+Status: DANGER
+
+Time: 2025-09-26 14:30:00
+Trend: Rising (+0.3m in 30min)
+
+📍 Location: 18.7883°N, 98.9853°E
+```
+
+## Advanced Features
+
+### Escalation Rules
+```yaml
+# Send to different rooms based on severity
+- if: severity == "critical"
+  receiver: matrix-emergency
+- if: severity == "warning"
+  receiver: matrix-alerts
+- if: time_of_day() outside "08:00-20:00"
+  receiver: matrix-night-duty
+```
+
+### Rate Limiting
+```yaml
+group_wait: 5m
+group_interval: 10m
+repeat_interval: 30m
+```
+
+## Testing Alerts
+
+1. **Test Contact Point** - Use Grafana's test button
+2. **Simulate Alert** - Manually trigger with test data
+3. **Verify Matrix** - Check message formatting and delivery
+
+## Troubleshooting
+
+### Common Issues
+- **403 Forbidden**: Check Matrix access token
+- **Room not found**: Verify room ID format
+- **No alerts**: Check query syntax and thresholds
+- **Spam**: Configure proper grouping and intervals

docs/GRAFANA_MATRIX_SETUP.md

@@ -0,0 +1,351 @@
+# Complete Grafana Matrix Alerting Setup Guide
+
+## Overview
+Configure Grafana to send water level alerts directly to Matrix channels when thresholds are exceeded.
+
+## Prerequisites
+- Grafana instance running (v8.0+)
+- PostgreSQL data source configured in Grafana
+- Matrix account
+- Matrix room for alerts
+
+## Step 1: Get Matrix Access Token
+
+### Method 1: Using curl
+```bash
+curl -X POST https://matrix.org/_matrix/client/v3/login \
+  -H "Content-Type: application/json" \
+  -d '{
+    "type": "m.login.password",
+    "user": "your_username",
+    "password": "your_password"
+  }'
+```
+
+### Method 2: Using Element Web Client
+1. Open Element in browser: https://app.element.io
+2. Login to your account
+3. Go to Settings → Help & About → Advanced
+4. Copy your Access Token
+
+### Method 3: Using Matrix Admin Panel
+- If you have admin access to your homeserver, generate token via admin API
+
+## Step 2: Create Alert Room
+
+```bash
+curl -X POST "https://matrix.org/_matrix/client/v3/createRoom" \
+  -H "Authorization: Bearer YOUR_ACCESS_TOKEN" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "name": "Water Level Alerts - Northern Thailand",
+    "topic": "Automated alerts for Ping River water monitoring",
+    "preset": "private_chat"
+  }'
+```
+
+Save the `room_id` from the response (format: !roomid:homeserver.com)
+
+## Step 3: Configure Grafana Contact Point
+
+### Navigate to Alerting
+1. In Grafana, go to **Alerting → Contact Points**
+2. Click **Add contact point**
+
+### Contact Point Settings
+```
+Name: matrix-water-alerts
+Integration: Webhook
+URL: https://matrix.org/_matrix/client/v3/rooms/!YOUR_ROOM_ID:matrix.org/send/m.room.message/{{ .GroupLabels.alertname }}_{{ .GroupLabels.severity }}_{{ now.Unix }}
+HTTP Method: POST
+```
+
+### Headers
+```
+Authorization: Bearer YOUR_MATRIX_ACCESS_TOKEN
+Content-Type: application/json
+```
+
+### Message Template (JSON Body)
+```json
+{
+  "msgtype": "m.text",
+  "body": "🌊 **PING RIVER WATER ALERT**\n\n**Alert:** {{ .GroupLabels.alertname }}\n**Severity:** {{ .GroupLabels.severity | toUpper }}\n**Station:** {{ .GroupLabels.station_code }} ({{ .GroupLabels.station_name }})\n\n{{ range .Alerts }}**Status:** {{ .Status | toUpper }}\n**Water Level:** {{ .Annotations.water_level }}m\n**Threshold:** {{ .Annotations.threshold }}m\n**Time:** {{ .StartsAt.Format \"2006-01-02 15:04:05\" }}\n{{ if .Annotations.discharge }}**Discharge:** {{ .Annotations.discharge }} cms\n{{ end }}{{ if .Annotations.message }}**Details:** {{ .Annotations.message }}\n{{ end }}{{ end }}\n📈 **Dashboard:** {{ .ExternalURL }}\n📍 **Location:** Northern Thailand Ping River"
+}
+```
+
+## Step 4: Create Alert Rules
+
+### High Water Level Alert
+```yaml
+# Rule Configuration
+Rule Name: high-water-level
+Evaluation Group: water-level-alerts
+Folder: Water Monitoring
+
+# Query A
+SELECT
+  station_code,
+  station_name_th as station_name,
+  water_level,
+  discharge,
+  timestamp
+FROM water_measurements
+WHERE
+  timestamp > now() - interval '5 minutes'
+  AND water_level > 6.0
+
+# Condition
+IS ABOVE 6.0 FOR 5 minutes
+
+# Labels
+severity: critical
+alertname: High Water Level
+station_code: {{ $labels.station_code }}
+station_name: {{ $labels.station_name }}
+
+# Annotations
+water_level: {{ $values.water_level }}
+threshold: 6.0
+discharge: {{ $values.discharge }}
+summary: Critical water level detected at {{ $labels.station_code }}
+```
+
+### Emergency Water Level Alert
+```yaml
+Rule Name: emergency-water-level
+Query: water_level > 8.0
+Condition: IS ABOVE 8.0 FOR 2 minutes
+Labels:
+  severity: emergency
+  alertname: Emergency Water Level
+Annotations:
+  threshold: 8.0
+  message: IMMEDIATE ACTION REQUIRED - Flood risk imminent
+```
+
+### Low Water Level Alert
+```yaml
+Rule Name: low-water-level
+Query: water_level < 1.0
+Condition: IS BELOW 1.0 FOR 15 minutes
+Labels:
+  severity: warning
+  alertname: Low Water Level
+Annotations:
+  threshold: 1.0
+  message: Drought conditions detected
+```
+
+### Data Gap Alert
+```yaml
+Rule Name: data-gap
+Query:
+  SELECT
+    station_code,
+    MAX(timestamp) as last_seen
+  FROM water_measurements
+  GROUP BY station_code
+  HAVING MAX(timestamp) < now() - interval '2 hours'
+
+Condition: HAS NO DATA FOR 30 minutes
+Labels:
+  severity: warning
+  alertname: Data Gap
+  issue: missing-data
+```
+
+### Rapid Level Change Alert
+```yaml
+Rule Name: rapid-level-change
+Query:
+  SELECT
+    station_code,
+    water_level,
+    LAG(water_level, 1) OVER (PARTITION BY station_code ORDER BY timestamp) as prev_level
+  FROM water_measurements
+  WHERE timestamp > now() - interval '15 minutes'
+  HAVING ABS(water_level - prev_level) > 0.5
+
+Condition: CHANGE > 0.5m FOR 1 minute
+Labels:
+  severity: warning
+  alertname: Rapid Water Level Change
+```
+
+## Step 5: Configure Notification Policy
+
+### Create Notification Policy
+```yaml
+# Policy Tree
+- receiver: matrix-water-alerts
+  match:
+    severity: emergency|critical
+  group_wait: 10s
+  group_interval: 5m
+  repeat_interval: 30m
+
+- receiver: matrix-water-alerts
+  match:
+    severity: warning
+  group_wait: 30s
+  group_interval: 10m
+  repeat_interval: 2h
+```
+
+### Grouping Rules
+```yaml
+group_by: [alertname, station_code]
+group_wait: 10s
+group_interval: 5m
+repeat_interval: 1h
+```
+
+## Step 6: Station-Specific Thresholds
+
+Create separate rules for each station with appropriate thresholds:
+
+```sql
+-- P.1 (Chiang Mai) - Urban area, higher thresholds
+SELECT * FROM water_measurements
+WHERE station_code = 'P.1' AND water_level > 6.5
+
+-- P.4A (Mae Ping) - Agricultural area
+SELECT * FROM water_measurements
+WHERE station_code = 'P.4A' AND water_level > 5.0
+
+-- P.20 (Downstream) - Lower threshold
+SELECT * FROM water_measurements
+WHERE station_code = 'P.20' AND water_level > 4.0
+```
+
+## Step 7: Advanced Features
+
+### Time-Based Routing
+```yaml
+# Different receivers for day/night
+time_intervals:
+  - name: working_hours
+    time_intervals:
+      - times:
+        - start_time: '08:00'
+          end_time: '20:00'
+        weekdays: ['monday:friday']
+
+routes:
+  - receiver: matrix-alerts-day
+    match:
+      severity: warning
+    active_time_intervals: [working_hours]
+
+  - receiver: matrix-alerts-night
+    match:
+      severity: warning
+    active_time_intervals: ['!working_hours']
+```
+
+### Multi-Channel Alerts
+```yaml
+# Send critical alerts to multiple rooms
+- receiver: matrix-emergency
+  webhook_configs:
+    - url: https://matrix.org/_matrix/client/v3/rooms/!emergency:matrix.org/send/m.room.message
+      http_config:
+        authorization:
+          credentials: "Bearer EMERGENCY_TOKEN"
+    - url: https://matrix.org/_matrix/client/v3/rooms/!general:matrix.org/send/m.room.message
+      http_config:
+        authorization:
+          credentials: "Bearer GENERAL_TOKEN"
+```
+
+## Step 8: Testing
+
+### Test Contact Point
+1. Go to Contact Points in Grafana
+2. Select your Matrix contact point
+3. Click "Test" button
+4. Check Matrix room for test message
+
+### Test Alert Rules
+1. Temporarily lower thresholds
+2. Wait for condition to trigger
+3. Verify alert appears in Grafana
+4. Verify Matrix message received
+5. Reset thresholds
+
+### Manual Alert Trigger
+```bash
+# Simulate high water level in database
+INSERT INTO water_measurements (station_code, water_level, timestamp)
+VALUES ('P.1', 7.5, NOW());
+```
+
+## Troubleshooting
+
+### Common Issues
+
+#### 403 Forbidden
+- **Cause**: Invalid Matrix access token
+- **Fix**: Regenerate token or check permissions
+
+#### Room Not Found
+- **Cause**: Incorrect room ID format
+- **Fix**: Ensure room ID starts with ! and includes homeserver
+
+#### No Alerts Firing
+- **Cause**: Query returns no results
+- **Fix**: Test queries in Grafana Explore, check data availability
+
+#### Alert Spam
+- **Cause**: No grouping configured
+- **Fix**: Configure proper group_by and intervals
+
+#### Messages Not Formatted
+- **Cause**: Template syntax errors
+- **Fix**: Validate JSON template, check Grafana template docs
+
+### Debug Steps
+1. Check Grafana alert rule status
+2. Verify contact point test succeeds
+3. Check Grafana logs: `/var/log/grafana/grafana.log`
+4. Test Matrix API directly with curl
+5. Verify database connectivity and query results
+
+## Environment Variables
+
+Add to your `.env`:
+```bash
+MATRIX_HOMESERVER=https://matrix.org
+MATRIX_ACCESS_TOKEN=your_access_token_here
+MATRIX_ROOM_ID=!your_room_id:matrix.org
+GRAFANA_URL=http://your-grafana-host:3000
+```
+
+## Example Alert Message
+Your Matrix messages will appear as:
+```
+🌊 **PING RIVER WATER ALERT**
+
+**Alert:** High Water Level
+**Severity:** CRITICAL
+**Station:** P.1 (สถานีเชียงใหม่)
+
+**Status:** FIRING
+**Water Level:** 6.75m
+**Threshold:** 6.0m
+**Time:** 2025-09-26 14:30:00
+**Discharge:** 450.2 cms
+
+📈 **Dashboard:** http://grafana:3000
+📍 **Location:** Northern Thailand Ping River
+```
+
+## Security Notes
+- Store Matrix tokens securely (environment variables)
+- Use room-specific tokens when possible
+- Enable rate limiting to prevent spam
+- Consider using dedicated alerting user account
+- Regularly rotate access tokens
+
+This setup provides comprehensive water level monitoring with immediate Matrix notifications when thresholds are exceeded.

docs/MATRIX_QUICK_START.md

@@ -0,0 +1,85 @@
+# Quick Matrix Alerting Setup
+
+## Step 1: Get Matrix Account
+1. Go to https://app.element.io or install Element app
+2. Create account or login with existing Matrix account
+
+## Step 2: Get Access Token
+
+### Method 1: Element Web (Recommended)
+1. Open Element in browser: https://app.element.io
+2. Login to your account
+3. Click Settings (gear icon) → Help & About → Advanced
+4. Copy your "Access Token" (starts with `syt_...` or similar)
+
+### Method 2: Command Line
+```bash
+curl -X POST https://matrix.org/_matrix/client/v3/login \
+  -H "Content-Type: application/json" \
+  -d '{
+    "type": "m.login.password",
+    "user": "your_username",
+    "password": "your_password"
+  }'
+```
+
+## Step 3: Create Alert Room
+1. In Element, click "+" to create new room
+2. Name: "Water Level Alerts"
+3. Set to Private
+4. Copy the room ID from room settings (format: `!roomid:matrix.org`)
+
+## Step 4: Configure .env File
+Add these to your `.env` file:
+```bash
+# Matrix Alerting Configuration
+MATRIX_HOMESERVER=https://matrix.org
+MATRIX_ACCESS_TOKEN=syt_your_access_token_here
+MATRIX_ROOM_ID=!your_room_id:matrix.org
+
+# Grafana Integration (optional)
+GRAFANA_URL=http://localhost:3000
+```
+
+## Step 5: Test Configuration
+```bash
+# Test Matrix connection
+uv run python run.py --alert-test
+
+# Check system status (shows Matrix config)
+uv run python run.py --status
+
+# Run alert check
+uv run python run.py --alert-check
+```
+
+## Example Alert Message
+When thresholds are exceeded, you'll receive messages like:
+```
+🌊 **WATER LEVEL ALERT**
+
+**Station:** P.1 (สถานีเชียงใหม่)
+**Alert Type:** Critical Water Level
+**Severity:** CRITICAL
+
+**Current Level:** 6.75m
+**Threshold:** 6.0m
+**Difference:** +0.75m
+**Discharge:** 450.2 cms
+
+**Time:** 2025-09-26 14:30:00
+
+📈 View dashboard: http://localhost:3000
+```
+
+## Cron Job Setup (Optional)
+Add to crontab for automatic alerting:
+```bash
+# Check water levels every 15 minutes
+*/15 * * * * cd /path/to/monitor && uv run python run.py --alert-check >> alerts.log 2>&1
+```
+
+## Troubleshooting
+- **403 Error**: Check Matrix access token is valid
+- **Room Not Found**: Verify room ID includes `!` prefix and `:homeserver.com` suffix
+- **No Alerts**: Check database has recent data with `uv run python run.py --status`

ping-river-monitor.spec

@@ -0,0 +1,38 @@
+# -*- mode: python ; coding: utf-8 -*-
+
+
+a = Analysis(
+    ['run.py'],
+    pathex=[],
+    binaries=[],
+    datas=[('.env', '.'), ('sql', 'sql'), ('README.md', '.'), ('POSTGRESQL_SETUP.md', '.'), ('SQLITE_MIGRATION.md', '.')],
+    hiddenimports=['psycopg2', 'sqlalchemy.dialects.postgresql', 'sqlalchemy.dialects.sqlite', 'dotenv', 'pydantic', 'fastapi', 'uvicorn', 'schedule', 'pandas'],
+    hookspath=[],
+    hooksconfig={},
+    runtime_hooks=[],
+    excludes=[],
+    noarchive=False,
+    optimize=0,
+)
+pyz = PYZ(a.pure)
+
+exe = EXE(
+    pyz,
+    a.scripts,
+    a.binaries,
+    a.datas,
+    [],
+    name='ping-river-monitor',
+    debug=False,
+    bootloader_ignore_signals=False,
+    strip=False,
+    upx=True,
+    upx_exclude=[],
+    runtime_tmpdir=None,
+    console=True,
+    disable_windowed_traceback=False,
+    argv_emulation=False,
+    target_arch=None,
+    codesign_identity=None,
+    entitlements_file=None,
+)

pyproject.toml

@@ -0,0 +1,130 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "northern-thailand-ping-river-monitor"
+version = "3.1.3"
+description = "Real-time water level monitoring system for the Ping River Basin in Northern Thailand"
+readme = "README.md"
+license = {text = "MIT"}
+authors = [
+    {name = "Ping River Monitor Team", email = "contact@example.com"}
+]
+keywords = [
+    "water monitoring",
+    "hydrology",
+    "thailand",
+    "ping river",
+    "environmental monitoring",
+    "time series",
+    "fastapi",
+    "real-time data"
+]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Science/Research",
+    "Intended Audience :: System Administrators",
+    "Topic :: Scientific/Engineering :: Hydrology",
+    "Topic :: System :: Monitoring",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Operating System :: OS Independent",
+    "Environment :: Web Environment",
+    "Framework :: FastAPI"
+]
+requires-python = ">=3.11"
+dependencies = [
+    # Core dependencies
+    "requests==2.31.0",
+    "schedule==1.2.0",
+    "pandas==2.0.3",
+    "numpy>=1.24,<2",
+    # Flood forecasting (ML)
+    "scikit-learn==1.9.0",
+    # Web API framework
+    "fastapi==0.104.1",
+    "uvicorn[standard]==0.24.0",
+    "pydantic==2.5.0",
+    # Database adapters
+    "sqlalchemy==2.0.23",
+    "influxdb==5.3.1",
+    "pymysql==1.1.0",
+    "psycopg2-binary==2.9.9",
+    # Monitoring and metrics
+    "psutil==5.9.6"
+]
+
+[project.optional-dependencies]
+dev = [
+    # Testing
+    "pytest==7.4.3",
+    "pytest-cov==4.1.0",
+    "pytest-asyncio==0.21.1",
+    # Code formatting and linting
+    "black==23.11.0",
+    "flake8==6.1.0",
+    "isort==5.12.0",
+    "mypy==1.7.1",
+    # Pre-commit hooks
+    "pre-commit==3.5.0",
+    # Development tools
+    "ipython==8.17.2",
+    "jupyter==1.0.0",
+    # Type stubs
+    "types-requests==2.31.0.10",
+    "types-python-dateutil==2.8.19.14"
+]
+docs = [
+    "sphinx==7.2.6",
+    "sphinx-rtd-theme==1.3.0",
+    "sphinx-autodoc-typehints==1.25.2"
+]
+all = [
+    "influxdb==5.3.1",
+    "pymysql==1.1.0",
+    "psycopg2-binary==2.9.9"
+]
+
+[project.scripts]
+ping-river-monitor = "src.main:main"
+ping-river-api = "src.web_api:main"
+
+[project.urls]
+Homepage = "https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor"
+Repository = "https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor"
+Issues = "https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/issues"
+Documentation = "https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/wiki"
+
+[dependency-groups]
+dev = [
+    # Testing
+    "pytest==7.4.3",
+    "pytest-cov==4.1.0",
+    "pytest-asyncio==0.21.1",
+    # Code formatting and linting
+    "black==23.11.0",
+    "flake8==6.1.0",
+    "isort==5.12.0",
+    "mypy==1.7.1",
+    # Pre-commit hooks
+    "pre-commit==3.5.0",
+    # Development tools
+    "ipython==8.17.2",
+    "jupyter==1.0.0",
+    # Type stubs
+    "types-requests==2.31.0.10",
+    "types-python-dateutil==2.8.19.14",
+    # Documentation
+    "sphinx==7.2.6",
+    "sphinx-rtd-theme==1.3.0",
+    "sphinx-autodoc-typehints==1.25.2",
+    "pyinstaller>=6.16.0",
+]
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-dir]
+"" = "src"

requirements.txt

@@ -2,6 +2,10 @@
 requests==2.31.0
 schedule==1.2.0
 pandas==2.0.3
+numpy>=1.24,<2  # pandas 2.0.3 wheels are ABI-incompatible with numpy 2.x
+
+# Flood forecasting (ML)
+scikit-learn==1.9.0
 
 # Web API framework
 fastapi==0.104.1

scripts/encode_password.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+"""
+Password URL encoder for PostgreSQL connection strings
+"""
+
+import urllib.parse
+import sys
+
+def encode_password(password: str) -> str:
+    """URL encode a password for use in connection strings"""
+    return urllib.parse.quote(password, safe='')
+
+def build_connection_string(username: str, password: str, host: str, port: int, database: str) -> str:
+    """Build a properly encoded PostgreSQL connection string"""
+    encoded_password = encode_password(password)
+    return f"postgresql://{username}:{encoded_password}@{host}:{port}/{database}"
+
+def main():
+    print("PostgreSQL Password URL Encoder")
+    print("=" * 40)
+
+    if len(sys.argv) > 1:
+        # Password provided as argument
+        password = sys.argv[1]
+    else:
+        # Interactive mode
+        password = input("Enter your password: ")
+
+    encoded = encode_password(password)
+
+    print(f"\nOriginal password: {password}")
+    print(f"URL encoded: {encoded}")
+
+    # Optional: build full connection string
+    try:
+        build_full = input("\nBuild full connection string? (y/N): ").strip().lower() == 'y'
+    except (EOFError, KeyboardInterrupt):
+        print("\nDone!")
+        return
+
+    if build_full:
+        username = input("Username: ").strip()
+        host = input("Host: ").strip()
+        port = input("Port [5432]: ").strip() or "5432"
+        database = input("Database [water_monitoring]: ").strip() or "water_monitoring"
+
+        connection_string = build_connection_string(username, password, host, int(port), database)
+
+        print(f"\nComplete connection string:")
+        print(f"POSTGRES_CONNECTION_STRING={connection_string}")
+
+        print(f"\nAdd this to your .env file:")
+        print(f"DB_TYPE=postgresql")
+        print(f"POSTGRES_CONNECTION_STRING={connection_string}")
+
+if __name__ == "__main__":
+    main()

scripts/generate_badges.py

@@ -29,7 +29,7 @@ def main():
         "FastAPI": generate_badge_url("FastAPI", "0.104%2B", "green"),
         "Docker": generate_badge_url("Docker", "Ready", "blue"),
         "License": generate_badge_url("License", "MIT", "green"),
-        "Version": generate_badge_url("Version", "v3.1.2", "blue"),
+        "Version": generate_badge_url("Version", "v3.1.3", "blue"),
     }
     
     print("# Status Badges")

scripts/init_git.bat

@@ -13,7 +13,7 @@ REM Add all files
 git add .
 
 REM Initial commit
-git commit -m "Initial commit: Northern Thailand Ping River Monitor v3.1.2
+git commit -m "Initial commit: Northern Thailand Ping River Monitor v3.1.3
 
 Features:
 - Real-time water level monitoring for Ping River Basin

scripts/init_git.sh

@@ -66,7 +66,7 @@ fi
 git add .
 
 # Initial commit
-git commit -m "Initial commit: Northern Thailand Ping River Monitor v3.1.2
+git commit -m "Initial commit: Northern Thailand Ping River Monitor v3.1.3
 
 Features:
 - Real-time water level monitoring for Ping River Basin

scripts/install.sh

@@ -0,0 +1,114 @@
+#!/usr/bin/env bash
+#
+# Install the Thailand Water Level Monitor as a hardened systemd service.
+#
+# Creates a dedicated system user, deploys the code to /opt, builds a uv-managed
+# virtualenv, installs the systemd unit, and enables the service. Idempotent:
+# safe to re-run to update an existing install.
+#
+# Usage (as root, from a checkout of the repo):
+#     sudo bash scripts/install.sh
+#
+# Override defaults via environment variables:
+#     APP_DIR=/opt/thailand-water-monitor SERVICE_USER=water-monitor sudo -E bash scripts/install.sh
+#
+set -euo pipefail
+
+APP_DIR="${APP_DIR:-/opt/thailand-water-monitor}"
+SERVICE_USER="${SERVICE_USER:-water-monitor}"
+SERVICE_GROUP="${SERVICE_GROUP:-${SERVICE_USER}}"
+SERVICE_NAME="water-monitor.service"
+
+# Resolve the repo root (parent of this scripts/ directory).
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
+
+log() { printf '\033[1;32m==>\033[0m %s\n' "$*"; }
+warn() { printf '\033[1;33m[warn]\033[0m %s\n' "$*"; }
+die() { printf '\033[1;31m[error]\033[0m %s\n' "$*" >&2; exit 1; }
+
+[ "$(id -u)" -eq 0 ] || die "This script must be run as root (use sudo)."
+
+# 1. Dedicated system user/group (no login, no home) --------------------------
+if ! getent group "${SERVICE_GROUP}" >/dev/null; then
+    log "Creating group ${SERVICE_GROUP}"
+    groupadd --system "${SERVICE_GROUP}"
+fi
+if ! id "${SERVICE_USER}" >/dev/null 2>&1; then
+    log "Creating system user ${SERVICE_USER}"
+    useradd --system --no-create-home --shell /usr/sbin/nologin \
+        --gid "${SERVICE_GROUP}" "${SERVICE_USER}"
+fi
+
+# 2. Deploy code to APP_DIR ----------------------------------------------------
+log "Deploying code to ${APP_DIR}"
+mkdir -p "${APP_DIR}"
+if command -v rsync >/dev/null 2>&1; then
+    rsync -a --delete \
+        --exclude '.git' --exclude '.venv' --exclude 'venv' \
+        --exclude '__pycache__' --exclude '*.pyc' \
+        --exclude '*.db' --exclude '.env' --exclude 'stations.json' \
+        "${REPO_DIR}/" "${APP_DIR}/"
+else
+    warn "rsync not found; falling back to cp (will not prune deleted files)"
+    cp -r "${REPO_DIR}/." "${APP_DIR}/"
+fi
+
+# 3. Build the uv-managed virtualenv ------------------------------------------
+# Prefer an already-installed uv. For stricter supply-chain control install uv
+# ahead of time via your distro / package manager; this script only fetches the
+# upstream installer (piped to a root shell) when AUTO_INSTALL_UV=1 is set, and
+# pins the version so the fetched script is reproducible.
+UV_VERSION="${UV_VERSION:-0.5.11}"
+if ! command -v uv >/dev/null 2>&1; then
+    if [ "${AUTO_INSTALL_UV:-0}" = "1" ]; then
+        warn "uv not found; installing pinned uv ${UV_VERSION} from astral.sh (runs as root)"
+        curl -LsSf "https://astral.sh/uv/${UV_VERSION}/install.sh" \
+            | env UV_INSTALL_DIR=/usr/local/bin sh
+    else
+        die "uv not found. Install it (e.g. your package manager, or 'pipx install uv'),
+     or re-run with AUTO_INSTALL_UV=1 to fetch the pinned upstream installer."
+    fi
+fi
+UV="$(command -v uv)"
+
+log "Creating virtualenv at ${APP_DIR}/venv"
+cd "${APP_DIR}"
+# Named 'venv' (not uv's default .venv) to match the systemd unit's ExecStart.
+"${UV}" venv venv
+"${UV}" pip install --python venv/bin/python -r requirements.txt
+
+# 4. Environment file ----------------------------------------------------------
+if [ ! -f "${APP_DIR}/.env" ]; then
+    if [ -f "${REPO_DIR}/.env" ]; then
+        log "Copying .env from checkout"
+        cp "${REPO_DIR}/.env" "${APP_DIR}/.env"
+    else
+        warn "No .env found. Copy .env.example to ${APP_DIR}/.env and fill in"
+        warn "MATRIX_ACCESS_TOKEN / MATRIX_ROOM_ID and DB settings before starting."
+    fi
+fi
+
+# 5. Ownership and permissions -------------------------------------------------
+# Service user needs write access for logs / stations.json.
+log "Setting ownership to ${SERVICE_USER}:${SERVICE_GROUP}"
+chown -R "${SERVICE_USER}:${SERVICE_GROUP}" "${APP_DIR}"
+# Restrict traversal to root + the service user, and lock down the secrets file
+# (contains the Matrix token and DB credentials).
+chmod 0750 "${APP_DIR}"
+if [ -f "${APP_DIR}/.env" ]; then
+    chmod 0600 "${APP_DIR}/.env"
+fi
+
+# 6. Install and enable the systemd unit --------------------------------------
+log "Installing systemd unit"
+install -m 0644 "${SCRIPT_DIR}/${SERVICE_NAME}" "/etc/systemd/system/${SERVICE_NAME}"
+systemctl daemon-reload
+systemctl enable "${SERVICE_NAME}"
+
+log "Done."
+echo
+echo "Next steps:"
+echo "  sudo systemctl start ${SERVICE_NAME}"
+echo "  systemctl status ${SERVICE_NAME}"
+echo "  sudo journalctl -u ${SERVICE_NAME} -f"

scripts/migrate_sqlite_to_postgres.py

@@ -0,0 +1,619 @@
+#!/usr/bin/env python3
+"""
+SQLite to PostgreSQL Migration Tool
+Migrates all data from SQLite database to PostgreSQL
+"""
+
+import os
+import sys
+import logging
+import sqlite3
+from datetime import datetime, timezone
+from typing import Dict, List, Optional, Tuple, Any
+from dataclasses import dataclass
+
+# Add src to path for imports
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src'))
+
+@dataclass
+class MigrationStats:
+    stations_migrated: int = 0
+    measurements_migrated: int = 0
+    errors: List[str] = None
+    start_time: Optional[datetime] = None
+    end_time: Optional[datetime] = None
+
+    def __post_init__(self):
+        if self.errors is None:
+            self.errors = []
+
+class SQLiteToPostgresMigrator:
+    def __init__(self, sqlite_path: str, postgres_config: Dict[str, Any]):
+        self.sqlite_path = sqlite_path
+        self.postgres_config = postgres_config
+        self.sqlite_conn = None
+        self.postgres_adapter = None
+        self.stats = MigrationStats()
+
+        # Setup logging with UTF-8 encoding
+        logging.basicConfig(
+            level=logging.INFO,
+            format='%(asctime)s - %(levelname)s - %(message)s',
+            handlers=[
+                logging.StreamHandler(),
+                logging.FileHandler('migration.log', encoding='utf-8')
+            ]
+        )
+        self.logger = logging.getLogger(__name__)
+
+    def connect_databases(self) -> bool:
+        """Connect to both SQLite and PostgreSQL databases"""
+        try:
+            # Connect to SQLite
+            if not os.path.exists(self.sqlite_path):
+                self.logger.error(f"SQLite database not found: {self.sqlite_path}")
+                return False
+
+            self.sqlite_conn = sqlite3.connect(self.sqlite_path)
+            self.sqlite_conn.row_factory = sqlite3.Row  # For dict-like access
+            self.logger.info(f"Connected to SQLite database: {self.sqlite_path}")
+
+            # Connect to PostgreSQL
+            from database_adapters import create_database_adapter
+            self.postgres_adapter = create_database_adapter(
+                self.postgres_config['type'],
+                connection_string=self.postgres_config['connection_string']
+            )
+
+            if not self.postgres_adapter.connect():
+                self.logger.error("Failed to connect to PostgreSQL")
+                return False
+
+            self.logger.info("Connected to PostgreSQL database")
+            return True
+
+        except Exception as e:
+            self.logger.error(f"Database connection error: {e}")
+            return False
+
+    def analyze_sqlite_schema(self) -> Dict[str, List[str]]:
+        """Analyze SQLite database structure"""
+        try:
+            cursor = self.sqlite_conn.cursor()
+
+            # Get all tables
+            cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'")
+            tables = [row[0] for row in cursor.fetchall()]
+
+            schema_info = {}
+            for table in tables:
+                cursor.execute(f"PRAGMA table_info({table})")
+                columns = [row[1] for row in cursor.fetchall()]
+                schema_info[table] = columns
+
+                # Get row count
+                cursor.execute(f"SELECT COUNT(*) FROM {table}")
+                count = cursor.fetchone()[0]
+                self.logger.info(f"Table '{table}': {len(columns)} columns, {count} rows")
+
+            return schema_info
+
+        except Exception as e:
+            self.logger.error(f"Schema analysis error: {e}")
+            return {}
+
+    def migrate_stations(self) -> bool:
+        """Migrate station data"""
+        try:
+            cursor = self.sqlite_conn.cursor()
+
+            # Try different possible table names and structures
+            station_queries = [
+                # Modern structure
+                """SELECT id, station_code, station_name_th as thai_name, station_name_en as english_name,
+                         latitude, longitude, geohash, created_at, updated_at
+                  FROM stations""",
+
+                # Alternative structure 1
+                """SELECT id, station_code, thai_name, english_name,
+                         latitude, longitude, geohash, created_at, updated_at
+                  FROM stations""",
+
+                # Legacy structure
+                """SELECT station_id as id, station_code, station_name as thai_name,
+                         station_name as english_name, lat as latitude, lon as longitude,
+                         NULL as geohash, datetime('now') as created_at, datetime('now') as updated_at
+                  FROM water_stations""",
+
+                # Simple structure
+                """SELECT rowid as id, station_code, name as thai_name, name as english_name,
+                         NULL as latitude, NULL as longitude, NULL as geohash,
+                         datetime('now') as created_at, datetime('now') as updated_at
+                  FROM stations""",
+            ]
+
+            stations_data = []
+
+            for query in station_queries:
+                try:
+                    cursor.execute(query)
+                    rows = cursor.fetchall()
+                    if rows:
+                        self.logger.info(f"Found {len(rows)} stations using query variant")
+
+                        for row in rows:
+                            station = {
+                                'station_id': row[0],
+                                'station_code': row[1] or f"STATION_{row[0]}",
+                                'station_name_th': row[2] or f"Station {row[0]}",
+                                'station_name_en': row[3] or f"Station {row[0]}",
+                                'latitude': row[4],
+                                'longitude': row[5],
+                                'geohash': row[6],
+                                'status': 'active'
+                            }
+                            stations_data.append(station)
+                        break
+
+                except sqlite3.OperationalError as e:
+                    if "no such table" in str(e).lower() or "no such column" in str(e).lower():
+                        continue
+                    else:
+                        raise
+
+            if not stations_data:
+                self.logger.warning("No stations found in SQLite database")
+                return True
+
+            # Insert stations into PostgreSQL using raw SQL
+            # Since the adapter is designed for measurements, we'll use direct SQL
+            try:
+                from sqlalchemy import create_engine, text
+                engine = create_engine(self.postgres_config['connection_string'])
+
+                # Process stations individually to avoid transaction rollback issues
+                for station in stations_data:
+                    try:
+                        with engine.begin() as conn:
+                            # Use PostgreSQL UPSERT syntax with correct column names
+                            station_sql = """
+                            INSERT INTO stations (id, station_code, thai_name, english_name, latitude, longitude, geohash)
+                            VALUES (:station_id, :station_code, :thai_name, :english_name, :latitude, :longitude, :geohash)
+                            ON CONFLICT (id) DO UPDATE SET
+                                thai_name = EXCLUDED.thai_name,
+                                english_name = EXCLUDED.english_name,
+                                latitude = EXCLUDED.latitude,
+                                longitude = EXCLUDED.longitude,
+                                geohash = EXCLUDED.geohash,
+                                updated_at = CURRENT_TIMESTAMP
+                            """
+
+                            conn.execute(text(station_sql), {
+                                'station_id': station['station_id'],
+                                'station_code': station['station_code'],
+                                'thai_name': station['station_name_th'],
+                                'english_name': station['station_name_en'],
+                                'latitude': station.get('latitude'),
+                                'longitude': station.get('longitude'),
+                                'geohash': station.get('geohash')
+                            })
+
+                            self.stats.stations_migrated += 1
+
+                    except Exception as e:
+                        error_msg = f"Error migrating station {station.get('station_code', 'unknown')}: {str(e)[:100]}..."
+                        self.logger.warning(error_msg)
+                        self.stats.errors.append(error_msg)
+
+                self.logger.info(f"Migrated {self.stats.stations_migrated} stations")
+
+            except Exception as e:
+                self.logger.error(f"Station migration failed: {e}")
+                return False
+
+            self.logger.info(f"Migrated {self.stats.stations_migrated} stations")
+            return True
+
+        except Exception as e:
+            self.logger.error(f"Station migration error: {e}")
+            return False
+
+    def migrate_measurements(self, batch_size: int = 5000) -> bool:
+        """Migrate measurement data in batches"""
+        try:
+            cursor = self.sqlite_conn.cursor()
+
+            # Try different possible measurement table structures
+            measurement_queries = [
+                # Modern structure
+                """SELECT w.timestamp, w.station_id, s.station_code, s.station_name_th, s.station_name_en,
+                         w.water_level, w.discharge, w.discharge_percent, w.status
+                  FROM water_measurements w
+                  JOIN stations s ON w.station_id = s.id
+                  ORDER BY w.timestamp""",
+
+                # Alternative with different join
+                """SELECT w.timestamp, w.station_id, s.station_code, s.thai_name, s.english_name,
+                         w.water_level, w.discharge, w.discharge_percent, 'active' as status
+                  FROM water_measurements w
+                  JOIN stations s ON w.station_id = s.id
+                  ORDER BY w.timestamp""",
+
+                # Legacy structure
+                """SELECT timestamp, station_id, station_code, station_name, station_name,
+                         water_level, discharge, discharge_percent, 'active' as status
+                  FROM measurements
+                  ORDER BY timestamp""",
+
+                # Simple structure without joins
+                """SELECT timestamp, station_id, 'UNKNOWN' as station_code, 'Unknown' as station_name_th, 'Unknown' as station_name_en,
+                         water_level, discharge, discharge_percent, 'active' as status
+                  FROM water_measurements
+                  ORDER BY timestamp""",
+            ]
+
+            measurements_processed = 0
+
+            for query in measurement_queries:
+                try:
+                    # Get total count first
+                    count_query = query.replace("SELECT", "SELECT COUNT(*) FROM (SELECT").replace("ORDER BY w.timestamp", "") + ")"
+                    cursor.execute(count_query)
+                    total_measurements = cursor.fetchone()[0]
+
+                    if total_measurements == 0:
+                        continue
+
+                    self.logger.info(f"Found {total_measurements} measurements to migrate")
+
+                    # Process in batches
+                    offset = 0
+                    while True:
+                        batch_query = f"{query} LIMIT {batch_size} OFFSET {offset}"
+                        cursor.execute(batch_query)
+                        rows = cursor.fetchall()
+
+                        if not rows:
+                            break
+
+                        # Convert to measurement format
+                        measurements = []
+                        for row in rows:
+                            try:
+                                # Parse timestamp
+                                timestamp_str = row[0]
+                                if isinstance(timestamp_str, str):
+                                    try:
+                                        timestamp = datetime.fromisoformat(timestamp_str.replace('Z', '+00:00'))
+                                    except:
+                                        # Try other common formats
+                                        for fmt in ['%Y-%m-%d %H:%M:%S', '%Y-%m-%d %H:%M:%S.%f', '%Y-%m-%dT%H:%M:%S']:
+                                            try:
+                                                timestamp = datetime.strptime(timestamp_str, fmt)
+                                                break
+                                            except:
+                                                continue
+                                        else:
+                                            timestamp = datetime.now()
+                                else:
+                                    timestamp = timestamp_str
+
+                                measurement = {
+                                    'timestamp': timestamp,
+                                    'station_id': row[1] or 999,
+                                    'station_code': row[2] or 'UNKNOWN',
+                                    'station_name_th': row[3] or 'Unknown',
+                                    'station_name_en': row[4] or 'Unknown',
+                                    'water_level': float(row[5]) if row[5] is not None else None,
+                                    'discharge': float(row[6]) if row[6] is not None else None,
+                                    'discharge_percent': float(row[7]) if row[7] is not None else None,
+                                    'status': row[8] or 'active'
+                                }
+                                measurements.append(measurement)
+
+                            except Exception as e:
+                                error_msg = f"Error processing measurement row: {e}"
+                                self.logger.warning(error_msg)
+                                continue
+
+                        # Save batch to PostgreSQL using fast bulk insert
+                        if measurements:
+                            try:
+                                self._fast_bulk_insert(measurements)
+                                measurements_processed += len(measurements)
+                                self.stats.measurements_migrated += len(measurements)
+                                self.logger.info(f"Migrated {measurements_processed}/{total_measurements} measurements")
+                            except Exception as e:
+                                error_msg = f"Error saving measurement batch: {e}"
+                                self.logger.error(error_msg)
+                                self.stats.errors.append(error_msg)
+
+                        offset += batch_size
+
+                    # If we processed measurements, we're done
+                    if measurements_processed > 0:
+                        break
+
+                except sqlite3.OperationalError as e:
+                    if "no such table" in str(e).lower() or "no such column" in str(e).lower():
+                        continue
+                    else:
+                        raise
+
+            if measurements_processed == 0:
+                self.logger.warning("No measurements found in SQLite database")
+            else:
+                self.logger.info(f"Successfully migrated {measurements_processed} measurements")
+
+            return True
+
+        except Exception as e:
+            self.logger.error(f"Measurement migration error: {e}")
+            return False
+
+    def _fast_bulk_insert(self, measurements: List[Dict]) -> bool:
+        """Super fast bulk insert using PostgreSQL COPY or VALUES clause"""
+        try:
+            import psycopg2
+            from urllib.parse import urlparse
+            import io
+
+            # Parse connection string for direct psycopg2 connection
+            parsed = urlparse(self.postgres_config['connection_string'])
+
+            # Try super fast COPY method first
+            try:
+                conn = psycopg2.connect(
+                    host=parsed.hostname,
+                    port=parsed.port or 5432,
+                    database=parsed.path[1:],
+                    user=parsed.username,
+                    password=parsed.password
+                )
+
+                with conn:
+                    with conn.cursor() as cur:
+                        # Prepare data for COPY
+                        data_buffer = io.StringIO()
+                        null_val = '\\N'
+                        for m in measurements:
+                            data_buffer.write(f"{m['timestamp']}\t{m['station_id']}\t{m['water_level'] or null_val}\t{m['discharge'] or null_val}\t{m['discharge_percent'] or null_val}\t{m['status']}\n")
+
+                        data_buffer.seek(0)
+
+                        # Use COPY for maximum speed
+                        cur.copy_from(
+                            data_buffer,
+                            'water_measurements',
+                            columns=('timestamp', 'station_id', 'water_level', 'discharge', 'discharge_percent', 'status'),
+                            sep='\t'
+                        )
+
+                conn.close()
+                return True
+
+            except Exception as copy_error:
+                # Fallback to SQLAlchemy bulk insert
+                self.logger.debug(f"COPY failed, using bulk VALUES: {copy_error}")
+
+                from sqlalchemy import create_engine, text
+                engine = create_engine(self.postgres_config['connection_string'])
+
+                with engine.begin() as conn:
+                    # Use PostgreSQL's fast bulk insert with ON CONFLICT
+                    values_list = []
+                    for m in measurements:
+                        timestamp = m['timestamp'].isoformat() if hasattr(m['timestamp'], 'isoformat') else str(m['timestamp'])
+                        values_list.append(
+                            f"('{timestamp}', {m['station_id']}, {m['water_level'] or 'NULL'}, "
+                            f"{m['discharge'] or 'NULL'}, {m['discharge_percent'] or 'NULL'}, '{m['status']}')"
+                        )
+
+                    # Build bulk insert query with ON CONFLICT handling
+                    bulk_sql = f"""
+                    INSERT INTO water_measurements (timestamp, station_id, water_level, discharge, discharge_percent, status)
+                    VALUES {','.join(values_list)}
+                    ON CONFLICT (timestamp, station_id) DO UPDATE SET
+                        water_level = EXCLUDED.water_level,
+                        discharge = EXCLUDED.discharge,
+                        discharge_percent = EXCLUDED.discharge_percent,
+                        status = EXCLUDED.status
+                    """
+
+                    conn.execute(text(bulk_sql))
+                    return True
+
+        except Exception as e:
+            self.logger.warning(f"Fast bulk insert failed: {e}")
+            # Final fallback to original method
+            try:
+                success = self.postgres_adapter.save_measurements(measurements)
+                return success
+            except Exception as fallback_e:
+                self.logger.error(f"All insert methods failed: {fallback_e}")
+                return False
+
+    def verify_migration(self) -> bool:
+        """Verify the migration by comparing counts"""
+        try:
+            # Get SQLite counts
+            cursor = self.sqlite_conn.cursor()
+
+            sqlite_stations = 0
+            sqlite_measurements = 0
+
+            # Try to get station count
+            for table in ['stations', 'water_stations']:
+                try:
+                    cursor.execute(f"SELECT COUNT(*) FROM {table}")
+                    sqlite_stations = cursor.fetchone()[0]
+                    break
+                except:
+                    continue
+
+            # Try to get measurement count
+            for table in ['water_measurements', 'measurements']:
+                try:
+                    cursor.execute(f"SELECT COUNT(*) FROM {table}")
+                    sqlite_measurements = cursor.fetchone()[0]
+                    break
+                except:
+                    continue
+
+            # Get PostgreSQL counts
+            postgres_measurements = self.postgres_adapter.get_latest_measurements(limit=999999)
+            postgres_count = len(postgres_measurements)
+
+            self.logger.info("Migration Verification:")
+            self.logger.info(f"SQLite stations: {sqlite_stations}")
+            self.logger.info(f"SQLite measurements: {sqlite_measurements}")
+            self.logger.info(f"PostgreSQL measurements retrieved: {postgres_count}")
+            self.logger.info(f"Migrated stations: {self.stats.stations_migrated}")
+            self.logger.info(f"Migrated measurements: {self.stats.measurements_migrated}")
+
+            return True
+
+        except Exception as e:
+            self.logger.error(f"Verification error: {e}")
+            return False
+
+    def run_migration(self, sqlite_path: str = None) -> bool:
+        """Run the complete migration process"""
+        self.stats.start_time = datetime.now()
+
+        if sqlite_path:
+            self.sqlite_path = sqlite_path
+
+        self.logger.info("=" * 60)
+        self.logger.info("SQLite to PostgreSQL Migration Tool")
+        self.logger.info("=" * 60)
+        self.logger.info(f"SQLite database: {self.sqlite_path}")
+        self.logger.info(f"PostgreSQL: {self.postgres_config['type']}")
+
+        try:
+            # Step 1: Connect to databases
+            self.logger.info("Step 1: Connecting to databases...")
+            if not self.connect_databases():
+                return False
+
+            # Step 2: Analyze SQLite schema
+            self.logger.info("Step 2: Analyzing SQLite database structure...")
+            schema_info = self.analyze_sqlite_schema()
+            if not schema_info:
+                self.logger.error("Could not analyze SQLite database structure")
+                return False
+
+            # Step 3: Migrate stations
+            self.logger.info("Step 3: Migrating station data...")
+            if not self.migrate_stations():
+                self.logger.error("Station migration failed")
+                return False
+
+            # Step 4: Migrate measurements
+            self.logger.info("Step 4: Migrating measurement data...")
+            if not self.migrate_measurements():
+                self.logger.error("Measurement migration failed")
+                return False
+
+            # Step 5: Verify migration
+            self.logger.info("Step 5: Verifying migration...")
+            self.verify_migration()
+
+            self.stats.end_time = datetime.now()
+            duration = self.stats.end_time - self.stats.start_time
+
+            # Final report
+            self.logger.info("=" * 60)
+            self.logger.info("MIGRATION COMPLETED")
+            self.logger.info("=" * 60)
+            self.logger.info(f"Duration: {duration}")
+            self.logger.info(f"Stations migrated: {self.stats.stations_migrated}")
+            self.logger.info(f"Measurements migrated: {self.stats.measurements_migrated}")
+
+            if self.stats.errors:
+                self.logger.warning(f"Errors encountered: {len(self.stats.errors)}")
+                for error in self.stats.errors[:10]:  # Show first 10 errors
+                    self.logger.warning(f"  - {error}")
+                if len(self.stats.errors) > 10:
+                    self.logger.warning(f"  ... and {len(self.stats.errors) - 10} more errors")
+            else:
+                self.logger.info("No errors encountered")
+
+            return True
+
+        except Exception as e:
+            self.logger.error(f"Migration failed: {e}")
+            return False
+
+        finally:
+            # Cleanup
+            if self.sqlite_conn:
+                self.sqlite_conn.close()
+
+def main():
+    """Main entry point"""
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Migrate SQLite data to PostgreSQL")
+    parser.add_argument("sqlite_path", nargs="?", help="Path to SQLite database file")
+    parser.add_argument("--batch-size", type=int, default=5000, help="Batch size for processing measurements")
+    parser.add_argument("--fast", action="store_true", help="Use maximum speed mode (batch-size 10000)")
+    parser.add_argument("--dry-run", action="store_true", help="Analyze only, don't migrate")
+
+    args = parser.parse_args()
+
+    # Set fast mode
+    if args.fast:
+        args.batch_size = 10000
+
+    # Get SQLite path
+    sqlite_path = args.sqlite_path
+    if not sqlite_path:
+        # Try to find common SQLite database files
+        possible_paths = [
+            "water_levels.db",
+            "water_monitoring.db",
+            "database.db",
+            "../water_levels.db"
+        ]
+
+        for path in possible_paths:
+            if os.path.exists(path):
+                sqlite_path = path
+                break
+
+        if not sqlite_path:
+            print("SQLite database file not found. Please specify the path:")
+            print("  python migrate_sqlite_to_postgres.py /path/to/database.db")
+            return False
+
+    # Get PostgreSQL configuration
+    try:
+        from config import Config
+        postgres_config = Config.get_database_config()
+
+        if postgres_config['type'] != 'postgresql':
+            print("Error: PostgreSQL not configured. Set DB_TYPE=postgresql in your .env file")
+            return False
+
+    except Exception as e:
+        print(f"Error loading PostgreSQL configuration: {e}")
+        return False
+
+    # Run migration
+    migrator = SQLiteToPostgresMigrator(sqlite_path, postgres_config)
+
+    if args.dry_run:
+        print("DRY RUN MODE - Analyzing SQLite database structure only")
+        if migrator.connect_databases():
+            schema_info = migrator.analyze_sqlite_schema()
+            print("\nSQLite database structure analysis complete.")
+            print("Run without --dry-run to perform the actual migration.")
+        return True
+
+    success = migrator.run_migration()
+    return success
+
+if __name__ == "__main__":
+    success = main()
+    sys.exit(0 if success else 1)

scripts/setup_postgres.py

@@ -0,0 +1,175 @@
+#!/usr/bin/env python3
+"""
+PostgreSQL setup script for Northern Thailand Ping River Monitor
+This script helps you configure and test your PostgreSQL connection
+"""
+
+import os
+import sys
+import logging
+from typing import Optional
+from urllib.parse import urlparse
+
+def setup_logging():
+    logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
+
+def test_postgres_connection(connection_string: str) -> bool:
+    """Test connection to PostgreSQL database"""
+    try:
+        from sqlalchemy import create_engine, text
+
+        # Test connection
+        engine = create_engine(connection_string, pool_pre_ping=True)
+        with engine.connect() as conn:
+            result = conn.execute(text("SELECT version()"))
+            version = result.fetchone()[0]
+            logging.info(f"✅ Connected to PostgreSQL successfully!")
+            logging.info(f"Database version: {version}")
+            return True
+
+    except ImportError:
+        logging.error("❌ psycopg2-binary not installed. Run: uv add psycopg2-binary")
+        return False
+    except Exception as e:
+        logging.error(f"❌ Connection failed: {e}")
+        return False
+
+def parse_connection_string(connection_string: str) -> dict:
+    """Parse PostgreSQL connection string into components"""
+    try:
+        parsed = urlparse(connection_string)
+        return {
+            'host': parsed.hostname,
+            'port': parsed.port or 5432,
+            'database': parsed.path[1:] if parsed.path else None,
+            'username': parsed.username,
+            'password': parsed.password,
+        }
+    except Exception as e:
+        logging.error(f"Failed to parse connection string: {e}")
+        return {}
+
+def create_database_if_not_exists(connection_string: str, database_name: str) -> bool:
+    """Create database if it doesn't exist"""
+    try:
+        from sqlalchemy import create_engine, text
+
+        # Connect to default postgres database to create our database
+        parsed = urlparse(connection_string)
+        admin_connection = connection_string.replace(f"/{parsed.path[1:]}", "/postgres")
+
+        engine = create_engine(admin_connection, pool_pre_ping=True)
+
+        with engine.connect() as conn:
+            # Check if database exists
+            result = conn.execute(text(
+                "SELECT 1 FROM pg_database WHERE datname = :db_name"
+            ), {"db_name": database_name})
+
+            if result.fetchone():
+                logging.info(f"✅ Database '{database_name}' already exists")
+                return True
+            else:
+                # Create database
+                conn.execute(text("COMMIT"))  # End transaction
+                conn.execute(text(f'CREATE DATABASE "{database_name}"'))
+                logging.info(f"✅ Created database '{database_name}'")
+                return True
+
+    except Exception as e:
+        logging.error(f"❌ Failed to create database: {e}")
+        return False
+
+def initialize_tables(connection_string: str) -> bool:
+    """Initialize database tables"""
+    try:
+        # Import the database adapter to create tables
+        sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src'))
+        from database_adapters import SQLAdapter
+
+        adapter = SQLAdapter(connection_string=connection_string, db_type='postgresql')
+        if adapter.connect():
+            logging.info("✅ Database tables initialized successfully")
+            return True
+        else:
+            logging.error("❌ Failed to initialize tables")
+            return False
+
+    except Exception as e:
+        logging.error(f"❌ Failed to initialize tables: {e}")
+        return False
+
+def interactive_setup():
+    """Interactive setup wizard"""
+    print("🐘 PostgreSQL Setup Wizard for Ping River Monitor")
+    print("=" * 50)
+
+    # Get connection details
+    host = input("PostgreSQL host (e.g., 192.168.1.100): ").strip()
+    port = input("PostgreSQL port [5432]: ").strip() or "5432"
+    database = input("Database name [water_monitoring]: ").strip() or "water_monitoring"
+    username = input("Username: ").strip()
+    password = input("Password: ").strip()
+
+    # Optional SSL
+    use_ssl = input("Use SSL connection? (y/N): ").strip().lower() == 'y'
+    ssl_params = "?sslmode=require" if use_ssl else ""
+
+    connection_string = f"postgresql://{username}:{password}@{host}:{port}/{database}{ssl_params}"
+
+    print(f"\nGenerated connection string:")
+    print(f"POSTGRES_CONNECTION_STRING={connection_string}")
+
+    return connection_string
+
+def main():
+    setup_logging()
+
+    print("🚀 Northern Thailand Ping River Monitor - PostgreSQL Setup")
+    print("=" * 60)
+
+    # Check if connection string is provided via environment
+    connection_string = os.getenv('POSTGRES_CONNECTION_STRING')
+
+    if not connection_string:
+        print("No POSTGRES_CONNECTION_STRING found in environment.")
+        print("Starting interactive setup...\n")
+        connection_string = interactive_setup()
+
+        # Suggest adding to .env file
+        print(f"\n💡 Add this to your .env file:")
+        print(f"DB_TYPE=postgresql")
+        print(f"POSTGRES_CONNECTION_STRING={connection_string}")
+
+    # Parse connection details
+    config = parse_connection_string(connection_string)
+    if not config.get('host'):
+        logging.error("Invalid connection string format")
+        return False
+
+    print(f"\n🔗 Connecting to PostgreSQL at {config['host']}:{config['port']}")
+
+    # Test connection
+    if not test_postgres_connection(connection_string):
+        return False
+
+    # Try to create database
+    database_name = config.get('database', 'water_monitoring')
+    if database_name:
+        create_database_if_not_exists(connection_string, database_name)
+
+    # Initialize tables
+    if not initialize_tables(connection_string):
+        return False
+
+    print("\n🎉 PostgreSQL setup completed successfully!")
+    print("\nNext steps:")
+    print("1. Update your .env file with the connection string")
+    print("2. Run: make run-test")
+    print("3. Run: make run-api")
+
+    return True
+
+if __name__ == "__main__":
+    success = main()
+    sys.exit(0 if success else 1)

scripts/setup_uv.bat

@@ -0,0 +1,48 @@
+@echo off
+REM Setup script for uv-based development environment on Windows
+
+echo 🚀 Setting up Northern Thailand Ping River Monitor with uv...
+
+REM Check if uv is installed
+uv --version >nul 2>&1
+if %errorlevel% neq 0 (
+    echo ❌ uv is not installed. Please install it first:
+    echo    powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"
+    exit /b 1
+)
+
+echo ✅ uv found
+uv --version
+
+REM Initialize uv project if not already initialized
+if not exist "uv.lock" (
+    echo 🔧 Initializing uv project...
+    uv sync
+) else (
+    echo 📦 Syncing dependencies with uv...
+    uv sync
+)
+
+REM Install pre-commit hooks
+echo 🎣 Installing pre-commit hooks...
+uv run pre-commit install
+
+REM Create .env file if it doesn't exist
+if not exist ".env" (
+    if exist ".env.example" (
+        echo 📝 Creating .env file from template...
+        copy .env.example .env
+        echo ⚠️  Please edit .env file with your configuration
+    )
+)
+
+echo ✅ Setup complete!
+echo.
+echo 📚 Quick start commands:
+echo   make install-dev  # Install all dependencies
+echo   make run-test     # Run a test cycle
+echo   make run-api      # Start the web API
+echo   make test         # Run tests
+echo   make lint         # Check code quality
+echo.
+echo 🎉 Happy monitoring!

scripts/setup_uv.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+# Setup script for uv-based development environment
+
+set -e
+
+echo "🚀 Setting up Northern Thailand Ping River Monitor with uv..."
+
+# Check if uv is installed
+if ! command -v uv &> /dev/null; then
+    echo "❌ uv is not installed. Please install it first:"
+    echo "   curl -LsSf https://astral.sh/uv/install.sh | sh"
+    exit 1
+fi
+
+echo "✅ uv found: $(uv --version)"
+
+# Initialize uv project if not already initialized
+if [ ! -f "uv.lock" ]; then
+    echo "🔧 Initializing uv project..."
+    uv sync
+else
+    echo "📦 Syncing dependencies with uv..."
+    uv sync
+fi
+
+# Install pre-commit hooks
+echo "🎣 Installing pre-commit hooks..."
+uv run pre-commit install
+
+# Create .env file if it doesn't exist
+if [ ! -f ".env" ] && [ -f ".env.example" ]; then
+    echo "📝 Creating .env file from template..."
+    cp .env.example .env
+    echo "⚠️  Please edit .env file with your configuration"
+fi
+
+echo "✅ Setup complete!"
+echo ""
+echo "📚 Quick start commands:"
+echo "  make install-dev  # Install all dependencies"
+echo "  make run-test     # Run a test cycle"
+echo "  make run-api      # Start the web API"
+echo "  make test         # Run tests"
+echo "  make lint         # Check code quality"
+echo ""
+echo "🎉 Happy monitoring!"

scripts/train_flood_model.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+"""CLI entry point for training the Ping River flood forecast models.
+
+Usage:
+    python scripts/train_flood_model.py --stations all
+    python scripts/train_flood_model.py --stations P.1,P.103 --skip-eval
+"""
+
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+from src.ml.train import main
+
+if __name__ == "__main__":
+    main()

scripts/water-monitor.service

@@ -1,6 +1,6 @@
 [Unit]
 Description=Thailand Water Level Monitor
-Documentation=https://github.com/your-username/thailand-water-monitor
+Documentation=https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor
 After=network.target
 Wants=network-online.target
 

setup.py.backup

@@ -11,8 +11,18 @@ with open("README.md", "r", encoding="utf-8") as fh:
     long_description = fh.read()
 
 # Read requirements
-with open("requirements.txt", "r", encoding="utf-8") as fh:
+try:
+    with open("requirements.txt", "r", encoding="utf-8") as fh:
         requirements = [line.strip() for line in fh if line.strip() and not line.startswith("#")]
+except FileNotFoundError:
+    # Fallback to minimal requirements if file not found
+    requirements = [
+        "requests>=2.31.0",
+        "schedule>=1.2.0",
+        "pandas>=2.1.0",
+        "fastapi>=0.104.0",
+        "uvicorn>=0.24.0",
+    ]
 
 # Extract core requirements (exclude dev dependencies)
 core_requirements = []
@@ -22,7 +32,7 @@ for req in requirements:
 
 setup(
     name="northern-thailand-ping-river-monitor",
-    version="3.1.2",
+    version="3.1.3",
     author="Ping River Monitor Team",
     author_email="contact@example.com",
     description="Real-time water level monitoring system for the Ping River Basin in Northern Thailand",

sql/init_postgres.sql

@@ -0,0 +1,162 @@
+-- Northern Thailand Ping River Monitor - PostgreSQL Database Schema
+-- This script initializes the database tables for water monitoring data
+
+-- Enable required extensions
+CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
+
+-- Create schema for better organization
+CREATE SCHEMA IF NOT EXISTS water_monitor;
+SET search_path TO water_monitor, public;
+
+-- Stations table - stores monitoring station information
+CREATE TABLE IF NOT EXISTS stations (
+    id SERIAL PRIMARY KEY,
+    station_code VARCHAR(10) UNIQUE NOT NULL,
+    thai_name VARCHAR(255) NOT NULL,
+    english_name VARCHAR(255) NOT NULL,
+    latitude DECIMAL(10,8),
+    longitude DECIMAL(11,8),
+    geohash VARCHAR(20),
+    elevation DECIMAL(8,2), -- meters above sea level
+    river_basin VARCHAR(100),
+    province VARCHAR(100),
+    district VARCHAR(100),
+    is_active BOOLEAN DEFAULT true,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+);
+
+-- Water measurements table - stores time series data
+CREATE TABLE IF NOT EXISTS water_measurements (
+    id BIGSERIAL PRIMARY KEY,
+    timestamp TIMESTAMP NOT NULL,
+    station_id INTEGER NOT NULL,
+    water_level NUMERIC(10,3), -- meters
+    discharge NUMERIC(10,2), -- cubic meters per second
+    discharge_percent NUMERIC(5,2), -- percentage of normal discharge
+    status VARCHAR(20) DEFAULT 'active',
+    data_quality VARCHAR(20) DEFAULT 'good', -- good, fair, poor, missing
+    remarks TEXT,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    FOREIGN KEY (station_id) REFERENCES stations(id) ON DELETE CASCADE,
+    UNIQUE(timestamp, station_id)
+);
+
+-- Alert thresholds table - stores warning/danger levels for each station
+CREATE TABLE IF NOT EXISTS alert_thresholds (
+    id SERIAL PRIMARY KEY,
+    station_id INTEGER NOT NULL,
+    threshold_type VARCHAR(20) NOT NULL, -- 'warning', 'danger', 'critical'
+    water_level_min NUMERIC(10,3),
+    water_level_max NUMERIC(10,3),
+    discharge_min NUMERIC(10,2),
+    discharge_max NUMERIC(10,2),
+    is_active BOOLEAN DEFAULT true,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    FOREIGN KEY (station_id) REFERENCES stations(id) ON DELETE CASCADE
+);
+
+-- Data quality log - tracks data collection issues
+CREATE TABLE IF NOT EXISTS data_quality_log (
+    id BIGSERIAL PRIMARY KEY,
+    timestamp TIMESTAMP NOT NULL,
+    station_id INTEGER,
+    issue_type VARCHAR(50) NOT NULL, -- 'connection_failed', 'invalid_data', 'missing_data'
+    description TEXT,
+    severity VARCHAR(20) DEFAULT 'info', -- 'info', 'warning', 'error', 'critical'
+    resolved_at TIMESTAMP,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    FOREIGN KEY (station_id) REFERENCES stations(id) ON DELETE SET NULL
+);
+
+-- Create indexes for better query performance
+CREATE INDEX IF NOT EXISTS idx_water_measurements_timestamp ON water_measurements(timestamp DESC);
+CREATE INDEX IF NOT EXISTS idx_water_measurements_station_id ON water_measurements(station_id);
+CREATE INDEX IF NOT EXISTS idx_water_measurements_station_timestamp ON water_measurements(station_id, timestamp DESC);
+CREATE INDEX IF NOT EXISTS idx_water_measurements_status ON water_measurements(status);
+CREATE INDEX IF NOT EXISTS idx_stations_code ON stations(station_code);
+CREATE INDEX IF NOT EXISTS idx_stations_active ON stations(is_active);
+CREATE INDEX IF NOT EXISTS idx_data_quality_timestamp ON data_quality_log(timestamp DESC);
+CREATE INDEX IF NOT EXISTS idx_data_quality_station ON data_quality_log(station_id);
+
+-- Create a view for latest measurements per station
+CREATE OR REPLACE VIEW latest_measurements AS
+SELECT
+    s.id as station_id,
+    s.station_code,
+    s.english_name,
+    s.thai_name,
+    s.latitude,
+    s.longitude,
+    s.province,
+    s.river_basin,
+    m.timestamp,
+    m.water_level,
+    m.discharge,
+    m.discharge_percent,
+    m.status,
+    m.data_quality,
+    CASE
+        WHEN m.timestamp > CURRENT_TIMESTAMP - INTERVAL '2 hours' THEN 'online'
+        WHEN m.timestamp > CURRENT_TIMESTAMP - INTERVAL '24 hours' THEN 'delayed'
+        ELSE 'offline'
+    END as station_status
+FROM stations s
+LEFT JOIN LATERAL (
+    SELECT * FROM water_measurements
+    WHERE station_id = s.id
+    ORDER BY timestamp DESC
+    LIMIT 1
+) m ON true
+WHERE s.is_active = true
+ORDER BY s.station_code;
+
+-- Create a function to update the updated_at timestamp
+CREATE OR REPLACE FUNCTION update_modified_column()
+RETURNS TRIGGER AS $$
+BEGIN
+    NEW.updated_at = CURRENT_TIMESTAMP;
+    RETURN NEW;
+END;
+$$ language 'plpgsql';
+
+-- Create triggers to automatically update updated_at
+DROP TRIGGER IF EXISTS update_stations_modtime ON stations;
+CREATE TRIGGER update_stations_modtime
+    BEFORE UPDATE ON stations
+    FOR EACH ROW
+    EXECUTE FUNCTION update_modified_column();
+
+-- Insert sample stations (Northern Thailand Ping River stations)
+INSERT INTO stations (id, station_code, thai_name, english_name, latitude, longitude, province, river_basin) VALUES
+(1, 'P.1', 'เชียงใหม่', 'Chiang Mai', 18.7883, 98.9853, 'Chiang Mai', 'Ping River'),
+(2, 'P.4A', 'ท่าแพ', 'Tha Phae', 18.7875, 99.0045, 'Chiang Mai', 'Ping River'),
+(3, 'P.12', 'สันป่าตอง', 'San Pa Tong', 18.6167, 98.9500, 'Chiang Mai', 'Ping River'),
+(4, 'P.20', 'ลำพูน', 'Lamphun', 18.5737, 99.0081, 'Lamphun', 'Ping River'),
+(5, 'P.30', 'ลี้', 'Li', 17.4833, 99.3000, 'Lamphun', 'Ping River'),
+(6, 'P.35', 'ป่าซาง', 'Pa Sang', 18.5444, 98.9397, 'Lamphun', 'Ping River'),
+(7, 'P.67', 'ตาก', 'Tak', 16.8839, 99.1267, 'Tak', 'Ping River'),
+(8, 'P.75', 'สามเงา', 'Sam Ngao', 17.1019, 99.4644, 'Tak', 'Ping River')
+ON CONFLICT (id) DO NOTHING;
+
+-- Insert sample alert thresholds
+INSERT INTO alert_thresholds (station_id, threshold_type, water_level_min, water_level_max) VALUES
+(1, 'warning', 4.5, NULL),
+(1, 'danger', 6.0, NULL),
+(1, 'critical', 7.5, NULL),
+(2, 'warning', 4.0, NULL),
+(2, 'danger', 5.5, NULL),
+(2, 'critical', 7.0, NULL)
+ON CONFLICT DO NOTHING;
+
+-- Grant permissions (adjust as needed for your setup)
+GRANT USAGE ON SCHEMA water_monitor TO postgres;
+GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA water_monitor TO postgres;
+GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA water_monitor TO postgres;
+
+-- Optional: Create a read-only user for reporting
+-- CREATE USER water_monitor_readonly WITH PASSWORD 'readonly_password';
+-- GRANT USAGE ON SCHEMA water_monitor TO water_monitor_readonly;
+-- GRANT SELECT ON ALL TABLES IN SCHEMA water_monitor TO water_monitor_readonly;
+
+COMMIT;

src/__init__.py

@@ -6,7 +6,7 @@ A comprehensive real-time water level monitoring system for the Ping River Basin
 in Northern Thailand, covering Royal Irrigation Department (RID) stations.
 """
 
-__version__ = "3.1.2"
+__version__ = "3.1.3"
 __author__ = "Ping River Monitor Team"
 __description__ = "Northern Thailand Ping River Monitoring System"
 

src/alerting.py

@@ -0,0 +1,570 @@
+#!/usr/bin/env python3
+"""
+Water Level Alerting System with Matrix Integration
+"""
+
+import datetime
+import html
+import os
+import re
+from dataclasses import dataclass
+from enum import Enum
+from typing import Dict, List, Optional
+
+import requests
+
+try:
+    from .config import Config
+    from .database_adapters import create_database_adapter
+    from .logging_config import get_logger
+except ImportError:
+    import logging
+
+    from config import Config
+    from database_adapters import create_database_adapter
+
+    def get_logger(name):
+        return logging.getLogger(name)
+
+
+logger = get_logger(__name__)
+
+
+_BOLD_RE = re.compile(r"\*\*(.+?)\*\*")
+_URL_RE = re.compile(r"(https?://[^\s<]+)")
+
+
+def markdown_to_matrix_html(text: str) -> str:
+    """Convert the small Markdown subset we emit into Matrix-compatible HTML.
+
+    Matrix clients do NOT render Markdown in the plain ``body`` field; formatting
+    only shows when an HTML ``formatted_body`` is sent alongside it. We only use
+    ``**bold**``, bare URLs and newlines, so a minimal converter is sufficient and
+    avoids adding a Markdown dependency.
+    """
+    # Escape HTML special chars first so station/message data can't inject markup.
+    result = html.escape(text, quote=False)
+    result = _BOLD_RE.sub(r"<strong>\1</strong>", result)
+    result = _URL_RE.sub(r'<a href="\1">\1</a>', result)
+    result = result.replace("\n", "<br/>")
+    return result
+
+
+def strip_markdown(text: str) -> str:
+    """Produce a clean plain-text fallback for the Matrix ``body`` field."""
+    return _BOLD_RE.sub(r"\1", text)
+
+
+class AlertLevel(Enum):
+    INFO = "info"
+    WARNING = "warning"
+    CRITICAL = "critical"
+    EMERGENCY = "emergency"
+
+
+@dataclass
+class WaterAlert:
+    station_code: str
+    station_name: str
+    alert_type: str
+    level: AlertLevel
+    water_level: float
+    threshold: float
+    discharge: Optional[float] = None
+    timestamp: Optional[datetime.datetime] = None
+    message: Optional[str] = None
+
+
+class MatrixNotifier:
+    def __init__(self, homeserver: str, access_token: str, room_id: str):
+        self.homeserver = homeserver.rstrip("/")
+        self.access_token = access_token
+        self.room_id = room_id
+        self.session = requests.Session()
+
+    def send_message(self, message: str, msgtype: str = "m.text", markdown: bool = True) -> bool:
+        """Send a message to the Matrix room.
+
+        When ``markdown`` is True (default) the ``message`` is treated as Markdown:
+        a rendered HTML ``formatted_body`` is sent so clients show real formatting,
+        with a plain-text ``body`` fallback for clients that ignore HTML.
+        """
+        try:
+            # Add transaction ID to prevent duplicates
+            txn_id = datetime.datetime.now().strftime("%Y%m%d_%H%M%S_%f")
+            url = f"{self.homeserver}/_matrix/client/v3/rooms/{self.room_id}/send/m.room.message/{txn_id}"
+
+            headers = {
+                "Authorization": f"Bearer {self.access_token}",
+                "Content-Type": "application/json",
+            }
+
+            if markdown:
+                data = {
+                    "msgtype": msgtype,
+                    "body": strip_markdown(message),
+                    "format": "org.matrix.custom.html",
+                    "formatted_body": markdown_to_matrix_html(message),
+                }
+            else:
+                data = {"msgtype": msgtype, "body": message}
+
+            # Matrix API requires PUT when transaction ID is in the URL path
+            response = self.session.put(url, headers=headers, json=data, timeout=10)
+            response.raise_for_status()
+
+            logger.info(f"Matrix message sent successfully: {response.json().get('event_id')}")
+            return True
+
+        except Exception as e:
+            logger.error(f"Failed to send Matrix message: {e}")
+            return False
+
+    def send_alert(self, alert: WaterAlert) -> bool:
+        """Send formatted water alert to Matrix"""
+        emoji_map = {
+            AlertLevel.INFO: "ℹ️",
+            AlertLevel.WARNING: "⚠️",
+            AlertLevel.CRITICAL: "🚨",
+            AlertLevel.EMERGENCY: "🆘",
+        }
+
+        emoji = emoji_map.get(alert.level, "📊")
+
+        message = f"""{emoji} **WATER LEVEL ALERT**
+
+**Station:** {alert.station_code} ({alert.station_name})
+**Alert Type:** {alert.alert_type}
+**Severity:** {alert.level.value.upper()}
+
+**Current Level:** {alert.water_level:.2f}m
+**Threshold:** {alert.threshold:.2f}m
+**Difference:** {(alert.water_level - alert.threshold):+.2f}m
+"""
+
+        if alert.discharge:
+            message += f"**Discharge:** {alert.discharge:.1f} cms\n"
+
+        if alert.timestamp:
+            message += f"**Time:** {alert.timestamp.strftime('%Y-%m-%d %H:%M:%S')}\n"
+
+        if alert.message:
+            message += f"\n**Details:** {alert.message}\n"
+
+        # Add Grafana public dashboard link
+        grafana_url = "https://metrics.b4l.co.th/public-dashboards/655730aa044f44f49b355d01386018ca"
+        message += f"\n📈 **View Dashboard:** {grafana_url}"
+
+        return self.send_message(message)
+
+
+class WaterLevelAlertSystem:
+    # Stations upstream of Chiang Mai (and CNX itself) to monitor
+    UPSTREAM_STATIONS = {
+        "P.20",  # Ban Chiang Dao
+        "P.75",  # Ban Chai Lat
+        "P.92",  # Ban Muang Aut
+        "P.4A",  # Ban Mae Taeng
+        "P.67",  # Ban Tae
+        "P.21",  # Ban Rim Tai
+        "P.103",  # Ring Bridge 3
+        "P.1",  # Nawarat Bridge (Chiang Mai)
+    }
+
+    def __init__(self):
+        self.db_adapter = None
+        self.matrix_notifier = None
+        self.thresholds = self._load_thresholds()
+
+        # Matrix configuration from environment
+        matrix_homeserver = os.getenv("MATRIX_HOMESERVER", "https://matrix.org")
+        matrix_token = os.getenv("MATRIX_ACCESS_TOKEN")
+        matrix_room = os.getenv("MATRIX_ROOM_ID")
+
+        if matrix_token and matrix_room:
+            self.matrix_notifier = MatrixNotifier(matrix_homeserver, matrix_token, matrix_room)
+            logger.info("Matrix notifications enabled")
+        else:
+            logger.warning("Matrix configuration missing - notifications disabled")
+
+    def _load_thresholds(self) -> Dict[str, Dict[str, float]]:
+        """Load alert thresholds from config or database"""
+        # Default thresholds for Northern Thailand stations
+        return {
+            "P.1": {
+                # Zone-based thresholds for Nawarat Bridge (P.1)
+                "zone_1": 3.7,
+                "zone_2": 3.9,
+                "zone_3": 4.0,
+                "zone_4": 4.1,
+                "zone_5": 4.2,
+                "zone_6": 4.3,
+                "zone_7": 4.6,
+                "zone_8": 4.8,
+                "newedge": 4.8,  # Same as zone 8 or adjust as needed
+                # Keep legacy thresholds for compatibility
+                "warning": 3.7,
+                "critical": 4.3,
+                "emergency": 4.8,
+            },
+            "P.4A": {"warning": 4.5, "critical": 6.0, "emergency": 7.5},
+            "P.20": {"warning": 3.0, "critical": 4.5, "emergency": 6.0},
+            "P.21": {"warning": 4.0, "critical": 5.5, "emergency": 7.0},
+            "P.67": {"warning": 6.0, "critical": 8.0, "emergency": 10.0},
+            "P.75": {"warning": 5.5, "critical": 7.5, "emergency": 9.5},
+            "P.103": {"warning": 7.0, "critical": 9.0, "emergency": 11.0},
+            # Default for unknown stations
+            "default": {"warning": 4.0, "critical": 6.0, "emergency": 8.0},
+        }
+
+    def connect_database(self):
+        """Initialize database connection"""
+        try:
+            db_config = Config.get_database_config()
+            self.db_adapter = create_database_adapter(
+                db_config["type"], connection_string=db_config["connection_string"]
+            )
+
+            if self.db_adapter.connect():
+                logger.info("Database connection established for alerting")
+                return True
+            else:
+                logger.error("Failed to connect to database")
+                return False
+
+        except Exception as e:
+            logger.error(f"Database connection error: {e}")
+            return False
+
+    def check_water_levels(self) -> List[WaterAlert]:
+        """Check current water levels against thresholds"""
+        alerts = []
+
+        if not self.db_adapter:
+            logger.error("Database not connected")
+            return alerts
+
+        try:
+            # Get latest measurements
+            measurements = self.db_adapter.get_latest_measurements(limit=50)
+
+            for measurement in measurements:
+                station_code = measurement.get("station_code", "UNKNOWN")
+                water_level = measurement.get("water_level")
+
+                if not water_level:
+                    continue
+
+                # Only alert for upstream stations and Chiang Mai
+                if station_code not in self.UPSTREAM_STATIONS:
+                    continue
+
+                # Get thresholds for this station
+                station_thresholds = self.thresholds.get(station_code, self.thresholds["default"])
+
+                # Check each threshold level
+                alert_level = None
+                threshold_value = None
+                alert_type = None
+
+                # Special handling for P.1 with zone-based thresholds
+                if station_code == "P.1" and "zone_1" in station_thresholds:
+                    # Check all zones in reverse order (highest to lowest)
+                    zones = [
+                        ("zone_8", 4.8, AlertLevel.EMERGENCY, "Zone 8 - Emergency"),
+                        ("newedge", 4.8, AlertLevel.EMERGENCY, "NewEdge Alert Level"),
+                        ("zone_7", 4.6, AlertLevel.CRITICAL, "Zone 7 - Critical"),
+                        ("zone_6", 4.3, AlertLevel.CRITICAL, "Zone 6 - Critical"),
+                        ("zone_5", 4.2, AlertLevel.WARNING, "Zone 5 - Warning"),
+                        ("zone_4", 4.1, AlertLevel.WARNING, "Zone 4 - Warning"),
+                        ("zone_3", 4.0, AlertLevel.WARNING, "Zone 3 - Warning"),
+                        ("zone_2", 3.9, AlertLevel.INFO, "Zone 2 - Info"),
+                        ("zone_1", 3.7, AlertLevel.INFO, "Zone 1 - Info"),
+                    ]
+
+                    for (
+                        zone_name,
+                        zone_threshold,
+                        zone_alert_level,
+                        zone_description,
+                    ) in zones:
+                        if water_level >= zone_threshold:
+                            alert_level = zone_alert_level
+                            threshold_value = zone_threshold
+                            alert_type = zone_description
+                            break
+
+                else:
+                    # Standard threshold checking for other stations
+                    if water_level >= station_thresholds.get("emergency", float("inf")):
+                        alert_level = AlertLevel.EMERGENCY
+                        threshold_value = station_thresholds["emergency"]
+                        alert_type = "Emergency Water Level"
+                    elif water_level >= station_thresholds.get("critical", float("inf")):
+                        alert_level = AlertLevel.CRITICAL
+                        threshold_value = station_thresholds["critical"]
+                        alert_type = "Critical Water Level"
+                    elif water_level >= station_thresholds.get("warning", float("inf")):
+                        alert_level = AlertLevel.WARNING
+                        threshold_value = station_thresholds["warning"]
+                        alert_type = "High Water Level"
+
+                if alert_level:
+                    alert = WaterAlert(
+                        station_code=station_code,
+                        station_name=measurement.get("station_name_th", f"Station {station_code}"),
+                        alert_type=alert_type,
+                        level=alert_level,
+                        water_level=water_level,
+                        threshold=threshold_value,
+                        discharge=measurement.get("discharge"),
+                        timestamp=measurement.get("timestamp"),
+                    )
+                    alerts.append(alert)
+
+        except Exception as e:
+            logger.error(f"Error checking water levels: {e}")
+
+        return alerts
+
+    def check_data_freshness(self, max_age_hours: int = 12) -> List[WaterAlert]:
+        """Check if data is fresh enough"""
+        alerts = []
+
+        if not self.db_adapter:
+            return alerts
+
+        try:
+            measurements = self.db_adapter.get_latest_measurements(limit=20)
+            cutoff_time = datetime.datetime.now() - datetime.timedelta(hours=max_age_hours)
+
+            for measurement in measurements:
+                timestamp = measurement.get("timestamp")
+                if timestamp and timestamp < cutoff_time:
+                    station_code = measurement.get("station_code", "UNKNOWN")
+
+                    age_hours = (datetime.datetime.now() - timestamp).total_seconds() / 3600
+
+                    alert = WaterAlert(
+                        station_code=station_code,
+                        station_name=measurement.get("station_name_th", f"Station {station_code}"),
+                        alert_type="Stale Data",
+                        level=AlertLevel.WARNING,
+                        water_level=measurement.get("water_level", 0),
+                        threshold=max_age_hours,
+                        timestamp=timestamp,
+                        message=f"No fresh data for {age_hours:.1f} hours",
+                    )
+                    alerts.append(alert)
+
+        except Exception as e:
+            logger.error(f"Error checking data freshness: {e}")
+
+        return alerts
+
+    def check_rate_of_change(self, lookback_hours: int = 3) -> List[WaterAlert]:
+        """Check for rapid water level changes over recent hours"""
+        alerts = []
+
+        if not self.db_adapter:
+            return alerts
+
+        try:
+            # Define rate-of-change thresholds (meters per hour)
+            rate_thresholds = {
+                "P.1": {
+                    "warning": 0.15,  # 15cm/hour - moderate rise
+                    "critical": 0.25,  # 25cm/hour - rapid rise
+                    "emergency": 0.40,  # 40cm/hour - very rapid rise
+                },
+                "default": {"warning": 0.20, "critical": 0.35, "emergency": 0.50},
+            }
+
+            # Get recent measurements for each station
+            cutoff_time = datetime.datetime.now() - datetime.timedelta(hours=lookback_hours)
+
+            # Get unique stations from latest data
+            latest = self.db_adapter.get_latest_measurements(limit=20)
+            station_codes = set(m.get("station_code") for m in latest if m.get("station_code"))
+
+            for station_code in station_codes:
+                try:
+                    # Only alert for upstream stations and Chiang Mai
+                    if station_code not in self.UPSTREAM_STATIONS:
+                        continue
+
+                    # Get measurements for this station in the time window
+                    current_time = datetime.datetime.now()
+                    measurements = self.db_adapter.get_measurements_by_timerange(
+                        start_time=cutoff_time,
+                        end_time=current_time,
+                        station_codes=[station_code],
+                    )
+
+                    if len(measurements) < 2:
+                        continue  # Need at least 2 points to calculate rate
+
+                    # Sort by timestamp
+                    measurements = sorted(measurements, key=lambda m: m.get("timestamp"))
+
+                    # Get oldest and newest measurements
+                    oldest = measurements[0]
+                    newest = measurements[-1]
+
+                    oldest_time = oldest.get("timestamp")
+                    oldest_level = oldest.get("water_level")
+                    newest_time = newest.get("timestamp")
+                    newest_level = newest.get("water_level")
+
+                    # Convert timestamp strings to datetime if needed
+                    if isinstance(oldest_time, str):
+                        oldest_time = datetime.datetime.fromisoformat(oldest_time)
+                    if isinstance(newest_time, str):
+                        newest_time = datetime.datetime.fromisoformat(newest_time)
+
+                    # Calculate rate of change
+                    time_diff_hours = (newest_time - oldest_time).total_seconds() / 3600
+                    if time_diff_hours == 0:
+                        continue
+
+                    level_change = newest_level - oldest_level
+                    rate_per_hour = level_change / time_diff_hours
+
+                    # Only alert on rising water (positive rate)
+                    if rate_per_hour <= 0:
+                        continue
+
+                    # Get station info from latest data
+                    station_info = next((m for m in latest if m.get("station_code") == station_code), {})
+                    station_name = station_info.get("station_name_th", station_code)
+
+                    # Get thresholds for this station
+                    station_rate_threshold = rate_thresholds.get(station_code, rate_thresholds["default"])
+
+                    alert_level = None
+                    threshold_value = None
+                    alert_type = None
+
+                    if rate_per_hour >= station_rate_threshold["emergency"]:
+                        alert_level = AlertLevel.EMERGENCY
+                        threshold_value = station_rate_threshold["emergency"]
+                        alert_type = "Very Rapid Water Level Rise"
+                    elif rate_per_hour >= station_rate_threshold["critical"]:
+                        alert_level = AlertLevel.CRITICAL
+                        threshold_value = station_rate_threshold["critical"]
+                        alert_type = "Rapid Water Level Rise"
+                    elif rate_per_hour >= station_rate_threshold["warning"]:
+                        alert_level = AlertLevel.WARNING
+                        threshold_value = station_rate_threshold["warning"]
+                        alert_type = "Moderate Water Level Rise"
+
+                    if alert_level:
+                        message = (
+                            f"Rising at {rate_per_hour:.2f}m/h over last {time_diff_hours:.1f}h "
+                            f"(change: {level_change:+.2f}m)"
+                        )
+
+                        alert = WaterAlert(
+                            station_code=station_code,
+                            station_name=station_name or f"Station {station_code}",
+                            alert_type=alert_type,
+                            level=alert_level,
+                            water_level=newest_level,
+                            threshold=threshold_value,
+                            timestamp=newest_time,
+                            message=message,
+                        )
+                        alerts.append(alert)
+
+                except Exception as station_error:
+                    logger.debug(f"Error checking rate of change for station {station_code}: {station_error}")
+                    continue
+
+        except Exception as e:
+            logger.error(f"Error checking rate of change: {e}")
+
+        return alerts
+
+    def send_alerts(self, alerts: List[WaterAlert]) -> int:
+        """Send alerts via configured channels"""
+        sent_count = 0
+
+        if not alerts:
+            return sent_count
+
+        if self.matrix_notifier:
+            for alert in alerts:
+                if self.matrix_notifier.send_alert(alert):
+                    sent_count += 1
+
+        # Could add other notification channels here:
+        # - Email
+        # - Discord
+        # - Telegram
+        # - SMS
+
+        return sent_count
+
+    def run_alert_check(self) -> Dict[str, int]:
+        """Run complete alert check cycle"""
+        if not self.connect_database():
+            return {"error": 1}
+
+        # Check water levels
+        water_alerts = self.check_water_levels()
+
+        # Check data freshness
+        data_alerts = self.check_data_freshness()
+
+        # Check rate of change (rapid rises)
+        rate_alerts = self.check_rate_of_change()
+
+        # Combine alerts
+        all_alerts = water_alerts + rate_alerts
+
+        # Send alerts
+        sent_count = self.send_alerts(all_alerts)
+
+        logger.info(f"Alert check complete: {len(all_alerts)} alerts, {sent_count} sent")
+
+        return {
+            "water_alerts": len(water_alerts),
+            "data_alerts": len(data_alerts),
+            "rate_alerts": len(rate_alerts),
+            "total_alerts": len(all_alerts),
+            "sent": sent_count,
+        }
+
+
+def main():
+    """Standalone alerting check"""
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Water Level Alert System")
+    parser.add_argument("--check", action="store_true", help="Run alert check")
+    parser.add_argument("--test", action="store_true", help="Send test message")
+    args = parser.parse_args()
+
+    alerting = WaterLevelAlertSystem()
+
+    if args.test:
+        if alerting.matrix_notifier:
+            test_message = (
+                "🧪 **Test Alert**\n\nThis is a test message from the Water Level Alert System.\n\n"
+                "If you received this, Matrix notifications are working correctly!"
+            )
+            success = alerting.matrix_notifier.send_message(test_message)
+            print(f"Test message sent: {success}")
+        else:
+            print("Matrix notifier not configured")
+
+    elif args.check:
+        results = alerting.run_alert_check()
+        print(f"Alert check results: {results}")
+
+    else:
+        print("Use --check or --test")
+
+
+if __name__ == "__main__":
+    main()

src/config.py

@@ -1,9 +1,18 @@
 import os
-from typing import Dict, Any, Optional
+from typing import Any, Dict
+
+# Load environment variables from .env file
+try:
+    from dotenv import load_dotenv
+
+    load_dotenv()
+except ImportError:
+    # python-dotenv not installed, continue without it
+    pass
 
 try:
     from .exceptions import ConfigurationError
-    from .models import DatabaseType, DatabaseConfig
+    from .models import DatabaseType
 except ImportError:
     # Handle case when running as standalone script
     class ConfigurationError(Exception):
@@ -18,55 +27,84 @@ except ImportError:
         INFLUXDB = "influxdb"
         VICTORIAMETRICS = "victoriametrics"
 
+
 class Config:
     """Configuration class for the Water Level Monitor"""
 
     # Database settings
-    DATABASE_PATH = os.getenv('WATER_DB_PATH', 'water_levels.db')
+    DATABASE_PATH = os.getenv("WATER_DB_PATH", "water_levels.db")
 
     # Website settings
     TARGET_URL = "https://hyd-app-db.rid.go.th/hydro1h.html"
     API_URL = "https://hyd-app-db.rid.go.th/webservice/getGroupHourlyWaterLevelReportAllHL.ashx"
-    REQUEST_TIMEOUT = int(os.getenv('REQUEST_TIMEOUT', '30'))
-    USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
+    THAIWATER_API_KEY = os.getenv("THAIWATER_API_KEY")
+    REQUEST_TIMEOUT = int(os.getenv("REQUEST_TIMEOUT", "30"))
+    USER_AGENT = (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+        "(KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
+    )
 
     # Database configuration
-    DB_TYPE = os.getenv('DB_TYPE', 'sqlite').lower()
+    # When DB_TYPE is not set explicitly, a configured Postgres connection wins over the sqlite default
+    DB_TYPE = os.getenv(
+        "DB_TYPE",
+        "postgresql" if os.getenv("POSTGRES_CONNECTION_STRING") else "sqlite",
+    ).lower()
 
     # VictoriaMetrics settings
-    VM_HOST = os.getenv('VM_HOST', 'vm.newedge.house')
-    VM_PORT = int(os.getenv('VM_PORT', '443'))
+    # Default to localhost; set VM_HOST in the environment for real deployments
+    # (avoids committing infrastructure hostnames to the repo).
+    VM_HOST = os.getenv("VM_HOST", "localhost")
+    VM_PORT = int(os.getenv("VM_PORT", "443"))
 
     # Support for HTTPS URLs (e.g., behind reverse proxy)
-    VM_URL = os.getenv('VM_URL')  # Full URL override (e.g., https://vm.example.com)
+    VM_URL = os.getenv("VM_URL")  # Full URL override (e.g., https://vm.example.com)
 
     # InfluxDB settings
-    INFLUX_HOST = os.getenv('INFLUX_HOST', 'localhost')
-    INFLUX_PORT = int(os.getenv('INFLUX_PORT', '8086'))
-    INFLUX_DATABASE = os.getenv('INFLUX_DATABASE', 'water_monitoring')
-    INFLUX_USERNAME = os.getenv('INFLUX_USERNAME')
-    INFLUX_PASSWORD = os.getenv('INFLUX_PASSWORD')
+    INFLUX_HOST = os.getenv("INFLUX_HOST", "localhost")
+    INFLUX_PORT = int(os.getenv("INFLUX_PORT", "8086"))
+    INFLUX_DATABASE = os.getenv("INFLUX_DATABASE", "water_monitoring")
+    INFLUX_USERNAME = os.getenv("INFLUX_USERNAME")
+    INFLUX_PASSWORD = os.getenv("INFLUX_PASSWORD")
 
     # PostgreSQL settings
-    POSTGRES_CONNECTION_STRING = os.getenv('POSTGRES_CONNECTION_STRING')
+    POSTGRES_CONNECTION_STRING = os.getenv("POSTGRES_CONNECTION_STRING")
+    POSTGRES_HOST = os.getenv("POSTGRES_HOST", "localhost")
+    POSTGRES_PORT = int(os.getenv("POSTGRES_PORT", "5432"))
+    POSTGRES_DB = os.getenv("POSTGRES_DB", "water_monitoring")
+    POSTGRES_USER = os.getenv("POSTGRES_USER", "postgres")
+    POSTGRES_PASSWORD = os.getenv("POSTGRES_PASSWORD")
 
     # MySQL settings
-    MYSQL_CONNECTION_STRING = os.getenv('MYSQL_CONNECTION_STRING')
+    MYSQL_CONNECTION_STRING = os.getenv("MYSQL_CONNECTION_STRING")
 
     # Scheduler settings
-    SCRAPING_INTERVAL_HOURS = int(os.getenv('SCRAPING_INTERVAL_HOURS', '1'))
+    SCRAPING_INTERVAL_HOURS = int(os.getenv("SCRAPING_INTERVAL_HOURS", "1"))
 
     # Logging settings
-    LOG_LEVEL = os.getenv('LOG_LEVEL', 'INFO')
-    LOG_FILE = os.getenv('LOG_FILE', 'water_monitor.log')
-    LOG_FORMAT = '%(asctime)s - %(levelname)s - %(message)s'
+    LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO")
+    LOG_FILE = os.getenv("LOG_FILE", "water_monitor.log")
+    LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s"
 
     # Data retention
-    DATA_RETENTION_DAYS = int(os.getenv('DATA_RETENTION_DAYS', '365'))
+    DATA_RETENTION_DAYS = int(os.getenv("DATA_RETENTION_DAYS", "365"))
 
     # Retry settings
-    MAX_RETRIES = int(os.getenv('MAX_RETRIES', '3'))
-    RETRY_DELAY_SECONDS = int(os.getenv('RETRY_DELAY_SECONDS', '60'))
+    MAX_RETRIES = int(os.getenv("MAX_RETRIES", "3"))
+    RETRY_DELAY_SECONDS = int(os.getenv("RETRY_DELAY_SECONDS", "60"))
+
+    # Station configuration
+    # Runtime-writable JSON file that persists the station mapping across restarts
+    # (station CRUD via the API writes here). If it does not exist, the bundled
+    # defaults in src/data/stations.json are used to seed it.
+    STATION_CONFIG_PATH = os.getenv("STATION_CONFIG_PATH", "stations.json")
+
+    # Web API / CORS settings
+    # Comma-separated list of allowed origins. Defaults to none (same-origin only);
+    # set CORS_ALLOW_ORIGINS to a specific list of front-end origins in production.
+    # Credentials are only enabled when explicit (non-wildcard) origins are set,
+    # because "*" + credentials is rejected by browsers and unsafe.
+    CORS_ALLOW_ORIGINS = [origin.strip() for origin in os.getenv("CORS_ALLOW_ORIGINS", "").split(",") if origin.strip()]
 
     @classmethod
     def validate_config(cls) -> bool:
@@ -80,23 +118,34 @@ class Config:
             errors.append(f"Invalid DB_TYPE: {cls.DB_TYPE}")
 
         # Validate database-specific settings
-        if cls.DB_TYPE == 'victoriametrics':
+        if cls.DB_TYPE == "victoriametrics":
             if not cls.VM_HOST:
                 errors.append("VM_HOST is required for VictoriaMetrics")
             if not isinstance(cls.VM_PORT, int) or cls.VM_PORT <= 0:
                 errors.append("VM_PORT must be a positive integer")
 
-        elif cls.DB_TYPE == 'influxdb':
+        elif cls.DB_TYPE == "influxdb":
             if not cls.INFLUX_HOST:
                 errors.append("INFLUX_HOST is required for InfluxDB")
             if not cls.INFLUX_DATABASE:
                 errors.append("INFLUX_DATABASE is required for InfluxDB")
 
-        elif cls.DB_TYPE in ['postgresql', 'mysql']:
-            connection_string = (cls.POSTGRES_CONNECTION_STRING if cls.DB_TYPE == 'postgresql' 
-                               else cls.MYSQL_CONNECTION_STRING)
-            if not connection_string:
-                errors.append(f"Connection string is required for {cls.DB_TYPE.upper()}")
+        elif cls.DB_TYPE in ["postgresql", "mysql"]:
+            if cls.DB_TYPE == "postgresql":
+                # Check if either connection string or individual components are provided
+                if not cls.POSTGRES_CONNECTION_STRING:
+                    # If no connection string, check individual components
+                    if not cls.POSTGRES_HOST:
+                        errors.append("POSTGRES_HOST is required for PostgreSQL")
+                    if not cls.POSTGRES_USER:
+                        errors.append("POSTGRES_USER is required for PostgreSQL")
+                    if not cls.POSTGRES_PASSWORD:
+                        errors.append("POSTGRES_PASSWORD is required for PostgreSQL")
+                    if not cls.POSTGRES_DB:
+                        errors.append("POSTGRES_DB is required for PostgreSQL")
+            else:  # mysql
+                if not cls.MYSQL_CONNECTION_STRING:
+                    errors.append("MYSQL_CONNECTION_STRING is required for MySQL")
 
         # Validate numeric settings
         if cls.SCRAPING_INTERVAL_HOURS <= 0:
@@ -113,59 +162,66 @@ class Config:
     @classmethod
     def get_database_config(cls) -> Dict[str, Any]:
         """Returns database configuration based on DB_TYPE"""
-        if cls.DB_TYPE == 'victoriametrics':
+        if cls.DB_TYPE == "victoriametrics":
+            return {"type": "victoriametrics", "host": cls.VM_HOST, "port": cls.VM_PORT}
+        elif cls.DB_TYPE == "influxdb":
             return {
-                'type': 'victoriametrics',
-                'host': cls.VM_HOST,
-                'port': cls.VM_PORT
+                "type": "influxdb",
+                "host": cls.INFLUX_HOST,
+                "port": cls.INFLUX_PORT,
+                "database": cls.INFLUX_DATABASE,
+                "username": cls.INFLUX_USERNAME,
+                "password": cls.INFLUX_PASSWORD,
             }
-        elif cls.DB_TYPE == 'influxdb':
+        elif cls.DB_TYPE == "postgresql":
+            # Use individual components if POSTGRES_CONNECTION_STRING is not provided
+            if cls.POSTGRES_CONNECTION_STRING:
                 return {
-                'type': 'influxdb',
-                'host': cls.INFLUX_HOST,
-                'port': cls.INFLUX_PORT,
-                'database': cls.INFLUX_DATABASE,
-                'username': cls.INFLUX_USERNAME,
-                'password': cls.INFLUX_PASSWORD
-            }
-        elif cls.DB_TYPE == 'postgresql':
-            return {
-                'type': 'postgresql',
-                'connection_string': cls.POSTGRES_CONNECTION_STRING or 
-                    'postgresql://postgres:password@localhost:5432/water_monitoring'
-            }
-        elif cls.DB_TYPE == 'mysql':
-            return {
-                'type': 'mysql',
-                'connection_string': cls.MYSQL_CONNECTION_STRING or 
-                    'mysql://root:password@localhost:3306/water_monitoring'
+                    "type": "postgresql",
+                    "connection_string": cls.POSTGRES_CONNECTION_STRING,
                 }
+            else:
+                # Build connection string from components (automatically URL-encodes password)
+                import urllib.parse
+
+                if not cls.POSTGRES_PASSWORD:
+                    raise ConfigurationError("POSTGRES_PASSWORD is required for PostgreSQL (no default is provided)")
+                password = urllib.parse.quote(cls.POSTGRES_PASSWORD, safe="")
+                connection_string = (
+                    f"postgresql://{cls.POSTGRES_USER}:{password}"
+                    f"@{cls.POSTGRES_HOST}:{cls.POSTGRES_PORT}/{cls.POSTGRES_DB}"
+                )
+                return {"type": "postgresql", "connection_string": connection_string}
+        elif cls.DB_TYPE == "mysql":
+            if not cls.MYSQL_CONNECTION_STRING:
+                raise ConfigurationError("MYSQL_CONNECTION_STRING is required for MySQL (no default is provided)")
+            return {"type": "mysql", "connection_string": cls.MYSQL_CONNECTION_STRING}
         else:  # sqlite
             return {
-                'type': 'sqlite',
-                'connection_string': f'sqlite:///{cls.DATABASE_PATH}'
+                "type": "sqlite",
+                "connection_string": f"sqlite:///{cls.DATABASE_PATH}",
             }
 
     @classmethod
     def get_all_settings(cls) -> Dict[str, Any]:
         """Returns all configuration settings"""
         return {
-            'DB_TYPE': cls.DB_TYPE,
-            'DATABASE_PATH': cls.DATABASE_PATH,
-            'TARGET_URL': cls.TARGET_URL,
-            'API_URL': cls.API_URL,
-            'REQUEST_TIMEOUT': cls.REQUEST_TIMEOUT,
-            'SCRAPING_INTERVAL_HOURS': cls.SCRAPING_INTERVAL_HOURS,
-            'LOG_LEVEL': cls.LOG_LEVEL,
-            'LOG_FILE': cls.LOG_FILE,
-            'DATA_RETENTION_DAYS': cls.DATA_RETENTION_DAYS,
-            'MAX_RETRIES': cls.MAX_RETRIES,
-            'RETRY_DELAY_SECONDS': cls.RETRY_DELAY_SECONDS,
-            'VM_HOST': cls.VM_HOST,
-            'VM_PORT': cls.VM_PORT,
-            'INFLUX_HOST': cls.INFLUX_HOST,
-            'INFLUX_PORT': cls.INFLUX_PORT,
-            'INFLUX_DATABASE': cls.INFLUX_DATABASE
+            "DB_TYPE": cls.DB_TYPE,
+            "DATABASE_PATH": cls.DATABASE_PATH,
+            "TARGET_URL": cls.TARGET_URL,
+            "API_URL": cls.API_URL,
+            "REQUEST_TIMEOUT": cls.REQUEST_TIMEOUT,
+            "SCRAPING_INTERVAL_HOURS": cls.SCRAPING_INTERVAL_HOURS,
+            "LOG_LEVEL": cls.LOG_LEVEL,
+            "LOG_FILE": cls.LOG_FILE,
+            "DATA_RETENTION_DAYS": cls.DATA_RETENTION_DAYS,
+            "MAX_RETRIES": cls.MAX_RETRIES,
+            "RETRY_DELAY_SECONDS": cls.RETRY_DELAY_SECONDS,
+            "VM_HOST": cls.VM_HOST,
+            "VM_PORT": cls.VM_PORT,
+            "INFLUX_HOST": cls.INFLUX_HOST,
+            "INFLUX_PORT": cls.INFLUX_PORT,
+            "INFLUX_DATABASE": cls.INFLUX_DATABASE,
         }
 
     @classmethod
@@ -174,18 +230,19 @@ class Config:
         print("=== Water Level Monitor Configuration ===")
         for key, value in cls.get_all_settings().items():
             # Hide sensitive information
-            if 'PASSWORD' in key and value:
-                value = '*' * len(str(value))
+            if "PASSWORD" in key and value:
+                value = "*" * len(str(value))
             print(f"{key}: {value}")
         print("=" * 45)
 
         print("\nDatabase Configuration:")
         db_config = cls.get_database_config()
         for key, value in db_config.items():
-            if 'password' in key and value:
-                value = '*' * len(str(value))
+            if "password" in key and value:
+                value = "*" * len(str(value))
             print(f"  {key}: {value}")
         print("=" * 45)
 
+
 if __name__ == "__main__":
     Config.print_settings()

src/data/stations.json

@@ -0,0 +1,130 @@
+{
+  "1": {
+    "code": "P.20",
+    "thai_name": "บ้านเชียงดาว",
+    "english_name": "Ban Chiang Dao",
+    "latitude": 19.36731448032191,
+    "longitude": 98.9688487015384,
+    "geohash": null
+  },
+  "2": {
+    "code": "P.75",
+    "thai_name": "บ้านช่อแล",
+    "english_name": "Ban Chai Lat",
+    "latitude": 19.145972935976225,
+    "longitude": 99.00735727149247,
+    "geohash": null
+  },
+  "3": {
+    "code": "P.92",
+    "thai_name": "บ้านเมืองกึ๊ด",
+    "english_name": "Ban Muang Aut",
+    "latitude": 19.220518985435646,
+    "longitude": 98.84733127007874,
+    "geohash": null
+  },
+  "4": {
+    "code": "P.4A",
+    "thai_name": "บ้านแม่แตง",
+    "english_name": "Ban Mae Taeng",
+    "latitude": 19.1222679952378,
+    "longitude": 98.94437462084075,
+    "geohash": null
+  },
+  "5": {
+    "code": "P.67",
+    "thai_name": "บ้านแม่แต",
+    "english_name": "Ban Tae",
+    "latitude": 19.009762080002453,
+    "longitude": 98.95978297135508,
+    "geohash": null
+  },
+  "6": {
+    "code": "P.21",
+    "thai_name": "บ้านริมใต้",
+    "english_name": "Ban Rim Tai",
+    "latitude": 18.917459157963293,
+    "longitude": 98.97018092996231,
+    "geohash": null
+  },
+  "7": {
+    "code": "P.103",
+    "thai_name": "สะพานวงแหวนรอบ 3",
+    "english_name": "Ring Bridge 3",
+    "latitude": 18.86664807441675,
+    "longitude": 98.9781107622432,
+    "geohash": null
+  },
+  "8": {
+    "code": "P.1",
+    "thai_name": "สะพานนวรัฐ",
+    "english_name": "Nawarat Bridge",
+    "latitude": 18.7875,
+    "longitude": 99.0045,
+    "geohash": "w5q6uuhvfcfp25"
+  },
+  "9": {
+    "code": "P.82",
+    "thai_name": "บ้านสบวิน",
+    "english_name": "Ban Sob win",
+    "latitude": 18.6519444,
+    "longitude": 98.69,
+    "geohash": null
+  },
+  "10": {
+    "code": "P.84",
+    "thai_name": "บ้านพันตน",
+    "english_name": "Ban Panton",
+    "latitude": 18.591315274591334,
+    "longitude": 98.79657058508496,
+    "geohash": null
+  },
+  "11": {
+    "code": "P.81",
+    "thai_name": "บ้านโป่ง",
+    "english_name": "Ban Pong",
+    "latitude": 18.693611,
+    "longitude": 99.081944,
+    "geohash": null
+  },
+  "12": {
+    "code": "P.5",
+    "thai_name": "สะพานท่านาง",
+    "english_name": "Tha Nang Bridge",
+    "latitude": 18.580269437546555,
+    "longitude": 99.01021397084362,
+    "geohash": null
+  },
+  "13": {
+    "code": "P.77",
+    "thai_name": "บ้านสบแม่สะป๊วด",
+    "english_name": "Baan Sop Mae Sapuord",
+    "latitude": 18.433347475179602,
+    "longitude": 99.08510036666527,
+    "geohash": null
+  },
+  "14": {
+    "code": "P.87",
+    "thai_name": "บ้านป่าซาง",
+    "english_name": "Ban Pa Sang",
+    "latitude": 18.519121825282486,
+    "longitude": 98.94224374138238,
+    "geohash": null
+  },
+  "15": {
+    "code": "P.76",
+    "thai_name": "บ้านแม่อีไฮ",
+    "english_name": "Banb Mae I Hai",
+    "latitude": 18.141465831254404,
+    "longitude": 98.89642508267181,
+    "geohash": null
+  },
+  "16": {
+    "code": "P.85",
+    "thai_name": "บ้านหล่ายแก้ว",
+    "english_name": "Baan Lai Kaew",
+    "latitude": 18.17856361002219,
+    "longitude": 98.63023114782287,
+    "geohash": null
+  }
+}

src/database_adapters.py

@@ -5,8 +5,9 @@ Database adapters for different storage backends
 
 import datetime
 import logging
-from typing import List, Dict, Optional, Any
 from abc import ABC, abstractmethod
+from typing import Dict, List, Optional
+
 
 # Base adapter interface
 class DatabaseAdapter(ABC):
@@ -23,15 +24,29 @@ class DatabaseAdapter(ABC):
         pass
 
     @abstractmethod
-    def get_measurements_by_timerange(self, start_time: datetime.datetime, 
+    def get_measurements_by_timerange(
+        self,
+        start_time: datetime.datetime,
         end_time: datetime.datetime,
-                                    station_codes: Optional[List[str]] = None) -> List[Dict]:
+        station_codes: Optional[List[str]] = None,
+    ) -> List[Dict]:
         pass
 
+    @abstractmethod
+    def get_measurements_for_date(self, target_date: datetime.datetime) -> List[Dict]:
+        pass
+
+
 # InfluxDB Adapter
 class InfluxDBAdapter(DatabaseAdapter):
-    def __init__(self, host: str = "localhost", port: int = 8086, 
-                 database: str = "water_monitoring", username: str = None, password: str = None):
+    def __init__(
+        self,
+        host: str = "localhost",
+        port: int = 8086,
+        database: str = "water_monitoring",
+        username: str = None,
+        password: str = None,
+    ):
         self.host = host
         self.port = port
         self.database = database
@@ -42,29 +57,30 @@ class InfluxDBAdapter(DatabaseAdapter):
     def connect(self):
         try:
             from influxdb import InfluxDBClient
+
             self.client = InfluxDBClient(
                 host=self.host,
                 port=self.port,
                 username=self.username,
                 password=self.password,
-                database=self.database
+                database=self.database,
             )
 
             # Create database if it doesn't exist
             databases = self.client.get_list_database()
-            if not any(db['name'] == self.database for db in databases):
+            if not any(db["name"] == self.database for db in databases):
                 self.client.create_database(self.database)
                 logging.info(f"Created InfluxDB database: {self.database}")
 
             # Create retention policy (keep data for 2 years, downsample after 30 days)
             retention_policies = self.client.get_list_retention_policies(self.database)
-            if not any(rp['name'] == 'water_data_policy' for rp in retention_policies):
+            if not any(rp["name"] == "water_data_policy" for rp in retention_policies):
                 self.client.create_retention_policy(
-                    'water_data_policy', 
-                    '730d',  # 2 years
-                    '1',     # replication factor
+                    "water_data_policy",
+                    "730d",  # 2 years
+                    "1",  # replication factor
                     database=self.database,
-                    default=True
+                    default=True,
                 )
 
             logging.info("Connected to InfluxDB successfully")
@@ -88,16 +104,20 @@ class InfluxDBAdapter(DatabaseAdapter):
                 point = {
                     "measurement": "water_data",
                     "tags": {
-                        "station_code": measurement['station_code'],
-                        "station_name_en": measurement['station_name_en'],
-                        "station_name_th": measurement['station_name_th']
+                        "station_code": measurement["station_code"],
+                        "station_name_en": measurement["station_name_en"],
+                        "station_name_th": measurement["station_name_th"],
                     },
-                    "time": measurement['timestamp'].isoformat(),
+                    "time": measurement["timestamp"].isoformat(),
                     "fields": {
-                        "water_level": float(measurement['water_level']),
-                        "discharge": float(measurement['discharge']),
-                        "discharge_percent": float(measurement['discharge_percent']) if measurement['discharge_percent'] else None
-                    }
+                        "water_level": float(measurement["water_level"]),
+                        "discharge": float(measurement["discharge"])
+                        if measurement.get("discharge") is not None
+                        else None,
+                        "discharge_percent": float(measurement["discharge_percent"])
+                        if measurement.get("discharge_percent")
+                        else None,
+                    },
                 }
                 points.append(point)
 
@@ -115,6 +135,8 @@ class InfluxDBAdapter(DatabaseAdapter):
             return []
 
         try:
+            # Cast limit to int so it can never carry an injection payload.
+            limit = int(limit)
             query = f"""
             SELECT last("water_level") as water_level,
                    last("discharge") as discharge,
@@ -128,15 +150,17 @@ class InfluxDBAdapter(DatabaseAdapter):
             measurements = []
 
             for point in result.get_points():
-                measurements.append({
-                    'timestamp': point['time'],
-                    'station_code': point.get('station_code'),
-                    'station_name_en': point.get('station_name_en'),
-                    'station_name_th': point.get('station_name_th'),
-                    'water_level': point.get('water_level'),
-                    'discharge': point.get('discharge'),
-                    'discharge_percent': point.get('discharge_percent')
-                })
+                measurements.append(
+                    {
+                        "timestamp": point["time"],
+                        "station_code": point.get("station_code"),
+                        "station_name_en": point.get("station_name_en"),
+                        "station_name_th": point.get("station_name_th"),
+                        "water_level": point.get("water_level"),
+                        "discharge": point.get("discharge"),
+                        "discharge_percent": point.get("discharge_percent"),
+                    }
+                )
 
             return measurements
 
@@ -144,17 +168,27 @@ class InfluxDBAdapter(DatabaseAdapter):
             logging.error(f"Error querying InfluxDB: {e}")
             return []
 
-    def get_measurements_by_timerange(self, start_time: datetime.datetime, 
+    def get_measurements_by_timerange(
+        self,
+        start_time: datetime.datetime,
         end_time: datetime.datetime,
-                                    station_codes: Optional[List[str]] = None) -> List[Dict]:
+        station_codes: Optional[List[str]] = None,
+    ) -> List[Dict]:
         if not self.client:
             return []
 
         try:
+            # start_time/end_time are datetime objects (fixed isoformat, injection-safe).
+            # station_codes are untrusted strings -> bind them as parameters.
+            bind_params = {}
             where_clause = f"time >= '{start_time.isoformat()}' AND time <= '{end_time.isoformat()}'"
             if station_codes:
-                station_filter = "'" + "','".join(station_codes) + "'"
-                where_clause += f" AND station_code IN ({station_filter})"
+                placeholders = []
+                for i, code in enumerate(station_codes):
+                    key = f"sc{i}"
+                    bind_params[key] = code
+                    placeholders.append(f"station_code = ${key}")
+                where_clause += " AND (" + " OR ".join(placeholders) + ")"
 
             query = f"""
             SELECT "water_level", "discharge", "discharge_percent", "station_code", "station_name_en", "station_name_th"
@@ -163,19 +197,21 @@ class InfluxDBAdapter(DatabaseAdapter):
             ORDER BY time DESC
             """
 
-            result = self.client.query(query)
+            result = self.client.query(query, bind_params=bind_params)
             measurements = []
 
             for point in result.get_points():
-                measurements.append({
-                    'timestamp': point['time'],
-                    'station_code': point.get('station_code'),
-                    'station_name_en': point.get('station_name_en'),
-                    'station_name_th': point.get('station_name_th'),
-                    'water_level': point.get('water_level'),
-                    'discharge': point.get('discharge'),
-                    'discharge_percent': point.get('discharge_percent')
-                })
+                measurements.append(
+                    {
+                        "timestamp": point["time"],
+                        "station_code": point.get("station_code"),
+                        "station_name_en": point.get("station_name_en"),
+                        "station_name_th": point.get("station_name_th"),
+                        "water_level": point.get("water_level"),
+                        "discharge": point.get("discharge"),
+                        "discharge_percent": point.get("discharge_percent"),
+                    }
+                )
 
             return measurements
 
@@ -183,6 +219,7 @@ class InfluxDBAdapter(DatabaseAdapter):
             logging.error(f"Error querying InfluxDB: {e}")
             return []
 
+
 # MySQL/PostgreSQL Adapter
 class SQLAdapter(DatabaseAdapter):
     def __init__(self, connection_string: str, db_type: str = "mysql"):
@@ -199,8 +236,7 @@ class SQLAdapter(DatabaseAdapter):
 
     def connect(self):
         try:
-            from sqlalchemy import create_engine, text
-            from sqlalchemy.orm import sessionmaker
+            from sqlalchemy import create_engine
 
             self.engine = create_engine(self.connection_string, pool_pre_ping=True)
 
@@ -254,7 +290,7 @@ class SQLAdapter(DatabaseAdapter):
             # Create indexes separately for SQLite
             index_sql = [
                 "CREATE INDEX IF NOT EXISTS idx_timestamp ON water_measurements(timestamp)",
-                "CREATE INDEX IF NOT EXISTS idx_station_timestamp ON water_measurements(station_id, timestamp)"
+                "CREATE INDEX IF NOT EXISTS idx_station_timestamp ON water_measurements(station_id, timestamp)",
             ]
 
         elif self.db_type == "postgresql":
@@ -289,7 +325,7 @@ class SQLAdapter(DatabaseAdapter):
 
             index_sql = [
                 "CREATE INDEX IF NOT EXISTS idx_timestamp ON water_measurements(timestamp)",
-                "CREATE INDEX IF NOT EXISTS idx_station_timestamp ON water_measurements(station_id, timestamp DESC)"
+                "CREATE INDEX IF NOT EXISTS idx_station_timestamp ON water_measurements(station_id, timestamp DESC)",
             ]
 
         else:  # MySQL
@@ -347,13 +383,21 @@ class SQLAdapter(DatabaseAdapter):
                 for measurement in measurements:
                     if self.db_type == "sqlite":
                         station_sql = """
-                        INSERT OR REPLACE INTO stations (id, station_code, thai_name, english_name, latitude, longitude, geohash, updated_at)
-                        VALUES (:station_id, :station_code, :thai_name, :english_name, :latitude, :longitude, :geohash, CURRENT_TIMESTAMP)
+                        INSERT OR REPLACE INTO stations
+                            (id, station_code, thai_name, english_name,
+                             latitude, longitude, geohash, updated_at)
+                        VALUES
+                            (:station_id, :station_code, :thai_name, :english_name,
+                             :latitude, :longitude, :geohash, CURRENT_TIMESTAMP)
                         """
                     elif self.db_type == "postgresql":
                         station_sql = """
-                        INSERT INTO stations (id, station_code, thai_name, english_name, latitude, longitude, geohash, updated_at)
-                        VALUES (:station_id, :station_code, :thai_name, :english_name, :latitude, :longitude, :geohash, NOW())
+                        INSERT INTO stations
+                            (id, station_code, thai_name, english_name,
+                             latitude, longitude, geohash, updated_at)
+                        VALUES
+                            (:station_id, :station_code, :thai_name, :english_name,
+                             :latitude, :longitude, :geohash, NOW())
                         ON CONFLICT (id) DO UPDATE SET
                             thai_name = EXCLUDED.thai_name,
                             english_name = EXCLUDED.english_name,
@@ -364,8 +408,12 @@ class SQLAdapter(DatabaseAdapter):
                         """
                     else:  # MySQL
                         station_sql = """
-                        INSERT INTO stations (id, station_code, thai_name, english_name, latitude, longitude, geohash, updated_at)
-                        VALUES (:station_id, :station_code, :thai_name, :english_name, :latitude, :longitude, :geohash, NOW())
+                        INSERT INTO stations
+                            (id, station_code, thai_name, english_name,
+                             latitude, longitude, geohash, updated_at)
+                        VALUES
+                            (:station_id, :station_code, :thai_name, :english_name,
+                             :latitude, :longitude, :geohash, NOW())
                         ON DUPLICATE KEY UPDATE
                             thai_name = VALUES(thai_name),
                             english_name = VALUES(english_name),
@@ -375,15 +423,18 @@ class SQLAdapter(DatabaseAdapter):
                             updated_at = NOW()
                         """
 
-                    conn.execute(text(station_sql), {
-                        'station_id': measurement['station_id'],
-                        'station_code': measurement['station_code'],
-                        'thai_name': measurement['station_name_th'],
-                        'english_name': measurement['station_name_en'],
-                        'latitude': measurement.get('latitude'),
-                        'longitude': measurement.get('longitude'),
-                        'geohash': measurement.get('geohash')
-                    })
+                    conn.execute(
+                        text(station_sql),
+                        {
+                            "station_id": measurement["station_id"],
+                            "station_code": measurement["station_code"],
+                            "thai_name": measurement["station_name_th"],
+                            "english_name": measurement["station_name_en"],
+                            "latitude": measurement.get("latitude"),
+                            "longitude": measurement.get("longitude"),
+                            "geohash": measurement.get("geohash"),
+                        },
+                    )
 
                 # Insert measurements
                 for measurement in measurements:
@@ -416,14 +467,17 @@ class SQLAdapter(DatabaseAdapter):
                             status = VALUES(status)
                         """
 
-                    conn.execute(text(measurement_sql), {
-                        'timestamp': measurement['timestamp'],
-                        'station_id': measurement['station_id'],
-                        'water_level': measurement['water_level'],
-                        'discharge': measurement['discharge'],
-                        'discharge_percent': measurement['discharge_percent'],
-                        'status': measurement['status']
-                    })
+                    conn.execute(
+                        text(measurement_sql),
+                        {
+                            "timestamp": measurement["timestamp"],
+                            "station_id": measurement["station_id"],
+                            "water_level": measurement["water_level"],
+                            "discharge": measurement["discharge"],
+                            "discharge_percent": measurement["discharge_percent"],
+                            "status": measurement["status"],
+                        },
+                    )
 
                 # Transaction is automatically committed when context manager exits
                 logging.info(f"Successfully saved {len(measurements)} measurements to {self.db_type.upper()}")
@@ -455,20 +509,22 @@ class SQLAdapter(DatabaseAdapter):
             """
 
             with self.engine.connect() as conn:
-                result = conn.execute(text(query), {'limit': limit})
+                result = conn.execute(text(query), {"limit": limit})
                 measurements = []
 
                 for row in result:
-                    measurements.append({
-                        'timestamp': row[0],
-                        'station_code': row[1],
-                        'station_name_en': row[2],
-                        'station_name_th': row[3],
-                        'water_level': float(row[4]) if row[4] else None,
-                        'discharge': float(row[5]) if row[5] else None,
-                        'discharge_percent': float(row[6]) if row[6] else None,
-                        'status': row[7]
-                    })
+                    measurements.append(
+                        {
+                            "timestamp": row[0],
+                            "station_code": row[1],
+                            "station_name_en": row[2],
+                            "station_name_th": row[3],
+                            "water_level": float(row[4]) if row[4] is not None else None,
+                            "discharge": float(row[5]) if row[5] is not None else None,
+                            "discharge_percent": float(row[6]) if row[6] is not None else None,
+                            "status": row[7],
+                        }
+                    )
 
                 return measurements
 
@@ -476,9 +532,12 @@ class SQLAdapter(DatabaseAdapter):
             logging.error(f"Error querying {self.db_type.upper()}: {e}")
             return []
 
-    def get_measurements_by_timerange(self, start_time: datetime.datetime, 
+    def get_measurements_by_timerange(
+        self,
+        start_time: datetime.datetime,
         end_time: datetime.datetime,
-                                    station_codes: Optional[List[str]] = None) -> List[Dict]:
+        station_codes: Optional[List[str]] = None,
+    ) -> List[Dict]:
         if not self.engine:
             return []
 
@@ -486,13 +545,13 @@ class SQLAdapter(DatabaseAdapter):
             from sqlalchemy import text
 
             where_clause = "m.timestamp BETWEEN :start_time AND :end_time"
-            params = {'start_time': start_time, 'end_time': end_time}
+            params = {"start_time": start_time, "end_time": end_time}
 
             if station_codes:
-                placeholders = ','.join([f':station_{i}' for i in range(len(station_codes))])
+                placeholders = ",".join([f":station_{i}" for i in range(len(station_codes))])
                 where_clause += f" AND s.station_code IN ({placeholders})"
                 for i, code in enumerate(station_codes):
-                    params[f'station_{i}'] = code
+                    params[f"station_{i}"] = code
 
             query = f"""
             SELECT m.timestamp, s.station_code, s.english_name, s.thai_name,
@@ -508,16 +567,18 @@ class SQLAdapter(DatabaseAdapter):
                 measurements = []
 
                 for row in result:
-                    measurements.append({
-                        'timestamp': row[0],
-                        'station_code': row[1],
-                        'station_name_en': row[2],
-                        'station_name_th': row[3],
-                        'water_level': float(row[4]) if row[4] else None,
-                        'discharge': float(row[5]) if row[5] else None,
-                        'discharge_percent': float(row[6]) if row[6] else None,
-                        'status': row[7]
-                    })
+                    measurements.append(
+                        {
+                            "timestamp": row[0],
+                            "station_code": row[1],
+                            "station_name_en": row[2],
+                            "station_name_th": row[3],
+                            "water_level": float(row[4]) if row[4] is not None else None,
+                            "discharge": float(row[5]) if row[5] is not None else None,
+                            "discharge_percent": float(row[6]) if row[6] is not None else None,
+                            "status": row[7],
+                        }
+                    )
 
                 return measurements
 
@@ -525,6 +586,52 @@ class SQLAdapter(DatabaseAdapter):
             logging.error(f"Error querying {self.db_type.upper()}: {e}")
             return []
 
+    def get_measurements_for_date(self, target_date: datetime.datetime) -> List[Dict]:
+        """Get all measurements for a specific date"""
+        if not self.engine:
+            return []
+
+        try:
+            from sqlalchemy import text
+
+            # Get start and end of the target date
+            start_of_day = target_date.replace(hour=0, minute=0, second=0, microsecond=0)
+            end_of_day = target_date.replace(hour=23, minute=59, second=59, microsecond=999999)
+
+            query = """
+            SELECT m.timestamp, m.station_id, s.station_code, s.thai_name,
+                   m.water_level, m.discharge, m.discharge_percent, m.status
+            FROM water_measurements m
+            LEFT JOIN stations s ON m.station_id = s.id
+            WHERE m.timestamp >= :start_time AND m.timestamp <= :end_time
+            ORDER BY m.timestamp DESC
+            """
+
+            with self.engine.connect() as conn:
+                result = conn.execute(text(query), {"start_time": start_of_day, "end_time": end_of_day})
+
+                measurements = []
+                for row in result:
+                    measurements.append(
+                        {
+                            "timestamp": row[0],
+                            "station_id": row[1],
+                            "station_code": row[2] or f"Station_{row[1]}",
+                            "station_name_th": row[3] or f"Station {row[1]}",
+                            "water_level": float(row[4]) if row[4] is not None else None,
+                            "discharge": float(row[5]) if row[5] is not None else None,
+                            "discharge_percent": float(row[6]) if row[6] is not None else None,
+                            "status": row[7],
+                        }
+                    )
+
+                return measurements
+
+        except Exception as e:
+            logging.error(f"Error querying {self.db_type.upper()} for date {target_date.date()}: {e}")
+            return []
+
+
 # VictoriaMetrics Adapter (using Prometheus format)
 class VictoriaMetricsAdapter(DatabaseAdapter):
     def __init__(self, host: str = "localhost", port: int = 8428):
@@ -532,11 +639,11 @@ class VictoriaMetricsAdapter(DatabaseAdapter):
         self.port = port
 
         # Handle HTTPS URLs and reverse proxy configurations
-        if host.startswith(('http://', 'https://')):
+        if host.startswith(("http://", "https://")):
             self.base_url = host
-            if port != 80 and port != 443 and not host.endswith(f':{port}'):
+            if port != 80 and port != 443 and not host.endswith(f":{port}"):
                 # Only add port if it's not standard and not already in URL
-                if '://' in host and ':' not in host.split('://')[1]:
+                if "://" in host and ":" not in host.split("://")[1]:
                     self.base_url = f"{host}:{port}"
         else:
             # Default to HTTP for localhost, HTTPS for remote hosts
@@ -546,14 +653,33 @@ class VictoriaMetricsAdapter(DatabaseAdapter):
             else:
                 self.base_url = f"{protocol}://{host}:{port}"
 
+    @staticmethod
+    def _escape_label(value) -> str:
+        """Escape a Prometheus label value per the exposition format spec.
+
+        Station names include arbitrary Thai text (and could be set via the API),
+        so backslashes, double-quotes and newlines must be escaped to avoid
+        producing malformed or injected exposition lines.
+        """
+        return str(value).replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n")
+
+    @staticmethod
+    def _metric_value(value) -> Optional[float]:
+        """Coerce a numeric field to float, or None if it isn't a valid number."""
+        try:
+            return float(value)
+        except (TypeError, ValueError):
+            return None
+
     def connect(self):
         try:
             import requests
+
             # Test connection with SSL verification and timeout
             response = requests.get(
                 f"{self.base_url}/api/v1/status/config",
                 timeout=10,
-                verify=True  # Enable SSL verification for HTTPS
+                verify=True,  # Enable SSL verification for HTTPS
             )
             if response.status_code == 200:
                 logging.info(f"Connected to VictoriaMetrics successfully at {self.base_url}")
@@ -580,38 +706,35 @@ class VictoriaMetricsAdapter(DatabaseAdapter):
             timestamp_ms = int(datetime.datetime.now().timestamp() * 1000)
 
             for measurement in measurements:
-                # Water level metric
-                metrics_data.append(
-                    f'water_level{{station_code="{measurement["station_code"]}",'
-                    f'station_name_en="{measurement["station_name_en"]}",'
-                    f'station_name_th="{measurement["station_name_th"]}"}} '
-                    f'{measurement["water_level"]} {timestamp_ms}'
+                # Escape label values once per measurement (untrusted Thai/English names).
+                labels = (
+                    f'station_code="{self._escape_label(measurement["station_code"])}",'
+                    f'station_name_en="{self._escape_label(measurement["station_name_en"])}",'
+                    f'station_name_th="{self._escape_label(measurement["station_name_th"])}"'
                 )
 
+                # Water level metric
+                water_level = self._metric_value(measurement.get("water_level"))
+                if water_level is not None:
+                    metrics_data.append(f"water_level{{{labels}}} {water_level} {timestamp_ms}")
+
                 # Discharge metric
-                metrics_data.append(
-                    f'water_discharge{{station_code="{measurement["station_code"]}",'
-                    f'station_name_en="{measurement["station_name_en"]}",'
-                    f'station_name_th="{measurement["station_name_th"]}"}} '
-                    f'{measurement["discharge"]} {timestamp_ms}'
-                )
+                discharge = self._metric_value(measurement.get("discharge"))
+                if discharge is not None:
+                    metrics_data.append(f"water_discharge{{{labels}}} {discharge} {timestamp_ms}")
 
                 # Discharge percentage metric
-                if measurement["discharge_percent"]:
-                    metrics_data.append(
-                        f'water_discharge_percent{{station_code="{measurement["station_code"]}",'
-                        f'station_name_en="{measurement["station_name_en"]}",'
-                        f'station_name_th="{measurement["station_name_th"]}"}} '
-                        f'{measurement["discharge_percent"]} {timestamp_ms}'
-                    )
+                discharge_percent = self._metric_value(measurement.get("discharge_percent"))
+                if discharge_percent is not None:
+                    metrics_data.append(f"water_discharge_percent{{{labels}}} {discharge_percent} {timestamp_ms}")
 
             # Send to VictoriaMetrics
-            data = '\n'.join(metrics_data)
+            data = "\n".join(metrics_data)
             response = requests.post(
                 f"{self.base_url}/api/v1/import/prometheus",
                 data=data,
-                headers={'Content-Type': 'text/plain'},
-                timeout=30
+                headers={"Content-Type": "text/plain"},
+                timeout=30,
             )
 
             if response.status_code == 204:
@@ -631,13 +754,22 @@ class VictoriaMetricsAdapter(DatabaseAdapter):
         logging.warning("get_latest_measurements not fully implemented for VictoriaMetrics")
         return []
 
-    def get_measurements_by_timerange(self, start_time: datetime.datetime, 
+    def get_measurements_by_timerange(
+        self,
+        start_time: datetime.datetime,
         end_time: datetime.datetime,
-                                    station_codes: Optional[List[str]] = None) -> List[Dict]:
+        station_codes: Optional[List[str]] = None,
+    ) -> List[Dict]:
         # VictoriaMetrics range queries would be implemented here
         logging.warning("get_measurements_by_timerange not fully implemented for VictoriaMetrics")
         return []
 
+    def get_measurements_for_date(self, target_date: datetime.datetime) -> List[Dict]:
+        """Get all measurements for a specific date"""
+        logging.warning("get_measurements_for_date not fully implemented for VictoriaMetrics")
+        return []
+
+
 # Factory function to create appropriate adapter
 def create_database_adapter(db_type: str, **kwargs) -> DatabaseAdapter:
     """
@@ -649,15 +781,15 @@ def create_database_adapter(db_type: str, **kwargs) -> DatabaseAdapter:
     """
     db_type = db_type.lower()
 
-    if db_type == 'influxdb':
+    if db_type == "influxdb":
         return InfluxDBAdapter(**kwargs)
-    elif db_type == 'mysql':
-        return SQLAdapter(db_type='mysql', **kwargs)
-    elif db_type == 'postgresql':
-        return SQLAdapter(db_type='postgresql', **kwargs)
-    elif db_type == 'sqlite':
-        return SQLAdapter(db_type='sqlite', **kwargs)
-    elif db_type == 'victoriametrics':
+    elif db_type == "mysql":
+        return SQLAdapter(db_type="mysql", **kwargs)
+    elif db_type == "postgresql":
+        return SQLAdapter(db_type="postgresql", **kwargs)
+    elif db_type == "sqlite":
+        return SQLAdapter(db_type="sqlite", **kwargs)
+    elif db_type == "victoriametrics":
         return VictoriaMetricsAdapter(**kwargs)
     else:
         raise ValueError(f"Unsupported database type: {db_type}")

src/main.py

@@ -7,6 +7,7 @@ import argparse
 import asyncio
 import sys
 import signal
+import time
 from datetime import datetime
 from typing import Optional
 
@@ -63,7 +64,7 @@ def run_test_cycle():
         return False
 
 def run_continuous_monitoring():
-    """Run continuous monitoring with scheduling"""
+    """Run continuous monitoring with adaptive scheduling and alerting"""
     logger.info("Starting continuous monitoring...")
 
     try:
@@ -74,24 +75,76 @@ def run_continuous_monitoring():
         db_config = Config.get_database_config()
         scraper = EnhancedWaterMonitorScraper(db_config)
 
+        # Initialize alerting system
+        from .alerting import WaterLevelAlertSystem
+        alerting = WaterLevelAlertSystem()
+
         # Setup signal handlers
         setup_signal_handlers(scraper)
 
         logger.info(f"Monitoring started with {Config.SCRAPING_INTERVAL_HOURS}h interval")
+        logger.info("Adaptive retry: switches to 1-minute intervals when no data available")
+        logger.info("Alerts: automatic check after each successful data fetch")
         logger.info("Press Ctrl+C to stop")
 
         # Run initial cycle
         logger.info("Running initial data collection...")
-        scraper.run_scraping_cycle()
+        initial_success = scraper.run_scraping_cycle()
 
-        # Start scheduled monitoring
-        import schedule
+        # Adaptive scheduling state
+        from datetime import datetime, timedelta
+        retry_mode = not initial_success
+        last_successful_fetch = None if not initial_success else datetime.now()
 
-        schedule.every(Config.SCRAPING_INTERVAL_HOURS).hours.do(scraper.run_scraping_cycle)
+        if retry_mode:
+            logger.warning("No data fetched in initial run - entering retry mode")
+            next_run = datetime.now() + timedelta(minutes=1)
+        else:
+            logger.info("Initial data fetch successful - using hourly schedule")
+            next_run = (datetime.now() + timedelta(hours=1)).replace(minute=0, second=0, microsecond=0)
+
+        logger.info(f"Next run at {next_run.strftime('%H:%M')}")
 
         while True:
-            schedule.run_pending()
-            time.sleep(60)  # Check every minute
+            current_time = datetime.now()
+
+            if current_time >= next_run:
+                logger.info("Running scheduled data collection...")
+                success = scraper.run_scraping_cycle()
+
+                if success:
+                    last_successful_fetch = current_time
+
+                    # Run alert check after every successful new data fetch
+                    logger.info("Running alert check...")
+                    try:
+                        alert_results = alerting.run_alert_check()
+                        if alert_results.get('total_alerts', 0) > 0:
+                            logger.info(f"Alerts: {alert_results['total_alerts']} generated, {alert_results['sent']} sent")
+                    except Exception as e:
+                        logger.error(f"Alert check failed: {e}")
+
+                    if retry_mode:
+                        logger.info("✅ Data fetch successful - switching back to hourly schedule")
+                        retry_mode = False
+                        # Schedule next run at the next full hour
+                        next_run = (current_time + timedelta(hours=1)).replace(minute=0, second=0, microsecond=0)
+                    else:
+                        # Continue hourly schedule
+                        next_run = (current_time + timedelta(hours=Config.SCRAPING_INTERVAL_HOURS)).replace(minute=0, second=0, microsecond=0)
+
+                    logger.info(f"Next scheduled run at {next_run.strftime('%H:%M')}")
+                else:
+                    if not retry_mode:
+                        logger.warning("⚠️ No data fetched - switching to retry mode (1-minute intervals)")
+                        retry_mode = True
+
+                    # Schedule retry in 1 minute
+                    next_run = current_time + timedelta(minutes=1)
+                    logger.info(f"Retrying in 1 minute at {next_run.strftime('%H:%M')}")
+
+            # Sleep for 10 seconds and check again
+            time.sleep(10)
 
     except KeyboardInterrupt:
         logger.info("Monitoring stopped by user")
@@ -153,6 +206,45 @@ def run_data_update(days_back: int):
         logger.error(f"❌ Data update failed: {e}")
         return False
 
+def run_historical_import(start_date_str: str, end_date_str: str, skip_existing: bool = True):
+    """Import historical data for a date range"""
+    try:
+        # Parse dates
+        start_date = datetime.strptime(start_date_str, "%Y-%m-%d")
+        end_date = datetime.strptime(end_date_str, "%Y-%m-%d")
+
+        if start_date > end_date:
+            logger.error("Start date must be before or equal to end date")
+            return False
+
+        logger.info(f"Importing historical data from {start_date.date()} to {end_date.date()}")
+        if skip_existing:
+            logger.info("Skipping dates that already have data")
+
+        # Validate configuration
+        Config.validate_config()
+
+        # Initialize scraper
+        db_config = Config.get_database_config()
+        scraper = EnhancedWaterMonitorScraper(db_config)
+
+        # Import historical data
+        imported_count = scraper.import_historical_data(start_date, end_date, skip_existing)
+
+        if imported_count > 0:
+            logger.info(f"✅ Imported {imported_count} historical data points")
+        else:
+            logger.info("✅ No new data imported")
+
+        return True
+
+    except ValueError as e:
+        logger.error(f"❌ Invalid date format. Use YYYY-MM-DD: {e}")
+        return False
+    except Exception as e:
+        logger.error(f"❌ Historical import failed: {e}")
+        return False
+
 def run_web_api():
     """Run the FastAPI web interface"""
     logger.info("Starting web API server...")
@@ -179,6 +271,65 @@ def run_web_api():
         logger.error(f"Web API failed: {e}")
         return False
 
+def run_alert_check():
+    """Run water level alert check"""
+    logger.info("Running water level alert check...")
+
+    try:
+        from .alerting import WaterLevelAlertSystem
+
+        # Initialize alerting system
+        alerting = WaterLevelAlertSystem()
+
+        # Run alert check
+        results = alerting.run_alert_check()
+
+        if 'error' in results:
+            logger.error("❌ Alert check failed due to database connection")
+            return False
+
+        logger.info(f"✅ Alert check completed:")
+        logger.info(f"  • Water level alerts: {results['water_alerts']}")
+        logger.info(f"  • Data freshness alerts: {results['data_alerts']}")
+        logger.info(f"  • Total alerts generated: {results['total_alerts']}")
+        logger.info(f"  • Alerts sent: {results['sent']}")
+
+        return True
+
+    except Exception as e:
+        logger.error(f"❌ Alert check failed: {e}")
+        return False
+
+def run_alert_test():
+    """Send test alert message"""
+    logger.info("Sending test alert message...")
+
+    try:
+        from .alerting import WaterLevelAlertSystem
+
+        # Initialize alerting system
+        alerting = WaterLevelAlertSystem()
+
+        if not alerting.matrix_notifier:
+            logger.error("❌ Matrix notifier not configured")
+            logger.info("Please set MATRIX_ACCESS_TOKEN and MATRIX_ROOM_ID in your .env file")
+            return False
+
+        # Send test message
+        test_message = "🧪 **Test Alert**\n\nThis is a test message from the Northern Thailand Ping River Monitor.\n\nIf you received this, Matrix notifications are working correctly!"
+        success = alerting.matrix_notifier.send_message(test_message)
+
+        if success:
+            logger.info("✅ Test alert message sent successfully")
+        else:
+            logger.error("❌ Test alert message failed to send")
+
+        return success
+
+    except Exception as e:
+        logger.error(f"❌ Test alert failed: {e}")
+        return False
+
 def show_status():
     """Show current system status"""
     logger.info("=== Northern Thailand Ping River Monitor Status ===")
@@ -209,6 +360,20 @@ def show_status():
         else:
             logger.error("❌ Database connection failed")
 
+        # Test alerting system
+        logger.info("\n=== Alerting System Status ===")
+        try:
+            from .alerting import WaterLevelAlertSystem
+            alerting = WaterLevelAlertSystem()
+
+            if alerting.matrix_notifier:
+                logger.info("✅ Matrix notifications configured")
+            else:
+                logger.warning("⚠️ Matrix notifications not configured")
+                logger.info("Set MATRIX_ACCESS_TOKEN and MATRIX_ROOM_ID in .env file")
+        except Exception as e:
+            logger.error(f"❌ Alerting system error: {e}")
+
         # Show metrics if available
         metrics_collector = get_metrics_collector()
         metrics = metrics_collector.get_all_metrics()
@@ -237,7 +402,10 @@ Examples:
   %(prog)s --web-api                # Start web API server
   %(prog)s --fill-gaps 7            # Fill missing data for last 7 days
   %(prog)s --update-data 2          # Update existing data for last 2 days
+  %(prog)s --import-historical 2024-01-01 2024-01-31  # Import historical data
   %(prog)s --status                 # Show system status
+  %(prog)s --alert-check            # Check water levels and send alerts
+  %(prog)s --alert-test             # Send test Matrix message
         """
     )
     
@@ -267,12 +435,37 @@ Examples:
         help="Update existing data for the specified number of days back"
     )
     
+    parser.add_argument(
+        "--import-historical",
+        nargs=2,
+        metavar=("START_DATE", "END_DATE"),
+        help="Import historical data for date range (YYYY-MM-DD format)"
+    )
+
+    parser.add_argument(
+        "--force-overwrite",
+        action="store_true",
+        help="Overwrite existing data when importing historical data"
+    )
+
     parser.add_argument(
         "--status",
         action="store_true",
         help="Show current system status"
     )
 
+    parser.add_argument(
+        "--alert-check",
+        action="store_true",
+        help="Run water level alert check"
+    )
+
+    parser.add_argument(
+        "--alert-test",
+        action="store_true",
+        help="Send test alert message to Matrix"
+    )
+
     parser.add_argument(
         "--log-level",
         choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
@@ -297,7 +490,7 @@ Examples:
     )
     
     logger.info("🏔️ Northern Thailand Ping River Monitor starting...")
-    logger.info(f"Version: 3.1.2")
+    logger.info(f"Version: 3.1.3")
     logger.info(f"Log level: {args.log_level}")
     
     try:
@@ -311,8 +504,16 @@ Examples:
             success = run_gap_filling(args.fill_gaps)
         elif args.update_data is not None:
             success = run_data_update(args.update_data)
+        elif args.import_historical is not None:
+            start_date, end_date = args.import_historical
+            skip_existing = not args.force_overwrite
+            success = run_historical_import(start_date, end_date, skip_existing)
         elif args.status:
             success = show_status()
+        elif args.alert_check:
+            success = run_alert_check()
+        elif args.alert_test:
+            success = run_alert_test()
         else:
             success = run_continuous_monitoring()
         

src/ml/__init__.py

@@ -0,0 +1 @@
+"""Flood forecasting ML package: data loading, feature/label engineering, training, and prediction."""

src/ml/data.py

@@ -0,0 +1,223 @@
+"""Loaders for flood-model training/inference data.
+
+Primary path reads raw measurements straight from PostgreSQL (keeping NULL
+discharge as NULL). HTTP fallback goes through the public API's history
+endpoint, which backfills missing discharge with a synthetic rating-curve
+estimate -- callers are told about that via the `discharge_maybe_synthetic`
+cache metadata flag.
+"""
+
+import datetime
+import gzip
+import json
+import logging
+import os
+from pathlib import Path
+from typing import Dict, List, Optional
+
+import pandas as pd
+from sqlalchemy import create_engine, text
+
+from ..config import Config
+from .features import UPSTREAM_LEADS
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_API_URL = "http://100.81.167.42:8000"
+CACHE_DIR = Path("models/cache")
+_MEASUREMENT_COLUMNS = ["timestamp", "station_code", "water_level", "discharge"]
+
+
+def resolve_db_url(db_url: Optional[str] = None) -> Optional[str]:
+    """Resolve a Postgres connection string: explicit param > FLOOD_ML_DB_URL env >
+    Config's postgresql connection string > None (caller should fall back to HTTP)."""
+    if db_url:
+        return db_url
+
+    env_url = os.getenv("FLOOD_ML_DB_URL")
+    if env_url:
+        return env_url
+
+    try:
+        db_config = Config.get_database_config()
+    except Exception as error:
+        logger.warning(f"Could not resolve database config: {error}")
+        return None
+
+    if db_config.get("type") == "postgresql":
+        return db_config.get("connection_string")
+    return None
+
+
+def _default_stations() -> List[str]:
+    return list(UPSTREAM_LEADS.keys())
+
+
+def _normalize_long(df: pd.DataFrame) -> pd.DataFrame:
+    if df.empty:
+        return pd.DataFrame(columns=_MEASUREMENT_COLUMNS)
+    df = df.copy()
+    df["timestamp"] = pd.to_datetime(df["timestamp"]).dt.floor("h")
+    df["water_level"] = pd.to_numeric(df["water_level"], errors="coerce")
+    df["discharge"] = pd.to_numeric(df["discharge"], errors="coerce")
+    df = df.drop_duplicates(subset=["station_code", "timestamp"], keep="last")
+    df = df.sort_values("timestamp").reset_index(drop=True)
+    return df[_MEASUREMENT_COLUMNS]
+
+
+def _fetch_from_db(
+    db_url: str,
+    stations: Optional[List[str]],
+    start: Optional[datetime.datetime],
+    end: Optional[datetime.datetime],
+) -> pd.DataFrame:
+    engine = create_engine(db_url, pool_pre_ping=True)
+    query = (
+        "SELECT m.timestamp, s.station_code, m.water_level, m.discharge "
+        "FROM water_measurements m JOIN stations s ON m.station_id = s.id WHERE 1=1"
+    )
+    params: Dict = {}
+    if start is not None:
+        query += " AND m.timestamp >= :start_time"
+        params["start_time"] = start
+    if end is not None:
+        query += " AND m.timestamp <= :end_time"
+        params["end_time"] = end
+    if stations:
+        placeholders = ", ".join(f":station_{i}" for i in range(len(stations)))
+        query += f" AND s.station_code IN ({placeholders})"
+        for i, code in enumerate(stations):
+            params[f"station_{i}"] = code
+    query += " ORDER BY m.timestamp"
+
+    with engine.connect() as connection:
+        df = pd.read_sql(text(query), connection, params=params)
+    return _normalize_long(df)
+
+
+def _fetch_station_from_api(api_url: str, station_code: str, hours: int, limit: int = 100000) -> pd.DataFrame:
+    import requests
+
+    response = requests.get(
+        f"{api_url}/measurements/history/{station_code}",
+        params={"hours": hours, "limit": limit},
+        timeout=30,
+    )
+    response.raise_for_status()
+    rows = response.json()
+    for row in rows:
+        row["station_code"] = station_code
+    return pd.DataFrame(rows, columns=_MEASUREMENT_COLUMNS + ["discharge_percent"])
+
+
+def _fetch_from_api(
+    api_url: str,
+    stations: List[str],
+    start: Optional[datetime.datetime],
+    end: Optional[datetime.datetime],
+) -> pd.DataFrame:
+    now = datetime.datetime.now()
+    reference_end = end or now
+    reference_start = start or (reference_end - datetime.timedelta(days=365 * 8))
+    hours = max(1, int((reference_end - reference_start).total_seconds() // 3600) + 1)
+
+    frames = []
+    for code in stations:
+        try:
+            frames.append(_fetch_station_from_api(api_url, code, hours))
+        except Exception as error:
+            logger.warning(f"HTTP fallback failed for station {code}: {error}")
+    if not frames:
+        return pd.DataFrame(columns=_MEASUREMENT_COLUMNS)
+    df = pd.concat(frames, ignore_index=True)
+    return _normalize_long(df)
+
+
+def _write_cache(df: pd.DataFrame, cache_dir: Path, source: str, discharge_maybe_synthetic: bool) -> None:
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    for code, group in df.groupby("station_code"):
+        path = cache_dir / f"{code}.csv.gz"
+        with gzip.open(path, "wt", encoding="utf-8", newline="") as handle:
+            group.to_csv(handle, index=False)
+    meta = {
+        "fetched_at": datetime.datetime.now().isoformat(),
+        "source": source,
+        "discharge_maybe_synthetic": discharge_maybe_synthetic,
+    }
+    with open(cache_dir / "meta.json", "w", encoding="utf-8") as handle:
+        json.dump(meta, handle)
+
+
+def _read_cache(cache_dir: Path, stations: Optional[List[str]]) -> pd.DataFrame:
+    if not cache_dir.exists():
+        return pd.DataFrame(columns=_MEASUREMENT_COLUMNS)
+    frames = []
+    for path in sorted(cache_dir.glob("*.csv.gz")):
+        code = path.name[: -len(".csv.gz")]
+        if stations and code not in stations:
+            continue
+        with gzip.open(path, "rt", encoding="utf-8") as handle:
+            frames.append(pd.read_csv(handle, parse_dates=["timestamp"]))
+    if not frames:
+        return pd.DataFrame(columns=_MEASUREMENT_COLUMNS)
+    return _normalize_long(pd.concat(frames, ignore_index=True))
+
+
+def load_measurements(
+    db_url: Optional[str] = None,
+    stations: Optional[List[str]] = None,
+    start: Optional[datetime.datetime] = None,
+    end: Optional[datetime.datetime] = None,
+    use_cache: bool = True,
+    cache_dir: Path = CACHE_DIR,
+    api_url: str = DEFAULT_API_URL,
+) -> pd.DataFrame:
+    """Load the long-format [timestamp, station_code, water_level, discharge] history.
+
+    Tries PostgreSQL first, then the HTTP API, then the on-disk cache as a last
+    resort. A successful DB/API fetch refreshes the cache; the cache itself is
+    never treated as a source of fresh data.
+    """
+    resolved_db_url = resolve_db_url(db_url)
+
+    if resolved_db_url:
+        try:
+            df = _fetch_from_db(resolved_db_url, stations, start, end)
+            if use_cache:
+                _write_cache(df, cache_dir, source="postgres", discharge_maybe_synthetic=False)
+            return df
+        except Exception as error:
+            logger.warning(f"PostgreSQL fetch failed, falling back to HTTP API: {error}")
+
+    try:
+        api_stations = stations or _default_stations()
+        df = _fetch_from_api(api_url, api_stations, start, end)
+        if not df.empty:
+            if use_cache:
+                _write_cache(df, cache_dir, source="api", discharge_maybe_synthetic=True)
+            return df
+    except Exception as error:
+        logger.warning(f"HTTP API fetch failed: {error}")
+
+    if use_cache:
+        logger.warning("Falling back to on-disk cache for measurement history")
+        return _read_cache(cache_dir, stations)
+
+    return pd.DataFrame(columns=_MEASUREMENT_COLUMNS)
+
+
+def load_latest(
+    db_url: Optional[str] = None,
+    hours: int = 336,
+    stations: Optional[List[str]] = None,
+) -> pd.DataFrame:
+    """Load the last `hours` of history for all (or given) stations. Never cached to disk."""
+    end = datetime.datetime.now()
+    start = end - datetime.timedelta(hours=hours)
+    return load_measurements(
+        db_url=db_url,
+        stations=stations,
+        start=start,
+        end=end,
+        use_cache=False,
+    )

src/ml/features.py

@@ -0,0 +1,310 @@
+"""Static config and feature/label engineering for the Ping River flood forecast models.
+
+All feature computation is strictly causal (no row uses information timestamped after
+itself) so it is safe to run identically at training time and at prediction time.
+"""
+
+import datetime
+import logging
+from dataclasses import dataclass
+from typing import Dict, List, Optional, Tuple
+
+import numpy as np
+import pandas as pd
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Static configuration
+# ---------------------------------------------------------------------------
+
+# Per-station (warning, danger) level thresholds in meters. "*" is the default
+# applied to any station without an explicit override.
+# Per-station (warning, danger) levels in metres on each gauge's own datum.
+# Calibrated 2026-08-10 from the DB's discharge_percent (RID % of channel
+# capacity): warning = median level at 75-85% capacity, danger = median level
+# at 95-105%. P.1 instead uses the official Chiang Mai inundation map keyed to
+# the P.1 gauge: city flooding begins at 3.70 m (stage 1) and reaches most
+# districts by 4.20 m (stage 5) — see P1_FLOOD_STAGES.
+THRESHOLDS: Dict[str, Tuple[float, float]] = {
+    "*": (3.0, 4.5),
+    "P.1": (3.70, 4.20),
+    "P.103": (5.95, 6.75),
+    "P.20": (2.35, 2.80),
+    "P.21": (3.20, 3.60),
+    "P.4A": (3.40, 3.90),
+    "P.5": (4.55, 4.95),
+    "P.67": (2.45, 2.90),
+    "P.75": (2.75, 3.50),
+    "P.76": (5.35, 5.45),
+    "P.77": (2.85, 3.35),
+    "P.81": (5.15, 6.30),
+    "P.82": (3.40, 3.80),
+    "P.84": (3.45, 3.90),
+    "P.85": (2.90, 3.35),
+    "P.87": (3.75, 4.05),
+    "P.92": (2.95, 3.60),
+}
+
+# Official Chiang Mai flood-onset stages at the P.1 gauge (Nawarat Bridge),
+# from the municipal inundation map (พื้นที่ท่วมตัวเมืองเชียงใหม่, events of
+# 2548/2554/2565 BE): gauge level in m, RID discharge in m³/s. Each stage
+# floods progressively more city zones.
+P1_FLOOD_STAGES: List[Dict[str, float]] = [
+    {"stage": 1, "level": 3.70, "discharge_cms": 405},
+    {"stage": 2, "level": 3.90, "discharge_cms": 438},
+    {"stage": 3, "level": 4.00, "discharge_cms": 458},
+    {"stage": 4, "level": 4.10, "discharge_cms": 478},
+    {"stage": 5, "level": 4.20, "discharge_cms": 493},
+    {"stage": 6, "level": 4.30, "discharge_cms": 508},
+    {"stage": 7, "level": 4.60, "discharge_cms": 558},
+]
+
+FLOOD_STAGES: Dict[str, List[Dict[str, float]]] = {"P.1": P1_FLOOD_STAGES}
+
+MONSOON_MONTHS = {6, 7, 8, 9, 10}
+FFILL_LIMIT_H = 3
+MIN_WINDOW_COVERAGE = 0.5
+
+BASIN_ANCHOR = "P.1"
+
+# Empirical hours a station's water-level anomaly leads the basin anchor (P.1),
+# derived from data-scout cross-correlation analysis. UPSTREAM_LEADS[station]
+# lists, for each station, the (upstream_code, lead_hours) pairs to use as
+# routed-upstream input features when forecasting `station`.
+UPSTREAM_LEADS: Dict[str, List[Tuple[str, int]]] = {
+    "P.1": [("P.103", 1), ("P.67", 7), ("P.21", 9), ("P.75", 12), ("P.4A", 12), ("P.92", 15), ("P.20", 17)],
+    "P.103": [("P.67", 6), ("P.21", 8), ("P.75", 11), ("P.4A", 11), ("P.92", 14), ("P.20", 16)],
+    "P.21": [("P.67", 1), ("P.75", 3), ("P.4A", 3), ("P.92", 6), ("P.20", 8)],
+    "P.67": [("P.75", 5), ("P.4A", 5), ("P.92", 8), ("P.20", 10)],
+    "P.75": [("P.92", 3), ("P.20", 5)],
+    "P.4A": [("P.92", 3), ("P.20", 5)],
+    "P.92": [("P.20", 2)],
+    "P.20": [],
+    "P.5": [("P.1", 12), ("P.103", 13)],
+    "P.81": [("P.1", 4), ("P.103", 5)],
+    "P.82": [],
+    "P.84": [],
+    "P.87": [],
+    "P.77": [],
+    "P.85": [],
+    "P.76": [],
+}
+
+# Per-station usable-from dates: data before this cutoff is excluded from training
+# because of known data-quality holes (see data-scout inventory).
+TRAIN_START: Dict[str, str] = {"P.5": "2022-01-01"}
+
+# Stations with data too sparse/broken to ever be a regression/classification
+# target. They are still usable as upstream *input* features (HGB tolerates NaN).
+NOT_TRAINABLE: Dict[str, str] = {"P.4A": "17% fill, dead 2019-2024"}
+
+
+def get_thresholds(station_code: str) -> Tuple[float, float]:
+    """Return (warning, danger) level thresholds for a station, falling back to the default."""
+    return THRESHOLDS.get(station_code, THRESHOLDS["*"])
+
+
+# ---------------------------------------------------------------------------
+# Hourly grid
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class HourlyGrid:
+    """A complete hourly time grid pivoted wide across stations.
+
+    observed: raw values, NaN where nothing was recorded that hour (pristine; used for labels).
+    filled:   observed forward-filled per column with limit=FFILL_LIMIT_H (causal; used for features).
+    mask:     boolean, True where `observed` has a real reading.
+    """
+
+    observed: pd.DataFrame
+    filled: pd.DataFrame
+    mask: pd.DataFrame
+
+
+def make_hourly_grid(df_long: pd.DataFrame) -> HourlyGrid:
+    """Pivot a long station/timestamp measurement frame onto a complete hourly grid.
+
+    df_long columns: timestamp, station_code, water_level, discharge.
+    """
+    if df_long.empty:
+        empty = pd.DataFrame(
+            index=pd.DatetimeIndex([], name="timestamp"),
+            columns=pd.MultiIndex.from_tuples([], names=["station_code", "field"]),
+        )
+        return HourlyGrid(observed=empty, filled=empty.copy(), mask=empty.copy())
+
+    df = df_long.copy()
+    df["timestamp"] = pd.to_datetime(df["timestamp"]).dt.floor("h")
+    df = df.drop_duplicates(subset=["station_code", "timestamp"], keep="last")
+
+    full_index = pd.date_range(df["timestamp"].min(), df["timestamp"].max(), freq="h", name="timestamp")
+
+    wide = df.pivot(index="timestamp", columns="station_code", values=["water_level", "discharge"])
+    wide = wide.reorder_levels([1, 0], axis=1).sort_index(axis=1)
+    wide = wide.reindex(full_index)
+
+    observed = wide
+    mask = observed.notna()
+    # Forward-fill only — never interpolate — so no row ever depends on a future value.
+    filled = observed.ffill(limit=FFILL_LIMIT_H)
+
+    return HourlyGrid(observed=observed, filled=filled, mask=mask)
+
+
+def _series(grid_frame: pd.DataFrame, station: str, field: str, index: pd.Index) -> pd.Series:
+    """Fetch a (station, field) column, or an all-NaN series if the station is absent."""
+    if (station, field) in grid_frame.columns:
+        return grid_frame[(station, field)]
+    return pd.Series(np.nan, index=index)
+
+
+def _hours_since_observed(mask_col: pd.Series) -> pd.Series:
+    """Hours since the last True in `mask_col` (0 at an observed hour; NaN if never observed yet)."""
+    idx = mask_col.index
+    obs_time = pd.Series(idx, index=idx).where(mask_col.to_numpy())
+    last_obs_time = obs_time.ffill()
+    age_hours = (idx.to_series() - last_obs_time).dt.total_seconds() / 3600.0
+    return age_hours
+
+
+# ---------------------------------------------------------------------------
+# Features
+# ---------------------------------------------------------------------------
+
+
+def build_features(grid: HourlyGrid, station: str) -> pd.DataFrame:
+    """Build the deterministic-order feature matrix for one target station."""
+    idx = grid.observed.index
+    cols: Dict[str, pd.Series] = {}
+
+    level = _series(grid.filled, station, "water_level", idx)
+    discharge = _series(grid.observed, station, "discharge", idx)
+    obs_mask = _series(grid.mask, station, "water_level", idx).fillna(False)
+
+    cols["level"] = level
+    for k in (1, 2, 3, 6, 12, 24, 48, 72):
+        cols[f"level_lag_{k}"] = level.shift(k)
+    for k in (1, 3, 6, 12, 24):
+        cols[f"rise_{k}"] = level - level.shift(k)
+    cols["roll_mean_6"] = level.rolling(6, min_periods=1).mean()
+    cols["roll_mean_24"] = level.rolling(24, min_periods=1).mean()
+    cols["roll_max_6"] = level.rolling(6, min_periods=1).max()
+    cols["roll_max_24"] = level.rolling(24, min_periods=1).max()
+    cols["roll_max_72"] = level.rolling(72, min_periods=1).max()
+    cols["roll_min_24"] = level.rolling(24, min_periods=1).min()
+
+    cols["discharge"] = discharge
+    cols["discharge_lag_6"] = discharge.shift(6)
+    cols["discharge_lag_24"] = discharge.shift(24)
+    cols["discharge_rise_6"] = discharge - discharge.shift(6)
+
+    obs_age_h = _hours_since_observed(obs_mask)
+    cols["obs_age_h"] = obs_age_h.where(obs_age_h <= FFILL_LIMIT_H)
+    cols["cov_24h"] = obs_mask.rolling(24, min_periods=1).mean()
+
+    for upstream_code, lead_h in UPSTREAM_LEADS.get(station, []):
+        u_level = _series(grid.filled, upstream_code, "water_level", idx)
+        u_rise_6 = u_level - u_level.shift(6)
+        u_rollmax_24 = u_level.rolling(24, min_periods=1).max()
+        near_lag = max(0, lead_h - 3)
+        cols[f"{upstream_code}_level_lag_{near_lag}"] = u_level.shift(near_lag)
+        cols[f"{upstream_code}_level_lag_{lead_h}"] = u_level.shift(lead_h)
+        cols[f"{upstream_code}_level_lag_{lead_h + 3}"] = u_level.shift(lead_h + 3)
+        cols[f"{upstream_code}_rise_6_lag_{lead_h}"] = u_rise_6.shift(lead_h)
+        cols[f"{upstream_code}_rollmax_24_lag_{near_lag}"] = u_rollmax_24.shift(near_lag)
+
+    if station != BASIN_ANCHOR:
+        p1_level = _series(grid.filled, BASIN_ANCHOR, "water_level", idx)
+        cols["P1_level"] = p1_level
+        cols["P1_rollmax_24"] = p1_level.rolling(24, min_periods=1).max()
+        cols["P1_rise_24"] = p1_level - p1_level.shift(24)
+
+    doy = idx.to_series().dt.dayofyear.astype(float)
+    cols["doy_sin"] = np.sin(2 * np.pi * doy / 365.25)
+    cols["doy_cos"] = np.cos(2 * np.pi * doy / 365.25)
+    cols["is_monsoon"] = idx.to_series().dt.month.isin(MONSOON_MONTHS).astype(float)
+
+    return pd.DataFrame(cols, index=idx)
+
+
+# ---------------------------------------------------------------------------
+# Labels
+# ---------------------------------------------------------------------------
+
+
+def _future_window_stats(col: pd.Series, horizon_h: int) -> Tuple[pd.Series, pd.Series]:
+    """For every t, (max, count) of observed values in the OPEN window (t, t+horizon_h]."""
+    reversed_col = col.iloc[::-1]
+    shifted = reversed_col.shift(1)  # excludes t itself
+    fut_max = shifted.rolling(horizon_h, min_periods=1).max().iloc[::-1]
+    fut_count = shifted.rolling(horizon_h, min_periods=1).count().iloc[::-1]
+    return fut_max, fut_count
+
+
+def build_labels(grid: HourlyGrid, station: str, horizons: Tuple[int, ...] = (6, 12, 24)) -> pd.DataFrame:
+    """Build max-level and threshold-exceedance labels for one target station."""
+    idx = grid.observed.index
+    observed_level = _series(grid.observed, station, "water_level", idx)
+    warn_thr, danger_thr = get_thresholds(station)
+
+    out: Dict[str, pd.Series] = {}
+    for horizon_h in horizons:
+        fut_max, fut_count = _future_window_stats(observed_level, horizon_h)
+        cov = fut_count / horizon_h
+        enough_cov = cov >= MIN_WINDOW_COVERAGE
+
+        exceed_warn = pd.Series(np.nan, index=idx)
+        exceed_warn[fut_max >= warn_thr] = 1.0
+        exceed_warn[enough_cov & exceed_warn.isna()] = 0.0
+
+        exceed_danger = pd.Series(np.nan, index=idx)
+        exceed_danger[fut_max >= danger_thr] = 1.0
+        exceed_danger[enough_cov & exceed_danger.isna()] = 0.0
+
+        max_level_valid = fut_max.where(enough_cov | (fut_max >= warn_thr))
+
+        out[f"max_level_{horizon_h}"] = max_level_valid
+        out[f"exceed_warn_{horizon_h}"] = exceed_warn
+        out[f"exceed_danger_{horizon_h}"] = exceed_danger
+
+    return pd.DataFrame(out, index=idx)
+
+
+# ---------------------------------------------------------------------------
+# Glue
+# ---------------------------------------------------------------------------
+
+
+def build_matrix(
+    df_long: pd.DataFrame,
+    station: str,
+    horizons: Tuple[int, ...] = (6, 12, 24),
+) -> Tuple[pd.DataFrame, pd.DataFrame, dict]:
+    """Build (X, Y, meta) training/inference matrices for one station."""
+    grid = make_hourly_grid(df_long)
+    X = build_features(grid, station)
+    Y = build_labels(grid, station, horizons)
+
+    keep = X["obs_age_h"].notna()
+    train_start = TRAIN_START.get(station)
+    if train_start:
+        keep &= X.index >= pd.Timestamp(train_start)
+
+    X = X.loc[keep]
+    Y = Y.loc[keep]
+
+    positive_counts = {
+        col: int(Y[col].sum()) for col in Y.columns if col.startswith("exceed_") and Y[col].notna().any()
+    }
+    meta = {
+        "station_code": station,
+        "n_rows": int(len(X)),
+        "span": (
+            (X.index.min().isoformat(), X.index.max().isoformat()) if len(X) else (None, None)
+        ),
+        "positive_counts": positive_counts,
+    }
+    return X, Y, meta

src/ml/predict.py

@@ -0,0 +1,298 @@
+"""Flood forecast inference.
+
+Integration contract (see get_forecasts / get_latest_forecasts): callers pass
+raw station readings, get back one forecast dict per station x horizon. A
+station with a stale, missing, or version-mismatched model transparently
+falls back to a simple persistence heuristic instead of raising -- this
+module must never crash the caller (e.g. the web API).
+"""
+
+import datetime
+import logging
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple, Union
+
+import joblib
+import numpy as np
+import pandas as pd
+
+from . import features
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_HORIZONS: Tuple[int, ...] = (6, 12, 24)
+STALE_AFTER_H = 6.0
+HEURISTIC_SIGMA = 0.3
+HEURISTIC_VERSION = "heuristic-v1"
+
+# Keyed by (path, mtime) so a retrained model (new mtime) invalidates the old entry.
+_MODEL_CACHE: Dict[Tuple[str, float], dict] = {}
+
+
+def _load_bundle(path: Path) -> dict:
+    # joblib.load runs arbitrary pickle code; safe here because `path` is always
+    # models/flood_{station}.joblib, an artifact this pipeline's own train.py wrote --
+    # never a user- or network-supplied file.
+    key = (str(path), path.stat().st_mtime)
+    cached = _MODEL_CACHE.get(key)
+    if cached is not None:
+        return cached
+    bundle = joblib.load(path)
+    for stale_key in [k for k in _MODEL_CACHE if k[0] == str(path)]:
+        del _MODEL_CACHE[stale_key]
+    _MODEL_CACHE[key] = bundle
+    return bundle
+
+
+def _readings_to_long_df(readings_by_station: Dict[str, List[dict]]) -> pd.DataFrame:
+    rows = []
+    for station_code, readings in readings_by_station.items():
+        for reading in readings:
+            timestamp = reading.get("timestamp")
+            if isinstance(timestamp, str):
+                timestamp = pd.to_datetime(timestamp)
+            rows.append(
+                {
+                    "timestamp": timestamp,
+                    "station_code": station_code,
+                    "water_level": reading.get("water_level"),
+                    "discharge": reading.get("discharge"),
+                }
+            )
+    if not rows:
+        return pd.DataFrame(columns=["timestamp", "station_code", "water_level", "discharge"])
+    df = pd.DataFrame(rows)
+    return df.dropna(subset=["timestamp"])
+
+
+def _clip_probability(value: float) -> float:
+    return float(min(max(value, 0.0), 1.0))
+
+
+def _sigmoid_probability(predicted_max: float, threshold: float, sigma: float) -> float:
+    return 1.0 / (1.0 + np.exp(-(predicted_max - threshold) / sigma))
+
+
+def _heuristic_forecast(
+    station_code: str,
+    as_of: pd.Timestamp,
+    current_level: float,
+    level_t_minus_3: Optional[float],
+    warn_thr: float,
+    danger_thr: float,
+    horizons: Tuple[int, ...],
+) -> List[dict]:
+    if level_t_minus_3 is None:
+        rate = 0.0
+    else:
+        rate = max(0.0, (current_level - level_t_minus_3) / 3.0)
+
+    results = []
+    for horizon_h in horizons:
+        predicted_max = max(current_level + rate * horizon_h * 0.7, current_level)
+        p_warning = _clip_probability(_sigmoid_probability(predicted_max, warn_thr, HEURISTIC_SIGMA))
+        p_danger = _clip_probability(_sigmoid_probability(predicted_max, danger_thr, HEURISTIC_SIGMA))
+        p_danger = min(p_danger, p_warning)
+        results.append(
+            {
+                "station_code": station_code,
+                "horizon_hours": horizon_h,
+                "p_warning": p_warning,
+                "p_danger": p_danger,
+                "predicted_max_level": predicted_max,
+                "current_level": current_level,
+                "as_of": as_of.isoformat(),
+                "model_version": HEURISTIC_VERSION,
+                "trained_at": None,
+                "source": "heuristic",
+                "threshold_warning": warn_thr,
+                "threshold_danger": danger_thr,
+            }
+        )
+    return results
+
+
+def _model_forecast(
+    station_code: str,
+    grid: features.HourlyGrid,
+    bundle: dict,
+    as_of: pd.Timestamp,
+    current_level: float,
+) -> List[dict]:
+    warn_thr = bundle["thresholds"]["warning"]
+    danger_thr = bundle["thresholds"]["danger"]
+
+    feature_row = features.build_features(grid, station_code).loc[[as_of]]
+    expected_columns = bundle["feature_names"]
+    missing = [c for c in expected_columns if c not in feature_row.columns]
+    if missing:
+        logger.error(f"Feature mismatch for {station_code} (missing {missing}); falling back to heuristic")
+        return None
+    feature_row = feature_row[expected_columns]
+
+    results = []
+    for horizon_h in bundle["horizons"]:
+        reg = bundle["heads"].get(f"max_{horizon_h}")
+        if reg is None:
+            results.append(None)
+            continue
+        predicted_max = max(float(reg.predict(feature_row)[0]), current_level)
+        sigma_h = bundle["sigma"].get(horizon_h, HEURISTIC_SIGMA)
+
+        warn_head = bundle["heads"].get(f"warn_{horizon_h}")
+        if warn_head is not None:
+            p_warning = float(warn_head.predict_proba(feature_row)[0][1])
+        else:
+            p_warning = _sigmoid_probability(predicted_max, warn_thr, sigma_h)
+
+        danger_head = bundle["heads"].get(f"danger_{horizon_h}")
+        if danger_head is not None:
+            p_danger = float(danger_head.predict_proba(feature_row)[0][1])
+        else:
+            p_danger = _sigmoid_probability(predicted_max, danger_thr, sigma_h)
+
+        p_warning = _clip_probability(p_warning)
+        p_danger = min(_clip_probability(p_danger), p_warning)
+
+        row = {
+            "station_code": station_code,
+            "horizon_hours": horizon_h,
+            "p_warning": p_warning,
+            "p_danger": p_danger,
+            "predicted_max_level": predicted_max,
+            "current_level": current_level,
+            "as_of": as_of.isoformat(),
+            "model_version": bundle["model_version"],
+            "trained_at": bundle["trained_at"],
+            "source": "model",
+            "threshold_warning": warn_thr,
+            "threshold_danger": danger_thr,
+        }
+        stages = features.FLOOD_STAGES.get(station_code)
+        if stages:
+            # Exceedance probability per official inundation stage, from the
+            # regression head and its validation-residual sigma. These are
+            # threshold-agnostic, so no retraining is needed to serve them.
+            row["stages"] = [
+                {
+                    "stage": s["stage"],
+                    "level": s["level"],
+                    "p_exceed": _clip_probability(
+                        _sigmoid_probability(predicted_max, s["level"], sigma_h)
+                    ),
+                }
+                for s in stages
+            ]
+        results.append(row)
+    return results
+
+
+def _forecast_station(
+    station_code: str,
+    grid: features.HourlyGrid,
+    models_dir: Path,
+    now: pd.Timestamp,
+    horizons: Tuple[int, ...],
+) -> List[dict]:
+    level_col = (station_code, "water_level")
+    if level_col not in grid.observed.columns:
+        logger.warning(f"No data for station {station_code}; omitting")
+        return []
+    observed_level = grid.observed[level_col].dropna()
+    if observed_level.empty:
+        logger.warning(f"No observed readings for station {station_code}; omitting")
+        return []
+
+    as_of = observed_level.index.max()
+    current_level = float(observed_level.loc[as_of])
+    staleness_h = (pd.Timestamp(now) - as_of).total_seconds() / 3600.0
+
+    warn_thr, danger_thr = features.get_thresholds(station_code)
+    t_minus_3 = as_of - pd.Timedelta(hours=3)
+    level_t_minus_3 = float(observed_level.loc[t_minus_3]) if t_minus_3 in observed_level.index else None
+
+    bundle_path = models_dir / f"flood_{station_code}.joblib"
+    if not bundle_path.exists() or staleness_h > STALE_AFTER_H:
+        return _heuristic_forecast(
+            station_code, as_of, current_level, level_t_minus_3, warn_thr, danger_thr, horizons
+        )
+
+    bundle = _load_bundle(bundle_path)
+    model_results = _model_forecast(station_code, grid, bundle, as_of, current_level)
+    if model_results is None:
+        return _heuristic_forecast(
+            station_code, as_of, current_level, level_t_minus_3, warn_thr, danger_thr, horizons
+        )
+
+    # Per-horizon heads that were skipped at train time (e.g. too few positives) still
+    # need a forecast row -- fall back to the single-horizon heuristic for just that row.
+    filled = []
+    for horizon_h, row in zip(bundle["horizons"], model_results):
+        if row is not None:
+            filled.append(row)
+        else:
+            filled.extend(
+                _heuristic_forecast(
+                    station_code, as_of, current_level, level_t_minus_3, warn_thr, danger_thr, (horizon_h,)
+                )
+            )
+    return filled
+
+
+def get_forecasts(
+    readings_by_station: Dict[str, List[dict]],
+    models_dir: Union[str, Path] = "models",
+    now: Optional[Union[datetime.datetime, str]] = None,
+) -> List[dict]:
+    """Produce flood forecasts for every station present in `readings_by_station`.
+
+    Each reading dict needs at least {timestamp, water_level, discharge}; extra
+    keys are ignored so raw API/DB rows can be passed straight through. At
+    least 96 hours of span is required to populate every feature; 336 hours
+    (14 days) is recommended.
+    """
+    models_dir = Path(models_dir)
+    if now is None:
+        now = datetime.datetime.now()
+    now = pd.Timestamp(now)
+
+    df_long = _readings_to_long_df(readings_by_station)
+    if df_long.empty:
+        return []
+
+    grid = features.make_hourly_grid(df_long)
+    results: List[dict] = []
+    for station_code in readings_by_station.keys():
+        try:
+            results.extend(_forecast_station(station_code, grid, models_dir, now, DEFAULT_HORIZONS))
+        except Exception as error:
+            logger.error(f"Forecast failed for station {station_code}: {error}")
+    return results
+
+
+def get_latest_forecasts(
+    db_url: Optional[str] = None,
+    models_dir: Union[str, Path] = "models",
+    hours: int = 336,
+) -> List[dict]:
+    """Convenience wrapper for web_api: load the latest window from the DB/API and forecast.
+
+    Raises FileNotFoundError when no trained model bundle exists at all, so the
+    API can 503 instead of serving purely heuristic output as if it were a forecast.
+    """
+    from .data import load_latest
+
+    if not sorted(Path(models_dir).glob("flood_*.joblib")):
+        raise FileNotFoundError(f"no trained model bundles in {models_dir}")
+
+    df_long = load_latest(db_url=db_url, hours=hours)
+    readings_by_station: Dict[str, List[dict]] = {}
+    if not df_long.empty:
+        for station_code, group in df_long.groupby("station_code"):
+            readings_by_station[station_code] = group[["timestamp", "water_level", "discharge"]].to_dict("records")
+
+    expected_stations = set(features.UPSTREAM_LEADS.keys())
+    for missing_station in expected_stations - set(readings_by_station.keys()):
+        logger.warning(f"No recent data for station {missing_station}; omitting from forecasts")
+
+    return get_forecasts(readings_by_station, models_dir=models_dir)

src/ml/train.py

@@ -0,0 +1,414 @@
+"""Training CLI for the Ping River flood forecast models.
+
+Per station: build the feature/label matrix once, evaluate with a strict
+temporal holdout (Split B), then refit each head on the full record for the
+deployed artifact. Hyperparameters are fixed (chosen via an earlier Split A
+sweep, not repeated here) -- no random search, no shuffling, no sklearn
+early_stopping (its internal validation split is random and would leak
+across time).
+"""
+
+import argparse
+import datetime
+import json
+import logging
+import subprocess
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+import joblib
+import numpy as np
+import pandas as pd
+import sklearn
+from sklearn.ensemble import HistGradientBoostingClassifier, HistGradientBoostingRegressor
+from sklearn.metrics import average_precision_score, brier_score_loss, mean_absolute_error, mean_squared_error
+
+from . import features
+from .data import DEFAULT_API_URL, load_measurements, resolve_db_url
+
+logger = logging.getLogger(__name__)
+
+HORIZONS: Tuple[int, ...] = (6, 12, 24)
+SPLIT_B_TRAIN_END = "2024-12-31"
+SPLIT_B_TEST_START = "2025-01-01"
+SPLIT_B_TEST_END = "2026-08-10"
+MIN_POSITIVES_FOR_CLASSIFIER = 30
+MIN_SIGMA = 0.15
+MIN_ROWS_TO_TRAIN = 200
+MIN_ROWS_FOR_HEAD = 50
+
+HGB_PARAMS = {
+    "max_iter": 300,
+    "learning_rate": 0.06,
+    "max_leaf_nodes": 31,
+    "min_samples_leaf": 50,
+    "l2_regularization": 1.0,
+    "early_stopping": False,
+    "random_state": 42,
+}
+
+
+def _git_short_sha() -> str:
+    try:
+        result = subprocess.run(
+            ["git", "rev-parse", "--short", "HEAD"], capture_output=True, text=True, timeout=5, check=True
+        )
+        sha = result.stdout.strip()
+        return sha or "nogit"
+    except Exception:
+        return "nogit"
+
+
+def _make_regressor(overrides: Optional[dict] = None) -> HistGradientBoostingRegressor:
+    params = {**HGB_PARAMS, **(overrides or {})}
+    return HistGradientBoostingRegressor(loss="squared_error", **params)
+
+
+def _make_classifier(overrides: Optional[dict] = None) -> HistGradientBoostingClassifier:
+    params = {**HGB_PARAMS, **(overrides or {})}
+    return HistGradientBoostingClassifier(**params)
+
+
+def _safe_fit(estimator, X: pd.DataFrame, y: pd.Series, head_key: str, skipped_heads: Dict[str, str]):
+    """Fit an estimator, converting any failure (e.g. HistGradientBoosting's binning
+    step rejecting an all-NaN/constant feature column) into a recorded skip rather
+    than a station-killing exception."""
+    try:
+        estimator.fit(X, y)
+        return estimator
+    except Exception as error:
+        skipped_heads[head_key] = f"fit failed: {error}"
+        logger.warning(f"{head_key}: fit failed, skipping ({error})")
+        return None
+
+
+def _recall_at_far(y_true: np.ndarray, y_score: np.ndarray, target_far: float) -> Optional[float]:
+    """Recall at the score threshold whose false-positive rate over true negatives is <= target_far."""
+    y_true = np.asarray(y_true)
+    y_score = np.asarray(y_score)
+    neg_scores = np.sort(y_score[y_true == 0])[::-1]
+    n_pos = int((y_true == 1).sum())
+    n_neg = len(neg_scores)
+    if n_pos == 0 or n_neg == 0:
+        return None
+    k = int(np.floor(target_far * n_neg))
+    threshold = neg_scores[k - 1] if k > 0 else neg_scores[0] + 1e-9
+    predicted_positive = y_score >= threshold
+    tp = int(np.sum(predicted_positive & (y_true == 1)))
+    return tp / n_pos
+
+
+def _p_warning_series(head, reg, X: pd.DataFrame, threshold: float, sigma: float) -> pd.Series:
+    """Model score if a classifier head exists, else the sigmoid-derived fallback probability."""
+    if head is not None:
+        return pd.Series(head.predict_proba(X)[:, 1], index=X.index)
+    predicted_max = pd.Series(reg.predict(X), index=X.index)
+    return 1.0 / (1.0 + np.exp(-(predicted_max - threshold) / sigma))
+
+
+def _find_events(observed_level: pd.Series, warn_thr: float) -> List[dict]:
+    """Group contiguous observed hours >= warn_thr into flood events."""
+    above = observed_level >= warn_thr
+    events: List[dict] = []
+    start = None
+    prev_t = None
+    for t, is_above in above.items():
+        if is_above and start is None:
+            start = t
+        elif not is_above and start is not None:
+            window = observed_level.loc[start:prev_t]
+            events.append({"crossed_warn_at": start, "peak_time": window.idxmax(), "peak_level": float(window.max())})
+            start = None
+        prev_t = t
+    if start is not None:
+        window = observed_level.loc[start:]
+        events.append({"crossed_warn_at": start, "peak_time": window.idxmax(), "peak_level": float(window.max())})
+    return events
+
+
+def _first_alert_at(p_series: pd.Series, crossed_at, lookback_h: int = 48):
+    """Earliest time p_warning was sustained (>=0.5 for 2 consecutive hours) within the prior lookback_h."""
+    window = p_series.loc[crossed_at - pd.Timedelta(hours=lookback_h) : crossed_at]
+    sustained = (window >= 0.5) & (window.shift(1) >= 0.5)
+    hits = sustained[sustained].index
+    if len(hits) == 0:
+        return None
+    return hits.min() - pd.Timedelta(hours=1)
+
+
+def _events_with_lead_time(
+    observed_level_test: pd.Series, warn_thr: float, p_warning_test: pd.Series
+) -> List[dict]:
+    events = _find_events(observed_level_test, warn_thr)
+    for event in events:
+        first_alert_at = _first_alert_at(p_warning_test, event["crossed_warn_at"])
+        event["first_alert_at"] = first_alert_at.isoformat() if first_alert_at is not None else None
+        if first_alert_at is not None:
+            lead_hours = (event["crossed_warn_at"] - first_alert_at).total_seconds() / 3600.0
+        else:
+            lead_hours = None
+        event["lead_hours"] = lead_hours
+        event["crossed_warn_at"] = event["crossed_warn_at"].isoformat()
+        event["peak_time"] = event["peak_time"].isoformat()
+    return events
+
+
+def train_station(
+    df_long: pd.DataFrame,
+    station: str,
+    horizons: Tuple[int, ...] = HORIZONS,
+    skip_eval: bool = False,
+    hgb_overrides: Optional[dict] = None,
+    split_train_end: str = SPLIT_B_TRAIN_END,
+    split_test_start: str = SPLIT_B_TEST_START,
+    split_test_end: str = SPLIT_B_TEST_END,
+) -> Tuple[Optional[dict], dict]:
+    """Train every head for one station. Returns (bundle_or_None, station_metrics)."""
+    X, Y, meta = features.build_matrix(df_long, station, horizons)
+    if meta["n_rows"] < MIN_ROWS_TO_TRAIN:
+        return None, {"status": "failed", "reason": f"only {meta['n_rows']} usable rows (< {MIN_ROWS_TO_TRAIN})"}
+
+    warn_thr, danger_thr = features.get_thresholds(station)
+    feature_names = list(X.columns)
+
+    if skip_eval:
+        train_mask = pd.Series(True, index=X.index)
+        test_mask = pd.Series(False, index=X.index)
+    else:
+        train_mask = X.index <= pd.Timestamp(split_train_end)
+        test_mask = (X.index >= pd.Timestamp(split_test_start)) & (X.index <= pd.Timestamp(split_test_end))
+    X_train, Y_train = X.loc[train_mask], Y.loc[train_mask]
+    X_test, Y_test = X.loc[test_mask], Y.loc[test_mask]
+    eval_X, eval_Y = (X, Y) if skip_eval else (X_train, Y_train)
+
+    heads: Dict[str, object] = {}
+    sigma: Dict[int, float] = {}
+    skipped_heads: Dict[str, str] = {}
+    per_horizon: Dict[int, dict] = {}
+    observed_grid = features.make_hourly_grid(df_long).observed
+
+    for h in horizons:
+        max_col, warn_col, danger_col = f"max_level_{h}", f"exceed_warn_{h}", f"exceed_danger_{h}"
+        horizon_metrics: dict = {}
+
+        # --- regression head (max level) ---
+        reg_labeled = eval_Y[max_col].notna()
+        reg = None
+        if reg_labeled.sum() >= MIN_ROWS_FOR_HEAD:
+            reg = _safe_fit(
+                _make_regressor(hgb_overrides),
+                eval_X.loc[reg_labeled],
+                eval_Y.loc[reg_labeled, max_col],
+                f"max_{h}",
+                skipped_heads,
+            )
+        else:
+            skipped_heads[f"max_{h}"] = f"only {int(reg_labeled.sum())} labeled rows"
+
+        sigma_h = MIN_SIGMA
+        if reg is not None and not skip_eval:
+            test_labeled = Y_test[max_col].notna()
+            if test_labeled.sum() > 0:
+                y_true = Y_test.loc[test_labeled, max_col]
+                y_pred = reg.predict(X_test.loc[test_labeled])
+                residuals = y_true.to_numpy() - y_pred
+                sigma_h = max(float(np.std(residuals)), MIN_SIGMA)
+                horizon_metrics["n_test"] = int(test_labeled.sum())
+                horizon_metrics["mae"] = float(mean_absolute_error(y_true, y_pred))
+                horizon_metrics["rmse"] = float(np.sqrt(mean_squared_error(y_true, y_pred)))
+                above_2m = y_true >= 2.0
+                horizon_metrics["mae_above_2m"] = (
+                    float(mean_absolute_error(y_true[above_2m], y_pred[above_2m])) if above_2m.any() else None
+                )
+        sigma[h] = sigma_h
+        horizon_metrics["sigma"] = sigma_h
+
+        # --- classification heads (warn / danger) ---
+        p_warning_test = None
+        for label_name, col, thr in (("warn", warn_col, warn_thr), ("danger", danger_col, danger_thr)):
+            train_labeled = eval_Y[col].notna()
+            n_pos = int(eval_Y.loc[train_labeled, col].sum()) if train_labeled.any() else 0
+            head_key = f"{label_name}_{h}"
+            clf = None
+            if n_pos >= MIN_POSITIVES_FOR_CLASSIFIER:
+                clf = _safe_fit(
+                    _make_classifier(hgb_overrides),
+                    eval_X.loc[train_labeled],
+                    eval_Y.loc[train_labeled, col],
+                    head_key,
+                    skipped_heads,
+                )
+            else:
+                skipped_heads[head_key] = f"only {n_pos} positives in train span (< {MIN_POSITIVES_FOR_CLASSIFIER})"
+            heads[head_key] = clf
+
+            if not skip_eval:
+                test_labeled = Y_test[col].notna()
+                horizon_metrics[f"base_rate_{label_name}"] = (
+                    float(Y_test.loc[test_labeled, col].mean()) if test_labeled.any() else None
+                )
+                if clf is not None and test_labeled.sum() > 0 and Y_test.loc[test_labeled, col].nunique() > 1:
+                    y_true = Y_test.loc[test_labeled, col]
+                    y_score = clf.predict_proba(X_test.loc[test_labeled])[:, 1]
+                    horizon_metrics[f"pr_auc_{label_name}"] = float(average_precision_score(y_true, y_score))
+                    horizon_metrics[f"brier_{label_name}"] = float(brier_score_loss(y_true, y_score))
+                    horizon_metrics[f"recall_{label_name}_at_far1pct"] = _recall_at_far(y_true, y_score, 0.01)
+                    horizon_metrics[f"recall_{label_name}_at_far5pct"] = _recall_at_far(y_true, y_score, 0.05)
+                else:
+                    horizon_metrics[f"pr_auc_{label_name}"] = None
+                    horizon_metrics[f"brier_{label_name}"] = None
+                    horizon_metrics[f"recall_{label_name}_at_far1pct"] = None
+                    horizon_metrics[f"recall_{label_name}_at_far5pct"] = None
+
+            if label_name == "warn" and not skip_eval and reg is not None:
+                p_warning_test = _p_warning_series(clf, reg, X_test, thr, sigma_h)
+
+        per_horizon[h] = horizon_metrics
+        heads[f"max_{h}"] = reg
+
+        if not skip_eval and reg is not None and p_warning_test is not None:
+            observed_test_level = observed_grid.get((station, "water_level"))
+            if observed_test_level is not None:
+                observed_test_level = observed_test_level.loc[observed_test_level.index.isin(X_test.index)]
+                per_horizon[h]["events"] = _events_with_lead_time(observed_test_level, warn_thr, p_warning_test)
+
+    # --- full refit on the ENTIRE record for the deployed artifact ---
+    # This may include/exclude different heads than the eval-phase gate above (the
+    # full record has more labeled rows), so skip reasons are re-derived here --
+    # skipped_heads must reflect what actually ends up in the saved bundle.
+    final_heads: Dict[str, object] = {}
+    for h in horizons:
+        max_col, warn_col, danger_col = f"max_level_{h}", f"exceed_warn_{h}", f"exceed_danger_{h}"
+        head_key = f"max_{h}"
+        labeled = Y[max_col].notna()
+        if labeled.sum() >= MIN_ROWS_FOR_HEAD:
+            reg = _safe_fit(_make_regressor(hgb_overrides), X.loc[labeled], Y.loc[labeled, max_col], head_key, skipped_heads)
+            final_heads[head_key] = reg
+            if reg is not None:
+                skipped_heads.pop(head_key, None)
+        else:
+            skipped_heads[head_key] = f"only {int(labeled.sum())} labeled rows"
+            final_heads[head_key] = None
+
+        for label_name, col in (("warn", warn_col), ("danger", danger_col)):
+            head_key = f"{label_name}_{h}"
+            train_labeled = Y[col].notna()
+            n_pos = int(Y.loc[train_labeled, col].sum()) if train_labeled.any() else 0
+            if n_pos >= MIN_POSITIVES_FOR_CLASSIFIER:
+                clf = _safe_fit(
+                    _make_classifier(hgb_overrides), X.loc[train_labeled], Y.loc[train_labeled, col], head_key, skipped_heads
+                )
+                final_heads[head_key] = clf
+                if clf is not None:
+                    skipped_heads.pop(head_key, None)
+            else:
+                skipped_heads[head_key] = f"only {n_pos} positives in train span (< {MIN_POSITIVES_FOR_CLASSIFIER})"
+                final_heads[head_key] = None
+
+    bundle = {
+        "station_code": station,
+        "model_version": f"hgb-v1+{_git_short_sha()}",
+        "trained_at": datetime.datetime.now().isoformat(),
+        "sklearn_version": sklearn.__version__,
+        "feature_names": feature_names,
+        "horizons": list(horizons),
+        "thresholds": {"warning": warn_thr, "danger": danger_thr},
+        "heads": final_heads,
+        "sigma": sigma,
+        "skipped_heads": skipped_heads,
+        "train_span": meta["span"],
+        "n_train_rows": meta["n_rows"],
+    }
+    station_metrics = {"status": "trained", "per_horizon": per_horizon}
+    return bundle, station_metrics
+
+
+def train_all(
+    df_long: pd.DataFrame,
+    stations: List[str],
+    horizons: Tuple[int, ...] = HORIZONS,
+    models_dir: Path = Path("models"),
+    skip_eval: bool = False,
+    hgb_overrides: Optional[dict] = None,
+) -> dict:
+    """Train and save every requested station's models. Returns the metrics.json payload."""
+    models_dir = Path(models_dir)
+    models_dir.mkdir(parents=True, exist_ok=True)
+    model_version = f"hgb-v1+{_git_short_sha()}"
+
+    station_results: Dict[str, dict] = {}
+    for station in stations:
+        if station in features.NOT_TRAINABLE:
+            reason = features.NOT_TRAINABLE[station]
+            logger.info(f"{station}: heuristic ({reason})")
+            station_results[station] = {"status": "heuristic", "reason": reason}
+            continue
+        try:
+            bundle, station_metrics = train_station(
+                df_long, station, horizons, skip_eval=skip_eval, hgb_overrides=hgb_overrides
+            )
+            if bundle is None:
+                logger.warning(f"{station}: failed ({station_metrics.get('reason')})")
+                station_results[station] = station_metrics
+                continue
+            joblib.dump(bundle, models_dir / f"flood_{station}.joblib")
+            logger.info(
+                f"{station}: trained, {bundle['n_train_rows']} rows, "
+                f"{len(bundle['skipped_heads'])} heads skipped"
+            )
+            station_results[station] = station_metrics
+        except Exception as error:
+            logger.error(f"{station}: failed with exception: {error}")
+            station_results[station] = {"status": "failed", "reason": str(error)}
+
+    metrics_payload = {
+        "generated_at": datetime.datetime.now().isoformat(),
+        "model_version": model_version,
+        "split": {
+            "train_end": SPLIT_B_TRAIN_END,
+            "test_start": SPLIT_B_TEST_START,
+            "test_end": SPLIT_B_TEST_END,
+        },
+        "stations": station_results,
+    }
+    with open(models_dir / "metrics.json", "w", encoding="utf-8") as handle:
+        json.dump(metrics_payload, handle, indent=2, default=str)
+    return metrics_payload
+
+
+def main(argv: Optional[List[str]] = None) -> None:
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
+
+    parser = argparse.ArgumentParser(description="Train Ping River flood forecast models")
+    parser.add_argument("--stations", default="all", help="'all' or a comma-separated list of station codes")
+    parser.add_argument("--models-dir", default="models")
+    parser.add_argument("--db-url", default=None)
+    parser.add_argument("--api-url", default=DEFAULT_API_URL)
+    parser.add_argument("--skip-eval", action="store_true", help="Refit-only fast path; skip Split B evaluation")
+    parser.add_argument("--start", default=None, help="ISO date; earliest measurement to load")
+    parser.add_argument("--end", default=None, help="ISO date; latest measurement to load")
+    args = parser.parse_args(argv)
+
+    if args.stations == "all":
+        stations = list(features.UPSTREAM_LEADS.keys())
+    else:
+        stations = [s.strip() for s in args.stations.split(",") if s.strip()]
+
+    start = datetime.datetime.fromisoformat(args.start) if args.start else None
+    end = datetime.datetime.fromisoformat(args.end) if args.end else None
+
+    logger.info(f"Loading measurements for {len(stations)} stations...")
+    df_long = load_measurements(
+        db_url=resolve_db_url(args.db_url), stations=None, start=start, end=end, api_url=args.api_url
+    )
+    logger.info(f"Loaded {len(df_long)} rows spanning {df_long['timestamp'].min()} .. {df_long['timestamp'].max()}")
+
+    metrics_payload = train_all(
+        df_long, stations, models_dir=Path(args.models_dir), skip_eval=args.skip_eval
+    )
+    trained = sum(1 for s in metrics_payload["stations"].values() if s["status"] == "trained")
+    logger.info(f"Done: {trained}/{len(stations)} stations trained. metrics.json written to {args.models_dir}")
+
+
+if __name__ == "__main__":
+    main()

src/postgres_history.py

@@ -0,0 +1,96 @@
+"""Read historical station measurements from PostgreSQL."""
+
+import datetime
+import os
+from typing import Dict, List, Optional, Tuple
+
+from sqlalchemy import create_engine, text
+
+
+# Stage-discharge rating curves: Q = a * (H - b)^c
+# Key: station_code, Value: (a, b, c)
+# Use linear fallback Q = slope * H if a curve is not defined.
+_RATING_CURVES: Dict[str, Tuple[float, float, float]] = {}
+_DEFAULT_LINEAR_SLOPE = 20.0  # m^3/s per meter
+
+
+def _calculate_discharge(water_level: Optional[float], station_code: str = None) -> Optional[float]:
+    """Estimate discharge from water level using a rating curve or linear fallback."""
+    if water_level is None:
+        return None
+
+    curve = _RATING_CURVES.get(station_code)
+    if curve:
+        a, b, c = curve
+        h_excess = water_level - b
+        if h_excess <= 0:
+            return 0.0
+        return round(a * (h_excess ** c), 2)
+    
+    # Linear fallback: Q = slope * H
+    return round(_DEFAULT_LINEAR_SLOPE * water_level, 2)
+
+
+class PostgresHistory:
+    def __init__(self, connection_string: Optional[str] = None, engine=None):
+        connection_string = connection_string or os.getenv("POSTGRES_CONNECTION_STRING")
+        if engine is None and not connection_string:
+            raise RuntimeError("POSTGRES_CONNECTION_STRING is not configured")
+        self.engine = engine or create_engine(connection_string, pool_pre_ping=True)
+
+    def station_history(
+        self,
+        station_code: str,
+        start: datetime.datetime,
+        end: datetime.datetime,
+        limit: int = 2000,
+    ) -> List[Dict]:
+        if not 1 <= limit <= 100000:
+            raise ValueError("limit must be between 1 and 100000")
+        if start >= end:
+            raise ValueError("start must be before end")
+
+        query = text(
+            """
+            SELECT m.timestamp, s.station_code, m.water_level,
+                   m.discharge, m.discharge_percent
+            FROM water_measurements m
+            JOIN stations s ON m.station_id = s.id
+            WHERE s.station_code = :station_code
+              AND m.timestamp >= :start_time
+              AND m.timestamp <= :end_time
+            ORDER BY m.timestamp ASC
+            LIMIT :limit
+            """
+        )
+        with self.engine.connect() as connection:
+            rows = connection.execute(
+                query,
+                {
+                    "station_code": station_code,
+                    "start_time": start,
+                    "end_time": end,
+                    "limit": limit,
+                },
+            )
+            result = []
+            for row in rows:
+                timestamp = row[0]
+                if isinstance(timestamp, str):
+                    timestamp = datetime.datetime.fromisoformat(timestamp)
+                station_code = row[1]
+                water_level = float(row[2]) if row[2] is not None else None
+                discharge = float(row[3]) if row[3] is not None else None
+                # Estimate discharge from water level if DB value is missing
+                if discharge is None and water_level is not None:
+                    discharge = _calculate_discharge(water_level, station_code)
+                result.append(
+                    {
+                        "timestamp": timestamp,
+                        "station_code": station_code,
+                        "water_level": water_level,
+                        "discharge": discharge,
+                        "discharge_percent": float(row[4]) if row[4] is not None else None,
+                    }
+                )
+            return result

src/schemas.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+"""Pydantic request/response schemas for the water monitoring web API."""
+
+from datetime import datetime
+from typing import Any, Dict, Optional
+
+from pydantic import BaseModel, Field
+
+
+class StationResponse(BaseModel):
+    station_id: int
+    station_code: str
+    thai_name: str
+    english_name: str
+    latitude: Optional[float] = None
+    longitude: Optional[float] = None
+    geohash: Optional[str] = None
+    status: str = "active"
+
+
+class StationCreateModel(BaseModel):
+    station_code: str = Field(..., description="Station code (e.g., P.1, P.20)")
+    thai_name: str = Field(..., description="Thai name of the station")
+    english_name: str = Field(..., description="English name of the station")
+    latitude: Optional[float] = Field(None, ge=-90, le=90, description="Latitude coordinate")
+    longitude: Optional[float] = Field(None, ge=-180, le=180, description="Longitude coordinate")
+    geohash: Optional[str] = Field(None, description="Geohash for the location")
+    status: str = Field("active", description="Station status")
+
+
+class StationUpdateModel(BaseModel):
+    thai_name: Optional[str] = Field(None, description="Thai name of the station")
+    english_name: Optional[str] = Field(None, description="English name of the station")
+    latitude: Optional[float] = Field(None, ge=-90, le=90, description="Latitude coordinate")
+    longitude: Optional[float] = Field(None, ge=-180, le=180, description="Longitude coordinate")
+    geohash: Optional[str] = Field(None, description="Geohash for the location")
+    status: Optional[str] = Field(None, description="Station status")
+
+
+class MeasurementResponse(BaseModel):
+    timestamp: datetime
+    station_code: str
+    station_name_en: str
+    station_name_th: str
+    water_level: float
+    discharge: Optional[float] = None
+    discharge_percent: Optional[float] = None
+    status: str = "active"
+
+
+class HealthResponse(BaseModel):
+    overall_status: str
+    timestamp: str
+    checks: Dict[str, Dict[str, Any]]
+
+
+class MetricsResponse(BaseModel):
+    counters: Dict[str, float]
+    gauges: Dict[str, float]
+    histograms: Dict[str, Dict[str, float]]
+
+
+class ScrapingStatusResponse(BaseModel):
+    is_running: bool
+    last_run: Optional[datetime] = None
+    next_run: Optional[datetime] = None
+    total_runs: int = 0
+    successful_runs: int = 0
+    failed_runs: int = 0

src/static/dashboard.html

@@ -0,0 +1,709 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Ping River Live Monitor</title>
+    <link rel="preconnect" href="https://unpkg.com">
+    <link rel="preconnect" href="https://tile.openstreetmap.org">
+    <link rel="stylesheet" href="https://unpkg.com/leaflet@1.9.4/dist/leaflet.css" integrity="sha384-sHL9NAb7lN7rfvG5lfHpm643Xkcjzp4jFvuavGOndn6pjVqS6ny56CAt3nsEVT4H" crossorigin="anonymous">
+    <style>
+        :root {
+            --ink: #132b35;
+            --muted: #64777d;
+            --paper: #f3f7f5;
+            --card: #ffffff;
+            --river: #087da5;
+            --river-light: #38b4d5;
+            --mint: #dff4e8;
+            --green: #1e8b60;
+            --amber: #d99018;
+            --red: #cc4b37;
+            --border: #dce7e3;
+            --shadow: 0 16px 40px rgba(23, 57, 67, .10);
+        }
+        * { box-sizing: border-box; }
+        html, body { margin: 0; min-height: 100%; }
+        body {
+            color: var(--ink);
+            background:
+                radial-gradient(circle at 8% 0%, rgba(56, 180, 213, .12), transparent 25rem),
+                var(--paper);
+            font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
+        }
+        .shell { max-width: 1500px; margin: 0 auto; padding: 24px; }
+        header {
+            display: flex; align-items: center; justify-content: space-between; gap: 20px;
+            margin-bottom: 20px;
+        }
+        .brand { display: flex; align-items: center; gap: 14px; }
+        .brand-mark {
+            width: 48px; height: 48px; border-radius: 15px; display: grid; place-items: center;
+            color: white; font-size: 25px; background: linear-gradient(145deg, #0a91b9, #076787);
+            box-shadow: 0 10px 22px rgba(8, 125, 165, .25);
+        }
+        h1 { margin: 0; font-size: clamp(1.4rem, 2.5vw, 2rem); letter-spacing: -.035em; }
+        .subtitle { margin: 4px 0 0; color: var(--muted); font-size: .92rem; }
+        .header-actions { display: flex; align-items: center; gap: 12px; }
+        .live-pill {
+            display: flex; gap: 8px; align-items: center; padding: 9px 13px; border-radius: 999px;
+            background: var(--mint); color: #146644; font-weight: 750; font-size: .8rem;
+        }
+        .live-dot { width: 8px; height: 8px; border-radius: 50%; background: #22a66c; box-shadow: 0 0 0 5px rgba(34,166,108,.12); }
+        button {
+            border: 1px solid var(--border); border-radius: 11px; background: white; color: var(--ink);
+            padding: 10px 14px; cursor: pointer; font-weight: 700; box-shadow: 0 3px 10px rgba(22,52,62,.05);
+        }
+        button:hover { border-color: #a9c4bb; transform: translateY(-1px); }
+        button:disabled { opacity: .55; cursor: wait; transform: none; }
+        .stats { display: grid; grid-template-columns: repeat(4, minmax(0,1fr)); gap: 14px; margin-bottom: 14px; }
+        .stat {
+            min-height: 112px; padding: 18px; background: var(--card); border: 1px solid var(--border);
+            border-radius: 17px; box-shadow: 0 6px 18px rgba(31,61,70,.045);
+        }
+        .stat-label { color: var(--muted); text-transform: uppercase; letter-spacing: .09em; font-size: .68rem; font-weight: 800; }
+        .stat-value { margin-top: 9px; font-size: 1.65rem; font-weight: 800; letter-spacing: -.04em; white-space: nowrap; }
+        .stat-note { color: var(--muted); margin-top: 3px; font-size: .77rem; }
+        .workspace { display: grid; grid-template-columns: minmax(0, 1fr) 330px; gap: 14px; min-height: 640px; }
+        .map-card, .side-card { background: var(--card); border: 1px solid var(--border); border-radius: 19px; box-shadow: var(--shadow); overflow: hidden; }
+        .map-card { position: relative; }
+        #station-map { height: 640px; width: 100%; background: #dcebea; }
+        .map-overlay {
+            position: absolute; z-index: 500; top: 16px; left: 52px; right: 16px;
+            display: flex; justify-content: space-between; align-items: flex-start; pointer-events: none;
+        }
+        .map-heading, .legend {
+            background: rgba(255,255,255,.93); backdrop-filter: blur(9px); border: 1px solid rgba(207,224,218,.9);
+            border-radius: 13px; padding: 11px 13px; box-shadow: 0 7px 20px rgba(22,58,68,.12);
+        }
+        .map-heading strong { display: block; font-size: .9rem; }
+        .map-heading span { color: var(--muted); font-size: .72rem; }
+        .legend { font-size: .7rem; color: var(--muted); }
+        .legend-title { color: var(--ink); font-weight: 800; margin-bottom: 7px; }
+        .legend-row { display: flex; align-items: center; gap: 6px; margin: 5px 0; }
+        .swatch { width: 9px; height: 9px; border-radius: 50%; }
+        .side-card { display: flex; flex-direction: column; max-height: 640px; }
+        .side-head { padding: 18px 18px 14px; border-bottom: 1px solid var(--border); }
+        .side-head h2 { margin: 0; font-size: 1rem; }
+        .side-head p { margin: 5px 0 0; color: var(--muted); font-size: .75rem; }
+        .station-list { overflow-y: auto; padding: 7px; }
+        .station-row {
+            width: 100%; display: grid; grid-template-columns: 40px minmax(0,1fr) auto; gap: 10px; align-items: center;
+            padding: 11px; border: 0; border-radius: 12px; box-shadow: none; text-align: left; background: transparent;
+        }
+        .station-row:hover { background: #f1f7f5; transform: none; }
+        .station-code { width: 40px; height: 40px; display: grid; place-items: center; border-radius: 11px; color: white; font-size: .68rem; font-weight: 850; }
+        .station-name { overflow: hidden; }
+        .station-name strong, .station-name span { display: block; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
+        .station-name strong { font-size: .78rem; }
+        .station-name span { color: var(--muted); font-size: .68rem; margin-top: 3px; }
+        .flow-value { text-align: right; font-size: .82rem; font-weight: 800; }
+        .flow-value span { display: block; color: var(--muted); font-size: .61rem; font-weight: 650; margin-top: 2px; }
+        .loading-panel, .error-panel { position: absolute; z-index: 600; inset: 0; display: grid; place-items: center; background: rgba(243,247,245,.88); }
+        .loading-card { background: white; padding: 18px 22px; border-radius: 14px; box-shadow: var(--shadow); font-weight: 750; }
+        .error-panel { display: none; color: #8d2f22; text-align: center; padding: 25px; }
+        .marker-wrap { background: none; border: 0; }
+        .flow-marker {
+            --marker-color: #087da5; --marker-size: 26px;
+            position: relative; width: var(--marker-size); height: var(--marker-size); display: grid; place-items: center;
+            border-radius: 50%; background: var(--marker-color); color: white; border: 3px solid white;
+            box-shadow: 0 4px 12px rgba(5,43,58,.35); font-size: 8px; font-weight: 900;
+        }
+        .flow-marker::before {
+            content: ""; position: absolute; inset: -6px; border-radius: 50%; border: 2px solid var(--marker-color);
+            opacity: .36; animation: pulse 2.2s ease-out infinite;
+        }
+        @keyframes pulse { 0% { transform: scale(.72); opacity: .55; } 75%,100% { transform: scale(1.35); opacity: 0; } }
+        .flow-line { animation: riverMove 3s linear infinite; }
+        .flow-idle { animation-duration: 5.5s; }
+        .flow-slow { animation-duration: 3s; }
+        .flow-med { animation-duration: 1.9s; }
+        .flow-fast { animation-duration: 1.15s; }
+        .flow-surge { animation-duration: .7s; }
+        @keyframes riverMove { to { stroke-dashoffset: -40; } }
+        @media (prefers-reduced-motion: reduce) { .flow-line, .flow-marker::before { animation: none; } }
+        .line-swatch { width: 24px; height: 4px; border-radius: 2px; flex: none; }
+        .forecast-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(215px, 1fr)); gap: 10px; margin-top: 14px; }
+        .forecast-station { border: 1px solid var(--border); border-radius: 12px; padding: 10px 12px; }
+        .forecast-station strong { font-size: .8rem; }
+        .forecast-station .fc-name { color: var(--muted); font-size: .68rem; margin: 2px 0 8px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
+        .risk-chips { display: flex; gap: 6px; }
+        .risk-chip { flex: 1; text-align: center; border-radius: 8px; padding: 5px 4px; font-size: .64rem; font-weight: 800; color: white; }
+        .risk-chip span { display: block; font-weight: 650; font-size: .58rem; opacity: .85; }
+        .p1-outlook { border: 1px solid var(--border); border-left: 4px solid var(--river); border-radius: 12px; padding: 12px 14px; margin-top: 14px; background: #f7fbfa; }
+        .p1-outlook-head { display: flex; justify-content: space-between; align-items: center; gap: 10px; flex-wrap: wrap; }
+        .p1-outlook-head strong { font-size: .88rem; }
+        .p1-peak { color: var(--muted); font-size: .76rem; }
+        .stage-strip { display: flex; gap: 6px; flex-wrap: wrap; margin-top: 10px; }
+        .stage-chip { min-width: 74px; text-align: center; border-radius: 9px; padding: 6px 8px; font-size: .7rem; font-weight: 800; color: white; }
+        .stage-chip small { display: block; font-weight: 650; font-size: .6rem; opacity: .88; }
+        .zones-button { font-size: .72rem; padding: 7px 11px; }
+        .leaflet-popup-content-wrapper { border-radius: 14px; box-shadow: 0 12px 35px rgba(14,45,54,.2); }
+        .popup { min-width: 190px; }
+        .popup-code { font-size: .7rem; color: var(--river); font-weight: 850; text-transform: uppercase; letter-spacing: .08em; }
+        .popup h3 { margin: 4px 0 2px; font-size: 1rem; }
+        .popup-th { color: var(--muted); font-size: .74rem; }
+        .popup-grid { display: grid; grid-template-columns: 1fr 1fr; gap: 8px; margin-top: 12px; }
+        .popup-metric { background: #f1f7f5; padding: 8px; border-radius: 9px; }
+        .popup-metric span { display: block; color: var(--muted); font-size: .62rem; }
+        .popup-metric strong { display: block; margin-top: 2px; font-size: .86rem; }
+        .popup-time { margin-top: 9px; color: var(--muted); font-size: .64rem; }
+        @media (max-width: 900px) {
+            .stats { grid-template-columns: repeat(2, 1fr); }
+            .workspace { grid-template-columns: 1fr; }
+            .side-card { max-height: 400px; }
+        }
+        @media (max-width: 560px) {
+            .shell { padding: 14px; }
+            header { align-items: flex-start; }
+            .subtitle, .live-pill { display: none; }
+            .stats { gap: 8px; }
+            .stat { min-height: 96px; padding: 14px; }
+            .stat-value { font-size: 1.25rem; }
+            #station-map { height: 540px; }
+            .map-overlay { left: 46px; }
+            .legend { display: none; }
+        }
+    </style>
+</head>
+<body>
+<main class="shell">
+    <header>
+        <div class="brand">
+            <div class="brand-mark">≋</div>
+            <div>
+                <h1>Ping River Live Monitor</h1>
+                <p class="subtitle">Current water level and discharge across Northern Thailand</p>
+            </div>
+        </div>
+        <div class="header-actions">
+            <div class="live-pill"><span class="live-dot"></span> LIVE DATA</div>
+            <button id="refresh-button" type="button">↻ Refresh</button>
+        </div>
+    </header>
+
+    <section class="stats" aria-label="River summary">
+        <article class="stat"><div class="stat-label">Reporting stations</div><div class="stat-value" id="station-count">—</div><div class="stat-note">with current readings</div></article>
+        <article class="stat"><div class="stat-label">Combined discharge</div><div class="stat-value" id="total-flow">—</div><div class="stat-note">sum of reported flows · m³/s</div></article>
+        <article class="stat"><div class="stat-label">Strongest flow</div><div class="stat-value" id="peak-flow">—</div><div class="stat-note" id="peak-station">Awaiting station data</div></article>
+        <article class="stat"><div class="stat-label">Last updated</div><div class="stat-value" id="last-updated">—</div><div class="stat-note" id="data-age">Loading latest readings</div></article>
+    </section>
+
+    <section class="workspace">
+        <article class="map-card">
+            <div id="station-map" role="application" aria-label="Interactive map of Ping River monitoring stations"></div>
+            <div class="map-overlay">
+                <div class="map-heading"><strong>Station flow map</strong><span>River width, colour &amp; dash speed follow live discharge</span></div>
+                <div class="legend">
+                    <div class="legend-title">Flow status</div>
+                    <div class="legend-row"><i class="swatch" style="background:#1e8b60"></i> Low &lt; 25 m³/s</div>
+                    <div class="legend-row"><i class="swatch" style="background:#087da5"></i> Moderate 25–100</div>
+                    <div class="legend-row"><i class="swatch" style="background:#d99018"></i> High 100–250</div>
+                    <div class="legend-row"><i class="swatch" style="background:#cc4b37"></i> Very high &gt; 250</div>
+                    <div class="legend-row"><i class="line-swatch" style="background:#69b7d0"></i> River · no nearby gauge</div>
+                    <div class="legend-row"><i class="line-swatch" style="background:linear-gradient(90deg,#1e8b60,#087da5,#d99018,#cc4b37)"></i> River · gauge colour, dashes = flow</div>
+                </div>
+            </div>
+            <div class="loading-panel" id="loading"><div class="loading-card">Loading river conditions…</div></div>
+            <div class="error-panel" id="error"><div><strong>Map data could not be loaded.</strong><br><span id="error-message"></span></div></div>
+        </article>
+
+        <aside class="side-card">
+            <div class="side-head"><h2>Current station flow</h2><p>Select a station to locate it and load PostgreSQL history</p></div>
+            <div class="station-list" id="river-flow" aria-live="polite"></div>
+            <div class="side-head"><h2>Additional ThaiWater sensors</h2><p id="thaiwater-count">Loading Ping basin sensors…</p></div>
+            <div class="station-list" id="thaiwater-sensors" aria-live="polite"></div>
+        </aside>
+    </section>
+
+    <section class="map-card" id="forecast-card" style="margin-top:14px;padding:20px;display:none">
+        <div style="display:flex;justify-content:space-between;align-items:center;gap:14px;flex-wrap:wrap">
+            <div><h2 style="margin:0;font-size:1rem">Flood risk outlook <span style="color:var(--muted);font-weight:650;font-size:.72rem">· experimental</span></h2>
+            <p id="forecast-status" class="subtitle">Model probability of reaching warning / danger levels within 6, 12 and 24 hours</p></div>
+        </div>
+        <div class="p1-outlook" id="p1-outlook" style="display:none">
+            <div class="p1-outlook-head">
+                <div><strong>Chiang Mai city flood outlook · P.1 Nawarat Bridge</strong>
+                <div class="p1-peak" id="p1-peak"></div></div>
+                <button type="button" class="zones-button" id="zones-toggle">Show flood zones on map</button>
+            </div>
+            <div class="stage-strip" id="p1-stages"></div>
+            <div class="p1-peak" style="margin-top:7px">Chance the river reaches each official inundation stage within 24 h — city flooding begins at stage 1 (3.70 m); each stage floods additional districts.</div>
+        </div>
+        <div class="forecast-grid" id="forecast-grid"></div>
+    </section>
+
+    <section class="map-card" id="history-card" style="margin-top:14px;padding:20px">
+        <div style="display:flex;justify-content:space-between;align-items:center;gap:14px;flex-wrap:wrap">
+            <div><h2 id="history-title" style="margin:0;font-size:1rem">PostgreSQL history</h2><p id="history-status" class="subtitle">Select a RID flow station to load the last 7 days</p></div>
+            <select id="history-range" style="padding:9px 12px;border:1px solid var(--border);border-radius:10px;background:white"><option value="24">24 hours</option><option value="168" selected>7 days</option><option value="720">30 days</option><option value="2160">90 days</option><option value="876000">All time</option></select>
+        </div>
+        <div style="height:260px;margin-top:14px;overflow:hidden;position:relative"><canvas id="history-chart" aria-label="Historical water level and discharge chart" style="display:block"></canvas></div>
+    </section>
+</main>
+
+<script src="https://unpkg.com/leaflet@1.9.4/dist/leaflet.js" integrity="sha384-cxOPjt7s7Iz04uaHJceBmS+qpjv2JkIHNVcuOrM+YHwZOmJGBXI00mdUXEq65HTH" crossorigin="anonymous"></script>
+<script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.7/dist/chart.umd.min.js" integrity="sha384-vsrfeLOOY6KuIYKDlmVH5UiBmgIdB1oEf7p01YgWHuqmOHfZr374+odEv96n9tNC" crossorigin="anonymous"></script>
+<script>
+(function () {
+    'use strict';
+    const state = { map: null, layers: [], markers: new Map(), hasFit: false, historyChart: null, selectedStation: null, historyRequestId: 0 };
+    const $ = (id) => document.getElementById(id);
+
+    function flowColor(flow) {
+        if (flow == null || Number.isNaN(flow)) return '#7b8f94';
+        if (flow < 25) return '#1e8b60';
+        if (flow < 100) return '#087da5';
+        if (flow < 250) return '#d99018';
+        return '#cc4b37';
+    }
+
+    function escapeHtml(value) {
+        return String(value == null ? '' : value).replace(/[&<>'"]/g, (char) => ({
+            '&': '&amp;', '<': '&lt;', '>': '&gt;', "'": '&#39;', '"': '&quot;'
+        })[char]);
+    }
+
+    function latestByStation(measurements) {
+        const latest = new Map();
+        measurements.forEach((item) => {
+            const prior = latest.get(item.station_code);
+            if (!prior || new Date(item.timestamp) > new Date(prior.timestamp)) latest.set(item.station_code, item);
+        });
+        return latest;
+    }
+
+    function formatFlow(value) {
+        return value == null || Number.isNaN(Number(value)) ? 'No data' : `${Number(value).toFixed(1)} m³/s`;
+    }
+
+    function markerSize(flow) {
+        if (flow == null) return 24;
+        return Math.max(24, Math.min(42, 22 + Math.sqrt(Math.max(0, flow)) * 1.05));
+    }
+
+    function initMap() {
+        if (!window.L) throw new Error('The map library did not load. Check the internet connection.');
+        if (state.map) return;
+        state.map = L.map('station-map', { zoomControl: true, attributionControl: true }).setView([18.78, 98.98], 8);
+        L.tileLayer('https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png', {
+            maxZoom: 18,
+            attribution: '&copy; OpenStreetMap contributors'
+        }).addTo(state.map);
+    }
+
+    function clearLayers() {
+        state.layers.forEach((layer) => state.map.removeLayer(layer));
+        state.layers = [];
+        state.markers.clear();
+    }
+
+    function buildPopup(station, measurement) {
+        const flow = measurement ? measurement.discharge : null;
+        const level = measurement ? measurement.water_level : null;
+        const time = measurement ? new Date(measurement.timestamp).toLocaleString([], { dateStyle: 'medium', timeStyle: 'short' }) : 'No reading available';
+        return `<div class="popup">
+            <div class="popup-code">${escapeHtml(station.station_code)}</div>
+            <h3>${escapeHtml(station.english_name)}</h3>
+            <div class="popup-th">${escapeHtml(station.thai_name)}</div>
+            <div class="popup-grid">
+                <div class="popup-metric"><span>Discharge</span><strong>${escapeHtml(formatFlow(flow))}</strong></div>
+                <div class="popup-metric"><span>Water level</span><strong>${level == null ? 'No data' : `${Number(level).toFixed(2)} m`}</strong></div>
+            </div>
+            <div class="popup-time">Reading: ${escapeHtml(time)}</div>
+        </div>`;
+    }
+
+    function nearestGaugeFlow(feature, gauges) {
+        const coords = feature.geometry && feature.geometry.coordinates;
+        if (!gauges.length || !coords || !coords.length) return null;
+        let best = Infinity, q = null;
+        const step = Math.max(1, Math.floor(coords.length / 5));
+        for (let i = 0; i < coords.length; i += step) {
+            const lon = coords[i][0], lat = coords[i][1];
+            gauges.forEach((g) => {
+                const d = (g.lat - lat) * (g.lat - lat) + (g.lon - lon) * (g.lon - lon);
+                if (d < best) { best = d; q = g.q; }
+            });
+        }
+        return best < 0.16 ? q : null; // only grade segments within ~0.4° (~45 km) of a gauge
+    }
+
+    function riverWeight(q) {
+        return q == null ? 2.5 : Math.max(3, Math.min(9, 2.5 + Math.sqrt(Math.max(0, q)) * .38));
+    }
+
+    function riverSpeedClass(q) {
+        if (q == null) return 'flow-idle';
+        if (q < 25) return 'flow-slow';
+        if (q < 100) return 'flow-med';
+        if (q < 250) return 'flow-fast';
+        return 'flow-surge';
+    }
+
+    function renderRiverNetwork(riverNetwork, stations, readings) {
+        if (!riverNetwork) return;
+        const gauges = stations
+            .filter((s) => Number.isFinite(s.latitude) && Number.isFinite(s.longitude))
+            .map((s) => ({ lat: s.latitude, lon: s.longitude, q: readings.get(s.station_code)?.discharge }))
+            .filter((g) => g.q != null)
+            .map((g) => ({ lat: g.lat, lon: g.lon, q: Number(g.q) }));
+        const flowBySegment = new Map();
+        (riverNetwork.features || []).forEach((f) => flowBySegment.set(f, nearestGaugeFlow(f, gauges)));
+        const casing = L.geoJSON(riverNetwork, {
+            style: (f) => ({ color: '#e3f4f8', weight: riverWeight(flowBySegment.get(f)) + 4.5, opacity: .8, lineCap: 'round' })
+        }).addTo(state.map);
+        const flow = L.geoJSON(riverNetwork, {
+            style: (f) => {
+                const q = flowBySegment.get(f);
+                return {
+                    color: q == null ? '#69b7d0' : flowColor(q),
+                    weight: riverWeight(q), opacity: .92, lineCap: 'round',
+                    dashArray: '6 14', className: `flow-line ${riverSpeedClass(q)}`
+                };
+            }
+        }).addTo(state.map);
+        flow.bringToBack();
+        casing.bringToBack();
+        state.layers.push(casing, flow);
+    }
+
+    function renderMap(stations, readings, riverNetwork) {
+        clearLayers();
+        renderRiverNetwork(riverNetwork, stations, readings);
+        const mapped = stations.filter((station) => Number.isFinite(station.latitude) && Number.isFinite(station.longitude));
+        const bounds = [];
+
+        mapped.forEach((station) => {
+            const measurement = readings.get(station.station_code);
+            const flow = measurement && measurement.discharge != null ? Number(measurement.discharge) : null;
+            const color = flowColor(flow);
+            const size = markerSize(flow);
+            const icon = L.divIcon({
+                className: 'marker-wrap',
+                html: `<div class="flow-marker" style="--marker-color:${color};--marker-size:${size}px">${escapeHtml(station.station_code.replace('P.', ''))}</div>`,
+                iconSize: [size, size], iconAnchor: [size / 2, size / 2], popupAnchor: [0, -size / 2]
+            });
+            const marker = L.marker([station.latitude, station.longitude], { icon, title: `${station.station_code} ${station.english_name}` })
+                .bindPopup(buildPopup(station, measurement))
+                .on('click', () => loadHistory(station.station_code))
+                .addTo(state.map);
+            state.layers.push(marker);
+            state.markers.set(station.station_code, marker);
+            bounds.push([station.latitude, station.longitude]);
+        });
+        if (!state.hasFit && bounds.length) {
+            state.map.fitBounds(bounds, { padding: [35, 35], maxZoom: 9 });
+            state.hasFit = true;
+        }
+    }
+
+    async function loadHistory(stationCode) {
+        state.selectedStation = stationCode;
+        state.historyRequestId++;
+        const reqId = state.historyRequestId;
+        $('history-title').textContent = `${stationCode} · PostgreSQL history`;
+        $('history-status').textContent = 'Loading historical measurements…';
+        try {
+            const response = await fetch(`/measurements/history/${encodeURIComponent(stationCode)}?hours=${$('history-range').value}`);
+            if (!response.ok) throw new Error((await response.json()).detail || `HTTP ${response.status}`);
+            const rows = await response.json();
+            if (reqId !== state.historyRequestId) return;
+
+            // Downsample to daily averages to prevent browser freezing with 50k+ points
+            const downsample = (data) => {
+                const buckets = {};
+                data.forEach((row) => {
+                    const date = new Date(row.timestamp);
+                    const key = `${date.getUTCFullYear()}-${date.getUTCMonth()}-${date.getUTCDate()}`;
+                    if (!buckets[key]) buckets[key] = { ts: row.timestamp, discharge: [], level: [] };
+                    const b = buckets[key];
+                    if (row.discharge != null) b.discharge.push(row.discharge);
+                    if (row.water_level != null) b.level.push(row.water_level);
+                });
+                return Object.values(buckets).map((b) => ({
+                    timestamp: b.ts,
+                    discharge: b.discharge.length ? b.discharge.reduce((a, c) => a + c, 0) / b.discharge.length : null,
+                    water_level: b.level.length ? b.level.reduce((a, c) => a + c, 0) / b.level.length : null,
+                }));
+            };
+            const sampled = rows.length > 2000 ? downsample(rows) : rows;
+
+            // Safely clear existing chart instance
+            if (state.historyChart) { state.historyChart.destroy(); state.historyChart = null; }
+            // Clear any orphan Chart.js instance on the canvas
+            const existingChart = Chart.getChart($('history-chart'));
+            if (existingChart) existingChart.destroy();
+
+            state.historyChart = new Chart($('history-chart'), {
+                type: 'line',
+                data: {
+                    labels: sampled.map((row) => new Date(row.timestamp).toLocaleString('en-TH', { timeZone: 'Asia/Bangkok', month: 'short', day: 'numeric', year: '2-digit', hour: '2-digit' })),
+                    datasets: [
+                        { label: 'Discharge (m³/s)', data: sampled.map((row) => row.discharge), borderColor: '#087da5', backgroundColor: 'rgba(8,125,165,.12)', yAxisID: 'flow', pointRadius: 0, tension: .25 },
+                        { label: 'Water level (m)', data: sampled.map((row) => row.water_level), borderColor: '#d99018', yAxisID: 'level', pointRadius: 0, tension: .25 }
+                    ]
+                },
+                options: {
+                    responsive: true, maintainAspectRatio: false, animation: { duration: 0 },
+                    interaction: { mode: 'index', intersect: false },
+                    scales: {
+                        flow: { type: 'linear', position: 'left' },
+                        level: { type: 'linear', position: 'right', grid: { drawOnChartArea: false } },
+                        x: { ticks: { maxTicksLimit: 12 } }
+                    },
+                    plugins: {
+                        legend: { display: true },
+                        floodBands: { enabled: true }
+                    }
+                },
+                plugins: [{
+                    id: 'floodBands',
+                    beforeDraw(chart, _args, pluginOptions) {
+                        if (!pluginOptions.enabled || !chart.chartArea) return;
+                        const levelScale = chart.scales?.level;
+                        if (!levelScale || !Number.isFinite(levelScale.min) || !Number.isFinite(levelScale.max)) return;
+                        const { ctx, chartArea } = chart;
+                        const zones = [
+                            { min: levelScale.min, max: 3.0, color: 'rgba(30,139,96,.08)' },
+                            { min: 3.0, max: 4.5, color: 'rgba(217,144,24,.12)' },
+                            { min: 4.5, max: levelScale.max, color: 'rgba(204,75,55,.15)' },
+                        ];
+                        ctx.save();
+                        zones.forEach((z) => {
+                            const min = Math.max(z.min, levelScale.min);
+                            const max = Math.min(z.max, levelScale.max);
+                            if (max <= min) return;
+                            const top = levelScale.getPixelForValue(max);
+                            const bottom = levelScale.getPixelForValue(min);
+                            ctx.fillStyle = z.color;
+                            ctx.fillRect(chartArea.left, top, chartArea.right - chartArea.left, bottom - top);
+                        });
+                        ctx.restore();
+                    }
+                }]
+            });
+            $('history-status').textContent = rows.length ? `${rows.length} measurements (${sampled.length} daily) from PostgreSQL` : 'No PostgreSQL measurements in this period';
+            $('history-card').scrollIntoView({ behavior: 'smooth', block: 'nearest' });
+        } catch (error) {
+            $('history-status').textContent = `History unavailable: ${error.message}`;
+        }
+    }
+
+    function renderList(stations, readings) {
+        const container = $('river-flow');
+        container.replaceChildren();
+        const ordered = [...stations].sort((a, b) => {
+            const af = readings.get(a.station_code)?.discharge;
+            const bf = readings.get(b.station_code)?.discharge;
+            return (bf == null ? -1 : Number(bf)) - (af == null ? -1 : Number(af));
+        });
+        ordered.forEach((station) => {
+            const measurement = readings.get(station.station_code);
+            const flow = measurement?.discharge == null ? null : Number(measurement.discharge);
+            const row = document.createElement('button');
+            row.type = 'button'; row.className = 'station-row';
+            row.innerHTML = `<span class="station-code" style="background:${flowColor(flow)}">${escapeHtml(station.station_code)}</span>
+                <span class="station-name"><strong>${escapeHtml(station.english_name)}</strong><span>${escapeHtml(station.thai_name)}</span></span>
+                <span class="flow-value">${flow == null ? '—' : flow.toFixed(1)}<span>m³/s</span></span>`;
+            row.addEventListener('click', () => {
+                const marker = state.markers.get(station.station_code);
+                if (marker) { state.map.flyTo(marker.getLatLng(), Math.max(state.map.getZoom(), 11), { duration: .8 }); marker.openPopup(); }
+                loadHistory(station.station_code);
+            });
+            container.appendChild(row);
+        });
+    }
+
+    function renderThaiWaterSensors(sensors, existingCodes) {
+        const container = $('thaiwater-sensors');
+        container.replaceChildren();
+        const additional = sensors.filter((sensor) => !existingCodes.has(sensor.station_code));
+        $('thaiwater-count').textContent = `${additional.length} additional Ping basin stations · water level`;
+        additional.forEach((sensor) => {
+            const percent = sensor.bank_percent == null ? null : Number(sensor.bank_percent);
+            const color = percent == null ? '#7b8f94' : percent >= 100 ? '#cc4b37' : percent >= 80 ? '#d99018' : '#6c73b8';
+            const icon = L.divIcon({
+                className: 'marker-wrap',
+                html: `<div class="flow-marker" style="--marker-color:${color};--marker-size:22px">+</div>`,
+                iconSize: [22, 22], iconAnchor: [11, 11], popupAnchor: [0, -11]
+            });
+            const level = sensor.water_level_msl == null ? 'No data' : `${Number(sensor.water_level_msl).toFixed(2)} m MSL`;
+            const bank = sensor.distance_to_bank == null ? 'Unknown' : `${Number(sensor.distance_to_bank).toFixed(2)} m below bank`;
+            const marker = L.marker([sensor.latitude, sensor.longitude], { icon, title: `${sensor.station_code} ${sensor.station_name}` })
+                .bindPopup(`<div class="popup"><div class="popup-code">${escapeHtml(sensor.station_code)} · ThaiWater</div><h3>${escapeHtml(sensor.station_name)}</h3><div class="popup-th">${escapeHtml(sensor.river_name || 'Ping basin')} · ${escapeHtml(sensor.agency || '')}</div><div class="popup-grid"><div class="popup-metric"><span>Water level</span><strong>${escapeHtml(level)}</strong></div><div class="popup-metric"><span>Bank status</span><strong>${escapeHtml(bank)}</strong></div></div></div>`)
+                .addTo(state.map);
+            state.layers.push(marker);
+            state.markers.set(`thaiwater:${sensor.station_code}`, marker);
+
+            const row = document.createElement('button');
+            row.type = 'button'; row.className = 'station-row';
+            row.innerHTML = `<span class="station-code" style="background:${color}">${escapeHtml(sensor.station_code)}</span><span class="station-name"><strong>${escapeHtml(sensor.station_name)}</strong><span>${escapeHtml(sensor.river_name || 'Ping basin')} · ThaiWater</span></span><span class="flow-value">${percent == null ? '—' : percent.toFixed(0) + '%'}<span>bank level</span></span>`;
+            row.addEventListener('click', () => { state.map.flyTo(marker.getLatLng(), Math.max(state.map.getZoom(), 11), { duration: .8 }); marker.openPopup(); });
+            container.appendChild(row);
+        });
+    }
+
+    function renderSummary(stations, readings) {
+        const current = stations.map((s) => readings.get(s.station_code)).filter(Boolean);
+        const flows = current.filter((m) => m.discharge != null).map((m) => ({ code: m.station_code, value: Number(m.discharge) }));
+        const total = flows.reduce((sum, item) => sum + item.value, 0);
+        const peak = flows.length ? flows.reduce((max, item) => item.value > max.value ? item : max) : null;
+        const timestamps = current.map((m) => new Date(m.timestamp)).filter((date) => !Number.isNaN(date.getTime()));
+        const latest = timestamps.length ? new Date(Math.max(...timestamps.map((date) => date.getTime()))) : null;
+        $('station-count').textContent = `${current.length} / ${stations.length}`;
+        $('total-flow').textContent = flows.length ? total.toLocaleString(undefined, { maximumFractionDigits: 1 }) : '—';
+        $('peak-flow').textContent = peak ? peak.value.toFixed(1) : '—';
+        $('peak-station').textContent = peak ? `${peak.code} · m³/s` : 'No discharge reported';
+        $('last-updated').textContent = latest ? latest.toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' }) : '—';
+        if (latest) {
+            const minutes = Math.max(0, Math.round((Date.now() - latest.getTime()) / 60000));
+            $('data-age').textContent = `${latest.toLocaleDateString([], { day: 'numeric', month: 'short' })} · ${minutes} min ago`;
+        } else $('data-age').textContent = 'No timestamp available';
+    }
+
+    async function loadDashboard() {
+        const refresh = $('refresh-button');
+        refresh.disabled = true;
+        $('loading').style.display = 'grid';
+        $('error').style.display = 'none';
+        try {
+            initMap();
+            const [stationResponse, measurementResponse, riverResponse, thaiWaterResponse] = await Promise.all([
+                fetch('/stations'),
+                fetch('/measurements/latest?limit=500'),
+                fetch('/static/ping-river-network.geojson'),
+                fetch('/sensors/thaiwater')
+            ]);
+            if (!stationResponse.ok || !measurementResponse.ok || !riverResponse.ok) {
+                throw new Error(`API returned ${stationResponse.status}/${measurementResponse.status}/${riverResponse.status}`);
+            }
+            const stations = await stationResponse.json();
+            const measurements = await measurementResponse.json();
+            const riverNetwork = await riverResponse.json();
+            const thaiWaterSensors = thaiWaterResponse.ok ? await thaiWaterResponse.json() : [];
+            const readings = latestByStation(measurements);
+            renderMap(stations, readings, riverNetwork);
+            renderList(stations, readings);
+            renderThaiWaterSensors(thaiWaterSensors, new Set(stations.map((station) => station.station_code)));
+            renderSummary(stations, readings);
+            $('loading').style.display = 'none';
+            loadForecasts(); // non-blocking; the card stays hidden until models are deployed
+        } catch (error) {
+            $('loading').style.display = 'none';
+            $('error').style.display = 'grid';
+            $('error-message').textContent = error.message;
+            console.error('Dashboard load failed:', error);
+        } finally {
+            refresh.disabled = false;
+        }
+    }
+
+    function riskColor(pWarning, pDanger) {
+        if (pDanger >= .5) return '#cc4b37';
+        if (pWarning >= .5 || pDanger >= .2) return '#d99018';
+        if (pWarning >= .2) return '#0a91b9';
+        return '#1e8b60';
+    }
+
+    // Official Chiang Mai inundation map (keyed to P.1), georeferenced approximately.
+    // Tune bounds if the river course in the scan drifts from the basemap.
+    const FLOOD_ZONE_IMAGE = '/static/flood-zones-p1.jpg';
+    const FLOOD_ZONE_BOUNDS = [[18.680, 98.925], [18.855, 99.105]];
+    let zoneOverlay = null;
+
+    function toggleFloodZones() {
+        if (!state.map) return;
+        const button = $('zones-toggle');
+        if (zoneOverlay) {
+            state.map.removeLayer(zoneOverlay);
+            zoneOverlay = null;
+            if (button) button.textContent = 'Show flood zones on map';
+        } else {
+            zoneOverlay = L.imageOverlay(FLOOD_ZONE_IMAGE, FLOOD_ZONE_BOUNDS, { opacity: .62, interactive: false }).addTo(state.map);
+            state.map.flyToBounds(FLOOD_ZONE_BOUNDS, { maxZoom: 13, duration: .8 });
+            if (button) button.textContent = 'Hide flood zones';
+            document.getElementById('station-map').scrollIntoView({ behavior: 'smooth', block: 'nearest' });
+        }
+    }
+
+    function stageColor(p) {
+        if (p >= .5) return '#cc4b37';
+        if (p >= .2) return '#d99018';
+        if (p >= .05) return '#0a91b9';
+        return '#1e8b60';
+    }
+
+    function renderP1Outlook(rows) {
+        const card = $('p1-outlook');
+        const p1rows = rows.filter((r) => r.station_code === 'P.1' && Array.isArray(r.stages));
+        if (!p1rows.length) { card.style.display = 'none'; return; }
+        const row = p1rows.reduce((best, r) => r.horizon_hours > best.horizon_hours ? r : best);
+        $('p1-peak').textContent = `Now ${Number(row.current_level).toFixed(2)} m · predicted peak next ${row.horizon_hours} h: ${Number(row.predicted_max_level).toFixed(2)} m`;
+        const strip = $('p1-stages');
+        strip.replaceChildren();
+        row.stages.forEach((s) => {
+            const chip = document.createElement('div');
+            chip.className = 'stage-chip';
+            chip.style.background = stageColor(s.p_exceed);
+            chip.title = `Stage ${s.stage}: river at ${s.level.toFixed(2)} m — ${Math.round(s.p_exceed * 100)}% within ${row.horizon_hours} h`;
+            chip.innerHTML = `${Math.round(s.p_exceed * 100)}%<small>S${s.stage} · ${s.level.toFixed(2)} m</small>`;
+            strip.appendChild(chip);
+        });
+        card.style.display = 'block';
+    }
+
+    async function loadForecasts() {
+        const card = $('forecast-card');
+        try {
+            const response = await fetch('/forecast');
+            if (!response.ok) { card.style.display = 'none'; return; }
+            const rows = await response.json();
+            if (!Array.isArray(rows) || !rows.length) { card.style.display = 'none'; return; }
+            const byStation = new Map();
+            rows.forEach((row) => {
+                if (!byStation.has(row.station_code)) byStation.set(row.station_code, []);
+                byStation.get(row.station_code).push(row);
+            });
+            renderP1Outlook(rows);
+            const grid = $('forecast-grid');
+            grid.replaceChildren();
+            const stations = [...byStation.entries()].map(([code, list]) => ({
+                code, list: list.sort((a, b) => a.horizon_hours - b.horizon_hours),
+                worst: Math.max(...list.map((r) => Math.max(r.p_danger ?? 0, (r.p_warning ?? 0) * .5)))
+            })).sort((a, b) => b.worst - a.worst);
+            stations.forEach(({ code, list }) => {
+                const first = list[0];
+                const cardEl = document.createElement('div');
+                cardEl.className = 'forecast-station';
+                const chips = list.map((r) => {
+                    const pw = r.p_warning ?? 0, pd = r.p_danger ?? 0;
+                    const pct = Math.round(Math.max(pw, pd) * 100);
+                    const title = `+${r.horizon_hours}h · warning ${Math.round(pw * 100)}% · danger ${Math.round(pd * 100)}%` +
+                        (r.predicted_max_level == null ? '' : ` · peak ~${Number(r.predicted_max_level).toFixed(2)} m`) +
+                        (r.source === 'heuristic' ? ' · heuristic fallback' : '');
+                    return `<div class="risk-chip" style="background:${riskColor(pw, pd)}" title="${escapeHtml(title)}">${pct}%<span>${r.horizon_hours}h</span></div>`;
+                }).join('');
+                cardEl.innerHTML = `<strong>${escapeHtml(code)}</strong>` +
+                    `<div class="fc-name">${first.current_level == null ? '' : `now ${Number(first.current_level).toFixed(2)} m · `}peak risk next 24h</div>` +
+                    `<div class="risk-chips">${chips}</div>`;
+                grid.appendChild(cardEl);
+            });
+            const asOf = rows[0].as_of ? new Date(rows[0].as_of).toLocaleString([], { dateStyle: 'medium', timeStyle: 'short' }) : null;
+            $('forecast-status').textContent = `Probability of exceeding warning / danger level within 6, 12 and 24 h` +
+                (asOf ? ` · based on readings up to ${asOf}` : '');
+            card.style.display = 'block';
+        } catch (error) {
+            card.style.display = 'none';
+        }
+    }
+
+    $('refresh-button').addEventListener('click', loadDashboard);
+    $('zones-toggle').addEventListener('click', toggleFloodZones);
+    $('history-range').addEventListener('change', () => { if (state.selectedStation) loadHistory(state.selectedStation); });
+    loadDashboard();
+    window.setInterval(loadDashboard, 5 * 60 * 1000);
+})();
+</script>
+</body>
+</html>

src/thaiwater.py

@@ -0,0 +1,61 @@
+"""Client for ThaiWater's public water-level sensor feed."""
+
+from typing import Dict, List, Optional
+
+import requests
+
+
+class ThaiWaterClient:
+    API_URL = "https://twa-api-public.thaiwater.net/v2/waterlevel"
+
+    def __init__(self, session=None, api_key: Optional[str] = None, timeout: int = 30):
+        self.session = session or requests.Session()
+        self.api_key = api_key
+        self.timeout = timeout
+
+    def fetch_ping_sensors(self) -> List[Dict]:
+        if not self.api_key:
+            raise RuntimeError("THAIWATER_API_KEY is not configured")
+
+        response = self.session.get(
+            self.API_URL,
+            headers={"Accept-Language": "en", "x-api-key": self.api_key},
+            timeout=self.timeout,
+        )
+        response.raise_for_status()
+        return self._parse_ping_features(response.json())
+
+    @staticmethod
+    def _parse_ping_features(payload: Dict) -> List[Dict]:
+        sensors = []
+        for collection in payload.get("data", {}).values():
+            for feature in collection.get("features", []):
+                properties = feature.get("properties") or {}
+                basin = properties.get("basin") or {}
+                if basin.get("basin") != "Ping":
+                    continue
+
+                geometry = feature.get("geometry") or {}
+                coordinates = geometry.get("coordinates") or []
+                if len(coordinates) < 2:
+                    continue
+
+                station = properties.get("station") or {}
+                station_code = station.get("stationCode", "")
+                sensors.append(
+                    {
+                        "id": f"thaiwater:{properties.get('id')}",
+                        "station_code": station_code.split("-", 1)[-1],
+                        "station_name": station.get("station"),
+                        "latitude": coordinates[1],
+                        "longitude": coordinates[0],
+                        "timestamp": properties.get("waterlevelDatetime"),
+                        "water_level_msl": properties.get("waterlevelMsl"),
+                        "bank_percent": properties.get("storagePercent"),
+                        "distance_to_bank": properties.get("diffWlBank"),
+                        "river_name": properties.get("riverName"),
+                        "agency": (properties.get("agency") or {}).get("agencyShort"),
+                        "source": "ThaiWater",
+                    }
+                )
+        return sensors

src/validators.py

@@ -26,8 +26,8 @@ class DataValidator:
     def validate_measurement(cls, measurement: Dict[str, Any]) -> bool:
         """Validate a single measurement"""
         try:
-            # Check required fields
-            required_fields = ['timestamp', 'station_id', 'water_level', 'discharge']
+            # Check required fields (discharge is now optional)
+            required_fields = ['timestamp', 'station_id', 'water_level']
             for field in required_fields:
                 if field not in measurement:
                     logger.warning(f"Missing required field: {field}")
@@ -38,14 +38,19 @@ class DataValidator:
                 logger.warning(f"Invalid timestamp type: {type(measurement['timestamp'])}")
                 return False
 
-            # Validate water level
+            # Validate water level (required)
+            if measurement['water_level'] is None:
+                logger.warning("Water level cannot be None")
+                return False
             water_level = float(measurement['water_level'])
             if not (cls.WATER_LEVEL_MIN <= water_level <= cls.WATER_LEVEL_MAX):
                 logger.warning(f"Water level out of range: {water_level}")
                 return False
 
-            # Validate discharge
-            discharge = float(measurement['discharge'])
+            # Validate discharge (optional - can be None)
+            discharge_value = measurement.get('discharge')
+            if discharge_value is not None:
+                discharge = float(discharge_value)
                 if not (cls.DISCHARGE_MIN <= discharge <= cls.DISCHARGE_MAX):
                     logger.warning(f"Discharge out of range: {discharge}")
                     return False

src/water_scraper_v3.py

@@ -3,26 +3,27 @@
 Enhanced Water Monitor Scraper with multiple database backend support
 """
 
-import requests
 import datetime
-import time
-import schedule
 import json
+import logging
 import os
-from typing import List, Dict, Optional
+import time
+from typing import Dict, List, Optional
+
+import requests
+import schedule
 
 try:
-    from .database_adapters import create_database_adapter, DatabaseAdapter
-    from .models import WaterMeasurement, StationInfo, ScrapingResult, StationStatus
-    from .validators import DataValidator
-    from .exceptions import APIConnectionError, DataValidationError, DatabaseConnectionError
-    from .metrics import increment_counter, set_gauge, record_histogram, Timer
-    from .rate_limiter import RateLimiter, RequestTracker
+    from .config import Config
+    from .database_adapters import create_database_adapter
     from .logging_config import get_logger
+    from .metrics import Timer, increment_counter, record_histogram, set_gauge
+    from .rate_limiter import RateLimiter, RequestTracker
+    from .validators import DataValidator
 except ImportError:
     # Handle case when running as standalone script
-    from database_adapters import create_database_adapter, DatabaseAdapter
-    import logging
+    from config import Config
+    from database_adapters import create_database_adapter
 
     def get_logger(name):
         return logging.getLogger(name)
@@ -39,20 +40,24 @@ except ImportError:
     class Timer:
         def __init__(self, *args, **kwargs):
             pass
+
         def __enter__(self):
             return self
+
         def __exit__(self, *args):
             pass
 
     class RateLimiter:
         def __init__(self, *args, **kwargs):
             pass
+
         def wait_if_needed(self):
             pass
 
     class RequestTracker:
         def __init__(self):
             pass
+
         def record_request(self, *args, **kwargs):
             pass
 
@@ -61,9 +66,11 @@ except ImportError:
         def validate_measurements(measurements):
             return measurements
 
+
 # Get logger instance
 logger = get_logger(__name__)
 
+
 class EnhancedWaterMonitorScraper:
     def __init__(self, db_config: Dict):
         """
@@ -87,153 +94,77 @@ class EnhancedWaterMonitorScraper:
 
         # HTTP session for API requests
         self.session = requests.Session()
-        self.session.headers.update({
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
-            'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
-            'Accept': 'application/json, text/javascript, */*; q=0.01',
-            'X-Requested-With': 'XMLHttpRequest'
-        })
+        self.session.headers.update(
+            {
+                "User-Agent": Config.USER_AGENT,
+                "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
+                "Accept": "application/json, text/javascript, */*; q=0.01",
+                "X-Requested-With": "XMLHttpRequest",
+            }
+        )
 
-        # Station mapping with correct names and geolocation data
-        self.station_mapping = {
-            '1': {
-                'code': 'P.20', 
-                'thai_name': 'บ้านเชียงดาว', 
-                'english_name': 'Ban Chiang Dao',
-                'latitude': 19.36731448032191,
-                'longitude': 98.9688487015384,
-                'geohash': None
-            },
-            '2': {
-                'code': 'P.75', 
-                'thai_name': 'บ้านช่อแล', 
-                'english_name': 'Ban Chai Lat',
-                'latitude': 19.145972935976225, 
-                'longitude': 99.00735727149247,
-                'geohash': None
-            },
-            '3': {
-                'code': 'P.92', 
-                'thai_name': 'บ้านเมืองกึ๊ด', 
-                'english_name': 'Ban Muang Aut',
-                'latitude': 19.220518985435646, 
-                'longitude': 98.84733127007874,
-                'geohash': None
-            },
-            '4': {
-                'code': 'P.4A', 
-                'thai_name': 'บ้านแม่แตง', 
-                'english_name': 'Ban Mae Taeng',
-                'latitude': 19.1222679952378,
-                'longitude': 98.94437462084075,
-                'geohash': None
-            },
-            '5': {
-                'code': 'P.67', 
-                'thai_name': 'บ้านแม่แต', 
-                'english_name': 'Ban Tae',
-                'latitude': 19.009762080002453,
-                'longitude': 98.95978297135508,
-                'geohash': None
-            },
-            '6': {
-                'code': 'P.21', 
-                'thai_name': 'บ้านริมใต้', 
-                'english_name': 'Ban Rim Tai',
-                'latitude': 18.917459157963293, 
-                'longitude': 98.97018092996231,
-                'geohash': None
-            },
-            '7': {
-                'code': 'P.103', 
-                'thai_name': 'สะพานวงแหวนรอบ 3', 
-                'english_name': 'Ring Bridge 3',
-                'latitude': 18.86664807441675, 
-                'longitude': 98.9781107622432,
-                'geohash': None
-            },
-            '8': {
-                'code': 'P.1', 
-                'thai_name': 'สะพานนวรัฐ', 
-                'english_name': 'Nawarat Bridge',
-                'latitude': 18.7875,
-                'longitude': 99.0045,
-                'geohash': 'w5q6uuhvfcfp25'
-            },
-            '9': {
-                'code': 'P.82', 
-                'thai_name': 'บ้านสบวิน', 
-                'english_name': 'Ban Sob win',
-                'latitude': 18.6519444,
-                'longitude': 98.69,
-                'geohash': None
-            },
-            '10': {
-                'code': 'P.84', 
-                'thai_name': 'บ้านพันตน', 
-                'english_name': 'Ban Panton',
-                'latitude': 18.591315274591334,
-                'longitude': 98.79657058508496,
-                'geohash': None
-            },
-            '11': {
-                'code': 'P.81', 
-                'thai_name': 'บ้านโป่ง', 
-                'english_name': 'Ban Pong',
-                'latitude': 13.805661820610888, 
-                'longitude': 99.87174946122846,
-                'geohash': None
-            },
-            '12': {
-                'code': 'P.5', 
-                'thai_name': 'สะพานท่านาง', 
-                'english_name': 'Tha Nang Bridge',
-                'latitude': 18.580269437546555,
-                'longitude': 99.01021397084362,
-                'geohash': None
-            },
-            '13': {
-                'code': 'P.77', 
-                'thai_name': 'บ้านสบแม่สะป๊วด', 
-                'english_name': 'Baan Sop Mae Sapuord',
-                'latitude': 18.433347475179602,
-                'longitude': 99.08510036666527,
-                'geohash': None
-            },
-            '14': {
-                'code': 'P.87', 
-                'thai_name': 'บ้านป่าซาง', 
-                'english_name': 'Ban Pa Sang',
-                'latitude': 18.519121825282486, 
-                'longitude': 98.94224374138238,
-                'geohash': None
-            },
-            '15': {
-                'code': 'P.76', 
-                'thai_name': 'บ้านแม่อีไฮ', 
-                'english_name': 'Banb Mae I Hai',
-                'latitude': 18.141465831254404,
-                'longitude': 98.89642508267181,
-                'geohash': None
-            },
-            '16': {
-                'code': 'P.85', 
-                'thai_name': 'บ้านหล่ายแก้ว', 
-                'english_name': 'Baan Lai Kaew',
-                'latitude': 18.17856361002219, 
-                'longitude': 98.63023114782287,
-                'geohash': None
-            }
-        }
+        # Station mapping is persisted to a JSON file so that station CRUD via the
+        # API survives restarts; on first run it is seeded from the bundled
+        # defaults in data/stations.json.
+        self.station_config_path = Config.STATION_CONFIG_PATH
+        self.station_mapping = self._load_station_mapping()
 
         self.init_database()
 
+    @staticmethod
+    def _default_station_mapping_path() -> str:
+        """Path to the bundled default station mapping shipped with the package."""
+        return os.path.join(os.path.dirname(os.path.abspath(__file__)), "data", "stations.json")
+
+    def _load_station_mapping(self) -> Dict:
+        """Load the station mapping, preferring the runtime-writable config file.
+
+        Order of precedence:
+        1. The runtime config file (STATION_CONFIG_PATH) if it exists — this holds
+           any changes made through the station CRUD API.
+        2. The bundled defaults in data/stations.json.
+        """
+        for source in (self.station_config_path, self._default_station_mapping_path()):
+            if source and os.path.exists(source):
+                try:
+                    with open(source, encoding="utf-8") as f:
+                        mapping = json.load(f)
+                    logger.info(f"Loaded {len(mapping)} stations from {source}")
+                    return mapping
+                except Exception as e:
+                    logger.error(f"Failed to load station mapping from {source}: {e}")
+
+        logger.error("No station mapping could be loaded; starting with an empty mapping")
+        return {}
+
+    def save_stations(self) -> bool:
+        """Persist the current station mapping to the runtime config file.
+
+        Written atomically (temp file + replace) so a crash mid-write cannot
+        corrupt the existing configuration.
+        """
+        path = self.station_config_path
+        if not path:
+            logger.warning("STATION_CONFIG_PATH not set; station changes will not persist")
+            return False
+        try:
+            tmp_path = f"{path}.tmp"
+            with open(tmp_path, "w", encoding="utf-8") as f:
+                json.dump(self.station_mapping, f, ensure_ascii=False, indent=2)
+                f.write("\n")
+            os.replace(tmp_path, path)
+            logger.info(f"Persisted {len(self.station_mapping)} stations to {path}")
+            return True
+        except Exception as e:
+            logger.error(f"Failed to persist station mapping to {path}: {e}")
+            return False
+
     def init_database(self):
         """Initialize database connection"""
         try:
             # Extract db_type and pass remaining config as kwargs
             db_config_copy = self.db_config.copy()
-            db_type = db_config_copy.pop('type')
+            db_type = db_config_copy.pop("type")
             self.db_adapter = create_database_adapter(db_type, **db_config_copy)
             success = self.db_adapter.connect()
 
@@ -267,15 +198,15 @@ class EnhancedWaterMonitorScraper:
 
                 # API parameters
                 payload = {
-                    'DW[UtokID]': '1',
-                    'DW[BasinID]': '6',
-                    'DW[TimeCurrent]': thai_date,
-                    '_search': 'false',
-                    'nd': str(int(time.time() * 1000)),
-                    'rows': '100',
-                    'page': '1',
-                    'sidx': 'indexhourly',
-                    'sord': 'asc'
+                    "DW[UtokID]": "1",
+                    "DW[BasinID]": "6",
+                    "DW[TimeCurrent]": thai_date,
+                    "_search": "false",
+                    "nd": str(int(time.time() * 1000)),
+                    "rows": "100",
+                    "page": "1",
+                    "sidx": "indexhourly",
+                    "sord": "asc",
                 }
 
                 logger.debug(f"API parameters: {payload}")
@@ -305,11 +236,11 @@ class EnhancedWaterMonitorScraper:
                 water_data = []
 
                 # Parse JSON data
-                if json_data and isinstance(json_data, dict) and 'rows' in json_data:
-                    for row in json_data['rows']:
+                if json_data and isinstance(json_data, dict) and "rows" in json_data:
+                    for row in json_data["rows"]:
                         try:
                             # Parse timestamp
-                            time_str = row.get('hourlytime', '')
+                            time_str = row.get("hourlytime", "")
                             if not time_str:
                                 continue
 
@@ -334,53 +265,81 @@ class EnhancedWaterMonitorScraper:
                             # Parse all water levels and discharge values
                             station_count = 0
                             for station_num in range(1, 17):  # Stations 1-16
-                                wl_key = f'wlvalues{station_num}'
-                                q_key = f'qvalues{station_num}'
-                                qp_key = f'QPercent{station_num}'
+                                wl_key = f"wlvalues{station_num}"
+                                q_key = f"qvalues{station_num}"
+                                qp_key = f"QPercent{station_num}"
 
-                                # Check if both water level and discharge data exist
-                                if wl_key in row and q_key in row:
+                                # Check if water level data exists (required)
+                                if wl_key in row:
                                     try:
                                         water_level = row[wl_key]
-                                        discharge = row[q_key]
-                                        discharge_percent = row.get(qp_key)
 
-                                        # Skip if values are None or invalid
-                                        if water_level is None or discharge is None:
+                                        # Skip if water level is None or invalid
+                                        if water_level is None:
                                             continue
 
-                                        # Convert to float
+                                        # Convert water level to float (required)
                                         water_level = float(water_level)
-                                        discharge = float(discharge)
-                                        discharge_percent = float(discharge_percent) if discharge_percent is not None else None
 
-                                        station_info = self.station_mapping.get(str(station_num), {
-                                            'code': f'P.{19+station_num}',
-                                            'thai_name': f'Station {station_num}',
-                                            'english_name': f'Station {station_num}'
-                                        })
+                                        # Try to parse discharge data (optional)
+                                        discharge = None
+                                        discharge_percent = None
 
-                                        water_data.append({
-                                            'timestamp': data_time,
-                                            'station_id': station_num,
-                                            'station_code': station_info['code'],
-                                            'station_name_en': station_info['english_name'],
-                                            'station_name_th': station_info['thai_name'],
-                                            'latitude': station_info.get('latitude'),
-                                            'longitude': station_info.get('longitude'),
-                                            'geohash': station_info.get('geohash'),
-                                            'water_level': water_level,
-                                            'water_level_unit': 'm',
-                                            'discharge': discharge,
-                                            'discharge_unit': 'cms',
-                                            'discharge_percent': discharge_percent,
-                                            'status': 'active'
-                                        })
+                                        if q_key in row:
+                                            try:
+                                                discharge_raw = row[q_key]
+                                                if discharge_raw is not None and discharge_raw != "***":
+                                                    discharge = float(discharge_raw)
+
+                                                    # Only parse discharge percent if discharge is valid
+                                                    discharge_percent_raw = row.get(qp_key)
+                                                    if discharge_percent_raw is not None:
+                                                        try:
+                                                            discharge_percent = float(discharge_percent_raw)
+                                                        except (ValueError, TypeError):
+                                                            discharge_percent = None
+                                                else:
+                                                    logger.debug(
+                                                        "Skipping malformed discharge data for "
+                                                        f"station {station_num}: {discharge_raw}"
+                                                    )
+                                            except (ValueError, TypeError) as e:
+                                                logger.debug(
+                                                    f"Could not parse discharge for station {station_num}: {e}"
+                                                )
+
+                                        station_info = self.station_mapping.get(
+                                            str(station_num),
+                                            {
+                                                "code": f"P.{19+station_num}",
+                                                "thai_name": f"Station {station_num}",
+                                                "english_name": f"Station {station_num}",
+                                            },
+                                        )
+
+                                        water_data.append(
+                                            {
+                                                "timestamp": data_time,
+                                                "station_id": station_num,
+                                                "station_code": station_info["code"],
+                                                "station_name_en": station_info["english_name"],
+                                                "station_name_th": station_info["thai_name"],
+                                                "latitude": station_info.get("latitude"),
+                                                "longitude": station_info.get("longitude"),
+                                                "geohash": station_info.get("geohash"),
+                                                "water_level": water_level,
+                                                "water_level_unit": "m",
+                                                "discharge": discharge,
+                                                "discharge_unit": "cms",
+                                                "discharge_percent": discharge_percent,
+                                                "status": "active",
+                                            }
+                                        )
 
                                         station_count += 1
 
                                     except (ValueError, TypeError) as e:
-                                        logger.warning(f"Could not parse data for station {station_num}: {e}")
+                                        logger.warning(f"Could not parse water level for station {station_num}: {e}")
                                         continue
 
                             logger.debug(f"Processed {station_count} stations for time {time_str}")
@@ -392,7 +351,10 @@ class EnhancedWaterMonitorScraper:
                 # Validate data
                 water_data = DataValidator.validate_measurements(water_data)
 
-                logger.info(f"Successfully fetched {len(water_data)} data points from API for {target_date.strftime('%Y-%m-%d')}")
+                logger.info(
+                    f"Successfully fetched {len(water_data)} data points from API "
+                    f"for {target_date.strftime('%Y-%m-%d')}"
+                )
                 return water_data
 
             except requests.RequestException as e:
@@ -407,9 +369,34 @@ class EnhancedWaterMonitorScraper:
                 return None
 
     def fetch_water_data(self) -> Optional[List[Dict]]:
-        """Fetch water levels and discharge data from API for current date"""
-        current_date = datetime.datetime.now()
-        return self.fetch_water_data_for_date(current_date)
+        """Fetch water levels and discharge data from API with smart date selection"""
+        current_time = datetime.datetime.now()
+
+        # If it's past 01:00, try today's data first, then yesterday as fallback
+        if current_time.hour >= 1:
+            logger.info("After 01:00 - trying today's data first, will fallback to yesterday if needed")
+
+            # Try today's data first
+            today_data = self.fetch_water_data_for_date(current_time)
+            if today_data and len(today_data) > 0:
+                logger.info(f"Successfully fetched {len(today_data)} data points for today")
+                return today_data
+
+            # Fallback to yesterday's data
+            logger.info("No data available for today, trying yesterday's data")
+            yesterday = current_time - datetime.timedelta(days=1)
+            yesterday_data = self.fetch_water_data_for_date(yesterday)
+            if yesterday_data and len(yesterday_data) > 0:
+                logger.info(f"Successfully fetched {len(yesterday_data)} data points for yesterday")
+                return yesterday_data
+
+            logger.warning("No data available for today or yesterday")
+            return None
+        else:
+            # Before 01:00 - only try yesterday's data (API likely hasn't updated yet)
+            logger.info("Before 01:00 - fetching yesterday's data only")
+            yesterday = current_time - datetime.timedelta(days=1)
+            return self.fetch_water_data_for_date(yesterday)
 
     def save_to_database(self, water_data: List[Dict], max_retries: int = 3) -> bool:
         """Save water measurements to database with retry logic"""
@@ -435,7 +422,7 @@ class EnhancedWaterMonitorScraper:
             except Exception as e:
                 if "database is locked" in str(e).lower() and attempt < max_retries - 1:
                     logger.warning(f"Database locked on attempt {attempt + 1}, retrying in {2 ** attempt} seconds...")
-                    time.sleep(2 ** attempt)  # Exponential backoff
+                    time.sleep(2**attempt)  # Exponential backoff
                     continue
                 else:
                     logger.error(f"Error saving to database (attempt {attempt + 1}): {e}")
@@ -456,23 +443,72 @@ class EnhancedWaterMonitorScraper:
             logger.error(f"Error getting latest data: {e}")
             return []
 
+    def _check_data_freshness(self, water_data: List[Dict]) -> bool:
+        """Check if the fetched data contains new data for the current hour"""
+        if not water_data:
+            return False
+
+        current_time = datetime.datetime.now()
+        current_hour = current_time.hour
+
+        # Find the most recent timestamp in the data
+        latest_timestamp = None
+        for data_point in water_data:
+            timestamp = data_point.get("timestamp")
+            if timestamp and (latest_timestamp is None or timestamp > latest_timestamp):
+                latest_timestamp = timestamp
+
+        if latest_timestamp is None:
+            logger.warning("No valid timestamps found in data")
+            return False
+
+        latest_hour = latest_timestamp.hour
+        time_diff = current_time - latest_timestamp
+        minutes_old = time_diff.total_seconds() / 60
+
+        logger.info(
+            f"Current time: {current_time.strftime('%H:%M')}, Latest data: {latest_timestamp.strftime('%H:%M')}"
+        )
+        logger.info(f"Current hour: {current_hour}, Latest data hour: {latest_hour}, Age: {minutes_old:.1f} minutes")
+
+        # Strict check: we need data from the current hour
+        # If it's 20:xx and we only have data up to 19:xx, that's stale - go to retry mode
+        has_current_hour_data = latest_hour >= current_hour
+
+        if not has_current_hour_data:
+            logger.warning(f"No new data available - expected hour {current_hour}, got {latest_hour}")
+            logger.warning("Switching to retry mode until new data becomes available")
+            return False
+        else:
+            logger.info(f"Fresh data available for current hour {current_hour}")
+            return True
+
     def run_scraping_cycle(self) -> bool:
-        """Run a complete scraping cycle"""
+        """Run a complete scraping cycle with freshness check"""
         logger.info("Starting scraping cycle...")
 
         try:
             # Fetch current data
             water_data = self.fetch_water_data()
             if water_data:
+                # Check if data is fresh/recent
+                is_fresh = self._check_data_freshness(water_data)
+
+                if is_fresh:
                     success = self.save_to_database(water_data)
                     if success:
-                    logger.info("Scraping cycle completed successfully")
+                        logger.info("Scraping cycle completed successfully with fresh data")
                         increment_counter("scraping_cycles_successful")
                         return True
                     else:
                         logger.error("Failed to save data")
                         increment_counter("scraping_cycles_failed")
                         return False
+                else:
+                    # Data exists but is stale
+                    logger.warning("Data fetched but is stale - treating as no fresh data available")
+                    increment_counter("scraping_cycles_failed")
+                    return False
             else:
                 logger.warning("No data fetched")
                 increment_counter("scraping_cycles_failed")
@@ -483,20 +519,165 @@ class EnhancedWaterMonitorScraper:
             increment_counter("scraping_cycles_failed")
             return False
 
+    def fill_data_gaps(self, days_back: int) -> int:
+        """Fill gaps in data for the specified number of days back"""
+        logger = get_logger(__name__)
+        filled_count = 0
+
+        try:
+            # Calculate date range
+            end_date = datetime.datetime.now()
+            start_date = end_date - datetime.timedelta(days=days_back)
+
+            logger.info(f"Checking for gaps from {start_date.date()} to {end_date.date()}")
+
+            # Iterate through each date in the range
+            current_date = start_date
+            while current_date <= end_date:
+                # Check if we have data for this date
+                has_data = self._check_data_exists_for_date(current_date)
+
+                if not has_data:
+                    logger.info(f"Filling gap for date: {current_date.date()}")
+
+                    # Fetch data for this specific date
+                    data = self.fetch_water_data_for_date(current_date)
+
+                    if data:
+                        # Save the data
+                        if self.save_to_database(data):
+                            filled_count += len(data)
+                            logger.info(f"Filled {len(data)} measurements for {current_date.date()}")
+                        else:
+                            logger.warning(f"Failed to save data for {current_date.date()}")
+                    else:
+                        logger.warning(f"No data available for {current_date.date()}")
+
+                current_date += datetime.timedelta(days=1)
+
+        except Exception as e:
+            logger.error(f"Gap filling error: {e}")
+
+        return filled_count
+
+    def update_existing_data(self, days_back: int) -> int:
+        """Update existing data with latest values for the specified number of days back"""
+        logger = get_logger(__name__)
+        updated_count = 0
+
+        try:
+            # Calculate date range
+            end_date = datetime.datetime.now()
+            start_date = end_date - datetime.timedelta(days=days_back)
+
+            logger.info(f"Updating data from {start_date.date()} to {end_date.date()}")
+
+            # Iterate through each date in the range
+            current_date = start_date
+            while current_date <= end_date:
+                logger.info(f"Updating data for date: {current_date.date()}")
+
+                # Fetch fresh data for this date
+                data = self.fetch_water_data_for_date(current_date)
+
+                if data:
+                    # Save the data (this will update existing records)
+                    if self.save_to_database(data):
+                        updated_count += len(data)
+                        logger.info(f"Updated {len(data)} measurements for {current_date.date()}")
+                    else:
+                        logger.warning(f"Failed to update data for {current_date.date()}")
+                else:
+                    logger.warning(f"No data available for {current_date.date()}")
+
+                current_date += datetime.timedelta(days=1)
+
+        except Exception as e:
+            logger.error(f"Data update error: {e}")
+
+        return updated_count
+
+    def _check_data_exists_for_date(self, target_date: datetime.datetime) -> bool:
+        """Check if data exists for a specific date"""
+        try:
+            if not self.db_adapter:
+                return False
+
+            # Get data for the specific date
+            measurements = self.db_adapter.get_measurements_for_date(target_date)
+            return len(measurements) > 0
+
+        except Exception as e:
+            logger = get_logger(__name__)
+            logger.debug(f"Error checking data existence: {e}")
+            return False
+
+    def import_historical_data(
+        self,
+        start_date: datetime.datetime,
+        end_date: datetime.datetime,
+        skip_existing: bool = True,
+    ) -> int:
+        """
+        Import historical data for a date range
+
+        Args:
+            start_date: Start date for historical import
+            end_date: End date for historical import
+            skip_existing: Skip dates that already have data (default: True)
+
+        Returns:
+            Number of data points imported
+        """
+        logger.info(f"Starting historical data import from {start_date.date()} to {end_date.date()}")
+
+        total_imported = 0
+        current_date = start_date
+
+        while current_date <= end_date:
+            try:
+                # Check if data already exists for this date
+                if skip_existing and self._check_data_exists_for_date(current_date):
+                    logger.info(f"Data already exists for {current_date.date()}, skipping...")
+                    current_date += datetime.timedelta(days=1)
+                    continue
+
+                logger.info(f"Importing data for {current_date.date()}...")
+
+                # Fetch data for this date
+                data = self.fetch_water_data_for_date(current_date)
+
+                if data:
+                    # Save to database
+                    if self.save_to_database(data):
+                        total_imported += len(data)
+                        logger.info(f"Successfully imported {len(data)} data points for {current_date.date()}")
+                    else:
+                        logger.warning(f"Failed to save data for {current_date.date()}")
+                else:
+                    logger.warning(f"No data available for {current_date.date()}")
+
+                # Add small delay to be respectful to the API
+                time.sleep(1)
+
+            except Exception as e:
+                logger.error(f"Error importing data for {current_date.date()}: {e}")
+
+            current_date += datetime.timedelta(days=1)
+
+        logger.info(f"Historical import completed. Total data points imported: {total_imported}")
+        return total_imported
+
+
 # Main execution for standalone usage
 if __name__ == "__main__":
     import argparse
     import sys
 
-    # Configure basic logging for standalone usage
-    import logging
     logging.basicConfig(
         level=logging.INFO,
-        format='%(asctime)s - %(levelname)s - %(message)s',
-        handlers=[
-            logging.FileHandler('water_monitor.log'),
-            logging.StreamHandler()
-        ]
+        format="%(asctime)s - %(levelname)s - %(message)s",
+        handlers=[logging.FileHandler("water_monitor.log"), logging.StreamHandler()],
     )
 
     parser = argparse.ArgumentParser(description="Thailand Water Monitor")
@@ -504,10 +685,7 @@ if __name__ == "__main__":
     args = parser.parse_args()
 
     # Default SQLite configuration
-    db_config = {
-        'type': 'sqlite',
-        'connection_string': 'sqlite:///water_levels.db'
-    }
+    db_config = {"type": "sqlite", "connection_string": "sqlite:///water_levels.db"}
 
     try:
         scraper = EnhancedWaterMonitorScraper(db_config)

src/web_api.py

@@ -4,81 +4,56 @@ FastAPI web interface for water monitoring system
 """
 
 import asyncio
-import threading
-from datetime import datetime, timedelta
-from typing import List, Dict, Any, Optional
+import os
+import time
 from contextlib import asynccontextmanager
+from datetime import datetime, timedelta
+from threading import Lock
+from typing import Any, Dict, List
 
-from fastapi import FastAPI, HTTPException, BackgroundTasks, Depends
-from fastapi.responses import HTMLResponse, JSONResponse
-from fastapi.staticfiles import StaticFiles
+import requests
+from fastapi import BackgroundTasks, FastAPI, HTTPException, Query
 from fastapi.middleware.cors import CORSMiddleware
-from pydantic import BaseModel, Field
+from fastapi.responses import HTMLResponse
+from fastapi.staticfiles import StaticFiles
 
-from .water_scraper_v3 import EnhancedWaterMonitorScraper
 from .config import Config
-from .models import WaterMeasurement, StationInfo, ScrapingResult, StationCreateRequest, StationUpdateRequest, StationStatus
-from .health_check import HealthCheckManager, DatabaseHealthCheck, APIHealthCheck, MemoryHealthCheck
+from .health_check import APIHealthCheck, DatabaseHealthCheck, HealthCheckManager, MemoryHealthCheck
+from .logging_config import get_logger, setup_logging
 from .metrics import get_metrics_collector, increment_counter, set_gauge
-from .logging_config import setup_logging, get_logger
+from .postgres_history import PostgresHistory
+from .schemas import (
+    HealthResponse,
+    MeasurementResponse,
+    MetricsResponse,
+    ScrapingStatusResponse,
+    StationCreateModel,
+    StationResponse,
+    StationUpdateModel,
+)
+from .thaiwater import ThaiWaterClient
+from .water_scraper_v3 import EnhancedWaterMonitorScraper
 
 logger = get_logger(__name__)
 
-# Pydantic models for API responses
-class StationResponse(BaseModel):
-    station_id: int
-    station_code: str
-    thai_name: str
-    english_name: str
-    latitude: Optional[float] = None
-    longitude: Optional[float] = None
-    geohash: Optional[str] = None
-    status: str = "active"
+# Simple thread-safe TTL cache for PostgreSQL history queries
+HISTORY_CACHE: Dict[str, tuple] = {}
+HISTORY_CACHE_LOCK = Lock()
+HISTORY_TTL = 300  # 5 minutes
 
-class StationCreateModel(BaseModel):
-    station_code: str = Field(..., description="Station code (e.g., P.1, P.20)")
-    thai_name: str = Field(..., description="Thai name of the station")
-    english_name: str = Field(..., description="English name of the station")
-    latitude: Optional[float] = Field(None, ge=-90, le=90, description="Latitude coordinate")
-    longitude: Optional[float] = Field(None, ge=-180, le=180, description="Longitude coordinate")
-    geohash: Optional[str] = Field(None, description="Geohash for the location")
-    status: str = Field("active", description="Station status")
+FORECAST_CACHE: Dict[str, tuple] = {}
+FORECAST_CACHE_LOCK = Lock()
+FORECAST_TTL = 900  # 15 minutes
 
-class StationUpdateModel(BaseModel):
-    thai_name: Optional[str] = Field(None, description="Thai name of the station")
-    english_name: Optional[str] = Field(None, description="English name of the station")
-    latitude: Optional[float] = Field(None, ge=-90, le=90, description="Latitude coordinate")
-    longitude: Optional[float] = Field(None, ge=-180, le=180, description="Longitude coordinate")
-    geohash: Optional[str] = Field(None, description="Geohash for the location")
-    status: Optional[str] = Field(None, description="Station status")
+# Dashboard HTML is loaded once at import from src/static/dashboard.html.
+_DASHBOARD_HTML_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "static", "dashboard.html")
+try:
+    with open(_DASHBOARD_HTML_PATH, encoding="utf-8") as _dashboard_file:
+        DASHBOARD_HTML = _dashboard_file.read()
+except OSError as _dashboard_error:  # pragma: no cover - defensive fallback
+    logger.error(f"Could not load dashboard HTML: {_dashboard_error}")
+    DASHBOARD_HTML = "<h1>Northern Thailand Ping River Monitor API</h1><p>See <code>/docs</code>.</p>"
 
-class MeasurementResponse(BaseModel):
-    timestamp: datetime
-    station_code: str
-    station_name_en: str
-    station_name_th: str
-    water_level: float
-    discharge: float
-    discharge_percent: Optional[float] = None
-    status: str = "active"
-
-class HealthResponse(BaseModel):
-    overall_status: str
-    timestamp: str
-    checks: Dict[str, Dict[str, Any]]
-
-class MetricsResponse(BaseModel):
-    counters: Dict[str, float]
-    gauges: Dict[str, float]
-    histograms: Dict[str, Dict[str, float]]
-
-class ScrapingStatusResponse(BaseModel):
-    is_running: bool
-    last_run: Optional[datetime] = None
-    next_run: Optional[datetime] = None
-    total_runs: int = 0
-    successful_runs: int = 0
-    failed_runs: int = 0
 
 # Global application state
 app_state = {
@@ -91,10 +66,11 @@ app_state = {
         "successful_runs": 0,
         "failed_runs": 0,
         "last_run": None,
-        "next_run": None
-    }
+        "next_run": None,
+    },
 }
 
+
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     """Application lifespan manager"""
@@ -139,23 +115,31 @@ async def lifespan(app: FastAPI):
 
     logger.info("Water Monitor API shutdown complete")
 
+
 # Create FastAPI app
 app = FastAPI(
     title="Northern Thailand Ping River Monitor API",
     description="Real-time water level monitoring system for Northern Thailand's Ping River Basin stations",
-    version="3.1.2",
-    lifespan=lifespan
+    version="3.1.3",
+    lifespan=lifespan,
 )
+app.mount("/static", StaticFiles(directory=os.path.dirname(_DASHBOARD_HTML_PATH)), name="static")
 
-# Add CORS middleware
+# Add CORS middleware.
+# Origins come from CORS_ALLOW_ORIGINS (comma-separated). When none are configured
+# we fall back to a wildcard WITHOUT credentials (a safe, spec-valid combination);
+# credentials are only enabled when explicit origins are provided.
+_cors_origins = Config.CORS_ALLOW_ORIGINS or ["*"]
+_cors_allow_credentials = bool(Config.CORS_ALLOW_ORIGINS)
 app.add_middleware(
     CORSMiddleware,
-    allow_origins=["*"],  # Configure appropriately for production
-    allow_credentials=True,
+    allow_origins=_cors_origins,
+    allow_credentials=_cors_allow_credentials,
     allow_methods=["*"],
     allow_headers=["*"],
 )
 
+
 async def background_scraping_task():
     """Background task for periodic data scraping"""
     while True:
@@ -170,7 +154,9 @@ async def background_scraping_task():
                     start_time = datetime.now()
 
                     try:
-                        result = scraper.run_scraping_cycle()
+                        # run_scraping_cycle() does blocking network/DB I/O and time.sleep
+                        # retries; run it in a thread so it doesn't freeze the event loop.
+                        result = await asyncio.get_event_loop().run_in_executor(None, scraper.run_scraping_cycle)
 
                         # Update stats
                         app_state["scraping_stats"]["total_runs"] += 1
@@ -209,64 +195,15 @@ async def background_scraping_task():
             logger.error(f"Error in background scraping task: {e}")
             await asyncio.sleep(60)  # Wait a minute before retrying
 
+
 # API Routes
 
+
 @app.get("/", response_class=HTMLResponse)
 async def root():
     """Root endpoint with basic dashboard"""
-    html_content = """
-    <!DOCTYPE html>
-    <html>
-    <head>
-        <title>Northern Thailand Ping River Monitor</title>
-        <style>
-            body { font-family: Arial, sans-serif; margin: 40px; }
-            .header { color: #2c3e50; border-bottom: 2px solid #3498db; padding-bottom: 10px; }
-            .section { margin: 20px 0; padding: 15px; border: 1px solid #ddd; border-radius: 5px; }
-            .status-healthy { color: #27ae60; }
-            .status-degraded { color: #f39c12; }
-            .status-unhealthy { color: #e74c3c; }
-            .endpoint { background: #f8f9fa; padding: 10px; margin: 5px 0; border-radius: 3px; }
-            .endpoint code { color: #2c3e50; }
-        </style>
-    </head>
-    <body>
-        <div class="header">
-            <h1>🏔️ Northern Thailand Ping River Monitor API</h1>
-            <p>Real-time water level monitoring system for the Ping River Basin in Northern Thailand</p>
-        </div>
+    return HTMLResponse(content=DASHBOARD_HTML)
 
-        <div class="section">
-            <h2>📊 Quick Status</h2>
-            <p>API is running and monitoring 16 water stations along the Ping River</p>
-            <p>Coverage: From Chiang Dao to Nakhon Sawan</p>
-            <p>Data collection interval: Every hour</p>
-        </div>
-        
-        <div class="section">
-            <h2>🔗 API Endpoints</h2>
-            <div class="endpoint"><code>GET /health</code> - System health status</div>
-            <div class="endpoint"><code>GET /metrics</code> - Application metrics</div>
-            <div class="endpoint"><code>GET /stations</code> - List all monitoring stations</div>
-            <div class="endpoint"><code>POST /stations</code> - Add new monitoring station</div>
-            <div class="endpoint"><code>PUT /stations/{station_id}</code> - Update station information</div>
-            <div class="endpoint"><code>GET /measurements/latest</code> - Latest measurements</div>
-            <div class="endpoint"><code>GET /measurements/station/{station_code}</code> - Station-specific data</div>
-            <div class="endpoint"><code>POST /scrape/trigger</code> - Trigger manual data collection</div>
-            <div class="endpoint"><code>GET /scraping/status</code> - Scraping status</div>
-            <div class="endpoint"><code>GET /docs</code> - Interactive API documentation</div>
-        </div>
-        
-        <div class="section">
-            <h2>📈 Monitoring</h2>
-            <p>• Grafana dashboards available for data visualization</p>
-            <p>• Health checks monitor database, API, and system resources</p>
-            <p>• Metrics collection for performance monitoring</p>
-        </div>
-    </body>
-    </html>
-    """
-    return HTMLResponse(content=html_content)
 
 @app.get("/health", response_model=HealthResponse)
 async def get_health():
@@ -277,12 +214,13 @@ async def get_health():
     if not health_manager:
         raise HTTPException(status_code=503, detail="Health manager not initialized")
 
-    # Run health checks
-    results = health_manager.run_all_checks()
+    # Run health checks (populates state read by get_health_summary)
+    health_manager.run_all_checks()
     summary = health_manager.get_health_summary()
 
     return HealthResponse(**summary)
 
+
 @app.get("/metrics", response_model=MetricsResponse)
 async def get_metrics():
     """Get application metrics"""
@@ -293,6 +231,7 @@ async def get_metrics():
 
     return MetricsResponse(**metrics)
 
+
 @app.get("/stations", response_model=List[StationResponse])
 async def get_stations():
     """Get list of all monitoring stations"""
@@ -304,18 +243,21 @@ async def get_stations():
 
     stations = []
     for station_id, station_info in scraper.station_mapping.items():
-        stations.append(StationResponse(
+        stations.append(
+            StationResponse(
                 station_id=int(station_id),
                 station_code=station_info["code"],
                 thai_name=station_info["thai_name"],
                 english_name=station_info["english_name"],
                 latitude=station_info.get("latitude"),
                 longitude=station_info.get("longitude"),
-            status="active"
-        ))
+                status="active",
+            )
+        )
 
     return stations
 
+
 @app.post("/stations", response_model=StationResponse)
 async def create_station(station: StationCreateModel):
     """Create a new monitoring station"""
@@ -330,15 +272,19 @@ async def create_station(station: StationCreateModel):
         existing_ids = [int(sid) for sid in scraper.station_mapping.keys()]
         new_station_id = max(existing_ids) + 1 if existing_ids else 1
 
-        # Add to station mapping
-        scraper.station_mapping[str(new_station_id)] = {
-            'code': station.station_code,
-            'thai_name': station.thai_name,
-            'english_name': station.english_name,
-            'latitude': station.latitude,
-            'longitude': station.longitude,
-            'geohash': station.geohash
+        # Add to station mapping and persist
+        new_key = str(new_station_id)
+        scraper.station_mapping[new_key] = {
+            "code": station.station_code,
+            "thai_name": station.thai_name,
+            "english_name": station.english_name,
+            "latitude": station.latitude,
+            "longitude": station.longitude,
+            "geohash": station.geohash,
         }
+        if not scraper.save_stations():
+            scraper.station_mapping.pop(new_key, None)
+            raise HTTPException(status_code=500, detail="Failed to persist new station")
 
         logger.info(f"Created new station: {station.station_code} ({station.english_name})")
 
@@ -350,13 +296,16 @@ async def create_station(station: StationCreateModel):
             latitude=station.latitude,
             longitude=station.longitude,
             geohash=station.geohash,
-            status=station.status
+            status=station.status,
         )
 
+    except HTTPException:
+        raise
     except Exception as e:
         logger.error(f"Error creating station: {e}")
         raise HTTPException(status_code=500, detail=str(e))
 
+
 @app.put("/stations/{station_id}", response_model=StationResponse)
 async def update_station(station_id: int, updates: StationUpdateModel):
     """Update an existing monitoring station"""
@@ -372,36 +321,44 @@ async def update_station(station_id: int, updates: StationUpdateModel):
 
     try:
         station_info = scraper.station_mapping[station_key]
+        original = dict(station_info)  # snapshot for rollback if persistence fails
 
         # Update fields if provided
         if updates.thai_name is not None:
-            station_info['thai_name'] = updates.thai_name
+            station_info["thai_name"] = updates.thai_name
         if updates.english_name is not None:
-            station_info['english_name'] = updates.english_name
+            station_info["english_name"] = updates.english_name
         if updates.latitude is not None:
-            station_info['latitude'] = updates.latitude
+            station_info["latitude"] = updates.latitude
         if updates.longitude is not None:
-            station_info['longitude'] = updates.longitude
+            station_info["longitude"] = updates.longitude
         if updates.geohash is not None:
-            station_info['geohash'] = updates.geohash
+            station_info["geohash"] = updates.geohash
+
+        if not scraper.save_stations():
+            scraper.station_mapping[station_key] = original
+            raise HTTPException(status_code=500, detail="Failed to persist station update")
 
         logger.info(f"Updated station {station_id}: {station_info['code']}")
 
         return StationResponse(
             station_id=station_id,
-            station_code=station_info['code'],
-            thai_name=station_info['thai_name'],
-            english_name=station_info['english_name'],
-            latitude=station_info.get('latitude'),
-            longitude=station_info.get('longitude'),
-            geohash=station_info.get('geohash'),
-            status=updates.status or "active"
+            station_code=station_info["code"],
+            thai_name=station_info["thai_name"],
+            english_name=station_info["english_name"],
+            latitude=station_info.get("latitude"),
+            longitude=station_info.get("longitude"),
+            geohash=station_info.get("geohash"),
+            status=updates.status or "active",
         )
 
+    except HTTPException:
+        raise
     except Exception as e:
         logger.error(f"Error updating station {station_id}: {e}")
         raise HTTPException(status_code=500, detail=str(e))
 
+
 @app.delete("/stations/{station_id}")
 async def delete_station(station_id: int):
     """Delete a monitoring station"""
@@ -417,14 +374,22 @@ async def delete_station(station_id: int):
 
     try:
         station_info = scraper.station_mapping.pop(station_key)
+
+        if not scraper.save_stations():
+            scraper.station_mapping[station_key] = station_info  # restore
+            raise HTTPException(status_code=500, detail="Failed to persist station deletion")
+
         logger.info(f"Deleted station {station_id}: {station_info['code']}")
 
         return {"message": f"Station {station_info['code']} deleted successfully"}
 
+    except HTTPException:
+        raise
     except Exception as e:
         logger.error(f"Error deleting station {station_id}: {e}")
         raise HTTPException(status_code=500, detail=str(e))
 
+
 @app.get("/stations/{station_id}", response_model=StationResponse)
 async def get_station(station_id: int):
     """Get details of a specific monitoring station"""
@@ -442,15 +407,125 @@ async def get_station(station_id: int):
 
     return StationResponse(
         station_id=station_id,
-        station_code=station_info['code'],
-        thai_name=station_info['thai_name'],
-        english_name=station_info['english_name'],
-        latitude=station_info.get('latitude'),
-        longitude=station_info.get('longitude'),
-        geohash=station_info.get('geohash'),
-        status="active"
+        station_code=station_info["code"],
+        thai_name=station_info["thai_name"],
+        english_name=station_info["english_name"],
+        latitude=station_info.get("latitude"),
+        longitude=station_info.get("longitude"),
+        geohash=station_info.get("geohash"),
+        status="active",
     )
 
+
+def _to_measurement_response(measurement: Dict[str, Any]) -> MeasurementResponse:
+    """Map a raw measurement dict from a DB adapter to the API response model.
+
+    ``discharge`` is optional in the data (some stations report only level), so
+    it is read with ``.get`` rather than assumed present.
+    """
+    return MeasurementResponse(
+        timestamp=measurement["timestamp"],
+        station_code=measurement["station_code"],
+        station_name_en=measurement["station_name_en"],
+        station_name_th=measurement["station_name_th"],
+        water_level=measurement["water_level"],
+        discharge=measurement.get("discharge"),
+        discharge_percent=measurement.get("discharge_percent"),
+        status=measurement.get("status", "active"),
+    )
+
+
+@app.get("/sensors/thaiwater")
+async def get_thaiwater_sensors():
+    """Get current ThaiWater water-level sensors in the Ping basin."""
+    increment_counter("api_requests", labels={"endpoint": "thaiwater_sensors"})
+    try:
+        client = ThaiWaterClient(
+            api_key=Config.THAIWATER_API_KEY,
+            timeout=Config.REQUEST_TIMEOUT,
+        )
+        return await asyncio.to_thread(client.fetch_ping_sensors)
+    except RuntimeError as error:
+        raise HTTPException(status_code=503, detail=str(error))
+    except requests.RequestException as error:
+        logger.error(f"Error fetching ThaiWater sensors: {error}")
+        raise HTTPException(status_code=502, detail="ThaiWater API unavailable")
+
+
+@app.get("/measurements/history/{station_code}")
+async def get_postgres_history(
+    station_code: str,
+    hours: int = Query(168, ge=1),
+    limit: int = Query(50000, ge=1, le=100000),
+):
+    """Get historical measurements for a station from the configured database."""
+    cache_key = f"{station_code}:{hours}:{limit}"
+    now = time.monotonic()
+    with HISTORY_CACHE_LOCK:
+        cached = HISTORY_CACHE.get(cache_key)
+        if cached and now - cached[0] < HISTORY_TTL:
+            return cached[1]
+    try:
+        db_config = Config.get_database_config()
+        end_time = datetime.now()
+        if db_config["type"] == "postgresql":
+            history = PostgresHistory(db_config["connection_string"])
+            data = await asyncio.to_thread(
+                history.station_history,
+                station_code,
+                end_time - timedelta(hours=hours),
+                end_time,
+                limit,
+            )
+        else:
+            scraper = app_state["scraper"]
+            if not scraper or not scraper.db_adapter:
+                raise RuntimeError("Database not available")
+            rows = await asyncio.to_thread(
+                scraper.db_adapter.get_measurements_by_timerange,
+                end_time - timedelta(hours=hours),
+                end_time,
+                [station_code],
+            )
+            # adapter returns newest-first; keep the newest `limit` rows, chart wants ascending
+            data = list(reversed(rows[:limit]))
+        with HISTORY_CACHE_LOCK:
+            HISTORY_CACHE[cache_key] = (now, data)
+        return data
+    except RuntimeError as error:
+        raise HTTPException(status_code=503, detail=str(error))
+    except Exception as error:
+        logger.error(f"Error fetching measurement history: {error}")
+        raise HTTPException(status_code=502, detail="Measurement history unavailable")
+
+
+@app.get("/forecast")
+async def get_flood_forecasts():
+    """Flood-risk forecasts per station for the 6/12/24 h horizons."""
+    increment_counter("api_requests", labels={"endpoint": "forecast"})
+    now = time.monotonic()
+    with FORECAST_CACHE_LOCK:
+        cached = FORECAST_CACHE.get("all")
+        if cached and now - cached[0] < FORECAST_TTL:
+            return cached[1]
+    try:
+        from .ml.predict import get_latest_forecasts
+    except ImportError as error:
+        raise HTTPException(status_code=503, detail=f"Forecasting unavailable: {error}")
+    try:
+        data = await asyncio.to_thread(get_latest_forecasts)
+    except FileNotFoundError:
+        raise HTTPException(status_code=503, detail="No trained flood models found")
+    except RuntimeError as error:
+        raise HTTPException(status_code=503, detail=str(error))
+    except Exception as error:
+        logger.error(f"Error computing flood forecasts: {error}")
+        raise HTTPException(status_code=502, detail="Flood forecast unavailable")
+    with FORECAST_CACHE_LOCK:
+        FORECAST_CACHE["all"] = (now, data)
+    return data
+
+
 @app.get("/measurements/latest", response_model=List[MeasurementResponse])
 async def get_latest_measurements(limit: int = 100):
     """Get latest measurements from all stations"""
@@ -463,31 +538,15 @@ async def get_latest_measurements(limit: int = 100):
     try:
         measurements = scraper.get_latest_data(limit=limit)
 
-        response = []
-        for measurement in measurements:
-            response.append(MeasurementResponse(
-                timestamp=measurement["timestamp"],
-                station_code=measurement["station_code"],
-                station_name_en=measurement["station_name_en"],
-                station_name_th=measurement["station_name_th"],
-                water_level=measurement["water_level"],
-                discharge=measurement["discharge"],
-                discharge_percent=measurement.get("discharge_percent"),
-                status=measurement.get("status", "active")
-            ))
-        
-        return response
+        return [_to_measurement_response(m) for m in measurements]
 
     except Exception as e:
         logger.error(f"Error fetching latest measurements: {e}")
         raise HTTPException(status_code=500, detail=str(e))
 
+
 @app.get("/measurements/station/{station_code}", response_model=List[MeasurementResponse])
-async def get_station_measurements(
-    station_code: str, 
-    hours: int = 24,
-    limit: int = 1000
-):
+async def get_station_measurements(station_code: str, hours: int = 24, limit: int = 1000):
     """Get measurements for a specific station"""
     increment_counter("api_requests", labels={"endpoint": "measurements_station"})
 
@@ -507,25 +566,13 @@ async def get_station_measurements(
         # Limit results
         measurements = measurements[:limit]
 
-        response = []
-        for measurement in measurements:
-            response.append(MeasurementResponse(
-                timestamp=measurement["timestamp"],
-                station_code=measurement["station_code"],
-                station_name_en=measurement["station_name_en"],
-                station_name_th=measurement["station_name_th"],
-                water_level=measurement["water_level"],
-                discharge=measurement["discharge"],
-                discharge_percent=measurement.get("discharge_percent"),
-                status=measurement.get("status", "active")
-            ))
-        
-        return response
+        return [_to_measurement_response(m) for m in measurements]
 
     except Exception as e:
         logger.error(f"Error fetching station measurements: {e}")
         raise HTTPException(status_code=500, detail=str(e))
 
+
 @app.post("/scrape/trigger")
 async def trigger_scraping(background_tasks: BackgroundTasks):
     """Trigger manual data scraping"""
@@ -568,6 +615,7 @@ async def trigger_scraping(background_tasks: BackgroundTasks):
 
     return {"message": "Scraping triggered", "status": "started"}
 
+
 @app.get("/scraping/status", response_model=ScrapingStatusResponse)
 async def get_scraping_status():
     """Get current scraping status"""
@@ -581,9 +629,10 @@ async def get_scraping_status():
         next_run=stats["next_run"],
         total_runs=stats["total_runs"],
         successful_runs=stats["successful_runs"],
-        failed_runs=stats["failed_runs"]
+        failed_runs=stats["failed_runs"],
     )
 
+
 @app.get("/config")
 async def get_config():
     """Get current configuration (sensitive data masked)"""
@@ -593,12 +642,13 @@ async def get_config():
 
     # Mask sensitive information
     for key in config:
-        if 'password' in key.lower() or 'secret' in key.lower():
+        if "password" in key.lower() or "secret" in key.lower():
             if config[key]:
-                config[key] = '*' * 8
+                config[key] = "*" * 8
 
     return config
 
+
 if __name__ == "__main__":
     import uvicorn
 
@@ -607,14 +657,8 @@ if __name__ == "__main__":
         log_level=Config.LOG_LEVEL,
         log_file=Config.LOG_FILE,
         enable_console=True,
-        enable_colors=True
+        enable_colors=True,
     )
 
     # Run the API server
-    uvicorn.run(
-        "web_api:app",
-        host="0.0.0.0",
-        port=8000,
-        reload=False,
-        log_config=None  # Use our custom logging
-    )
+    uvicorn.run("web_api:app", host="0.0.0.0", port=8000, reload=False, log_config=None)  # Use our custom logging

tests/conftest.py

@@ -0,0 +1,12 @@
+"""Shared pytest configuration.
+
+Ensures the repository root is on sys.path so tests can import the ``src``
+package regardless of the working directory pytest is invoked from.
+"""
+
+import os
+import sys
+
+REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+if REPO_ROOT not in sys.path:
+    sys.path.insert(0, REPO_ROOT)

tests/test_alerting.py

@@ -0,0 +1,383 @@
+#!/usr/bin/env python3
+"""
+Comprehensive tests for the alerting system
+Tests both zone-based and rate-of-change alerts
+"""
+
+import sys
+import os
+import datetime
+import sqlite3
+import time
+import gc
+
+# Add src directory to path
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from src.alerting import WaterLevelAlertSystem, AlertLevel
+from src.database_adapters import create_database_adapter
+
+
+def setup_test_database(test_name='default'):
+    """Create a test database with sample data"""
+    db_path = f'test_alerts_{test_name}.db'
+
+    # Remove existing test database
+    if os.path.exists(db_path):
+        try:
+            os.remove(db_path)
+        except PermissionError:
+            # If locked, use a different name with timestamp
+            import random
+            db_path = f'test_alerts_{test_name}_{random.randint(1000, 9999)}.db'
+
+    # Create new database
+    conn = sqlite3.connect(db_path)
+    cursor = conn.cursor()
+
+    # Create stations table
+    cursor.execute("""
+    CREATE TABLE stations (
+        id INTEGER PRIMARY KEY,
+        station_code TEXT NOT NULL UNIQUE,
+        english_name TEXT,
+        thai_name TEXT,
+        latitude REAL,
+        longitude REAL,
+        basin TEXT,
+        province TEXT,
+        status TEXT DEFAULT 'active',
+        created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+    )
+    """)
+
+    # Create water_measurements table
+    cursor.execute("""
+    CREATE TABLE water_measurements (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        timestamp DATETIME NOT NULL,
+        station_id INTEGER NOT NULL,
+        water_level REAL NOT NULL,
+        discharge REAL,
+        discharge_percent REAL,
+        status TEXT DEFAULT 'active',
+        created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+        FOREIGN KEY (station_id) REFERENCES stations (id)
+    )
+    """)
+
+    # Insert P.1 station (id=8 to match existing data)
+    cursor.execute("""
+    INSERT INTO stations (id, station_code, english_name, thai_name, basin, province)
+    VALUES (8, 'P.1', 'Nawarat Bridge', 'สะพานนวรัฐ', 'Ping', 'Chiang Mai')
+    """)
+
+    conn.commit()
+    conn.close()
+
+    return db_path
+
+
+def test_zone_level_alerts():
+    """Test that zone-based alerts trigger correctly"""
+    print("="*70)
+    print("TEST 1: Zone-Based Water Level Alerts")
+    print("="*70)
+
+    db_path = setup_test_database('zone_tests')
+
+    # Test cases for P.1 zone thresholds
+    test_cases = [
+        (2.5, None, "Below all zones"),
+        (3.7, AlertLevel.INFO, "Zone 1"),
+        (3.9, AlertLevel.INFO, "Zone 2"),
+        (4.0, AlertLevel.WARNING, "Zone 3"),
+        (4.2, AlertLevel.WARNING, "Zone 5"),
+        (4.3, AlertLevel.CRITICAL, "Zone 6"),
+        (4.6, AlertLevel.CRITICAL, "Zone 7"),
+        (4.8, AlertLevel.EMERGENCY, "Zone 8/NewEdge"),
+        (5.0, AlertLevel.EMERGENCY, "Above all zones"),
+    ]
+
+    print("\nTesting P.1 (Nawarat Bridge) zone thresholds:")
+    print("-" * 70)
+
+    passed = 0
+    failed = 0
+
+    for water_level, expected_level, zone_description in test_cases:
+        # Insert test data
+        conn = sqlite3.connect(db_path)
+        cursor = conn.cursor()
+        cursor.execute("DELETE FROM water_measurements")
+
+        current_time = datetime.datetime.now()
+        cursor.execute("""
+        INSERT INTO water_measurements (timestamp, station_id, water_level, discharge)
+        VALUES (?, 8, ?, 350.0)
+        """, (current_time, water_level))
+
+        conn.commit()
+        conn.close()
+
+        # Check alerts
+        alerting = WaterLevelAlertSystem()
+        alerting.db_adapter = create_database_adapter('sqlite', connection_string=f'sqlite:///{db_path}')
+        alerting.db_adapter.connect()
+
+        alerts = alerting.check_water_levels()
+
+        # Verify result
+        if expected_level is None:
+            # Should not trigger any alert
+            if len(alerts) == 0:
+                print(f"[PASS] {water_level:.1f}m: {zone_description} - No alert")
+                passed += 1
+            else:
+                print(f"[FAIL] {water_level:.1f}m: {zone_description} - Unexpected alert")
+                failed += 1
+        else:
+            # Should trigger alert with specific level
+            if len(alerts) > 0 and alerts[0].level == expected_level:
+                print(f"[PASS] {water_level:.1f}m: {zone_description} - {expected_level.value.upper()} alert")
+                passed += 1
+            elif len(alerts) == 0:
+                print(f"[FAIL] {water_level:.1f}m: {zone_description} - No alert triggered")
+                failed += 1
+            else:
+                print(f"[FAIL] {water_level:.1f}m: {zone_description} - Wrong alert level: {alerts[0].level.value}")
+                failed += 1
+
+    print("-" * 70)
+    print(f"Zone Alert Tests: {passed} passed, {failed} failed")
+
+    # Cleanup - force garbage collection and wait briefly before removing file
+    gc.collect()
+    time.sleep(0.5)
+    try:
+        os.remove(db_path)
+    except PermissionError:
+        print(f"Warning: Could not remove test database {db_path}")
+
+    return failed == 0
+
+
+def test_rate_of_change_alerts():
+    """Test that rate-of-change alerts trigger correctly"""
+    print("\n" + "="*70)
+    print("TEST 2: Rate-of-Change Water Level Alerts")
+    print("="*70)
+
+    db_path = setup_test_database('rate_tests')
+
+    # Test cases: (initial_level, final_level, hours_elapsed, expected_alert_level, description)
+    test_cases = [
+        (3.0, 3.1, 3.0, None, "Slow rise (0.03m/h)"),
+        (3.0, 3.5, 3.0, AlertLevel.WARNING, "Moderate rise (0.17m/h)"),
+        (3.0, 3.8, 3.0, AlertLevel.CRITICAL, "Rapid rise (0.27m/h)"),
+        (3.0, 4.2, 3.0, AlertLevel.EMERGENCY, "Very rapid rise (0.40m/h)"),
+        (4.0, 3.5, 3.0, None, "Falling water (negative rate)"),
+    ]
+
+    print("\nTesting P.1 rate-of-change thresholds:")
+    print("  Warning: 0.15 m/h (15 cm/h)")
+    print("  Critical: 0.25 m/h (25 cm/h)")
+    print("  Emergency: 0.40 m/h (40 cm/h)")
+    print("-" * 70)
+
+    passed = 0
+    failed = 0
+
+    for initial_level, final_level, hours, expected_level, description in test_cases:
+        # Insert test data simulating water level change over time
+        conn = sqlite3.connect(db_path)
+        cursor = conn.cursor()
+        cursor.execute("DELETE FROM water_measurements")
+
+        current_time = datetime.datetime.now()
+        start_time = current_time - datetime.timedelta(hours=hours)
+
+        # Insert initial measurement
+        cursor.execute("""
+        INSERT INTO water_measurements (timestamp, station_id, water_level, discharge)
+        VALUES (?, 8, ?, 350.0)
+        """, (start_time, initial_level))
+
+        # Insert final measurement
+        cursor.execute("""
+        INSERT INTO water_measurements (timestamp, station_id, water_level, discharge)
+        VALUES (?, 8, ?, 380.0)
+        """, (current_time, final_level))
+
+        conn.commit()
+        conn.close()
+
+        # Check rate-of-change alerts
+        alerting = WaterLevelAlertSystem()
+        alerting.db_adapter = create_database_adapter('sqlite', connection_string=f'sqlite:///{db_path}')
+        alerting.db_adapter.connect()
+
+        rate_alerts = alerting.check_rate_of_change(lookback_hours=int(hours) + 1)
+
+        # Calculate actual rate for display
+        level_change = final_level - initial_level
+        rate = level_change / hours if hours > 0 else 0
+
+        # Verify result
+        if expected_level is None:
+            # Should not trigger any alert
+            if len(rate_alerts) == 0:
+                print(f"[PASS] {rate:+.2f}m/h: {description} - No alert")
+                passed += 1
+            else:
+                print(f"[FAIL] {rate:+.2f}m/h: {description} - Unexpected alert")
+                print(f"   Alert: {rate_alerts[0].alert_type} - {rate_alerts[0].level.value}")
+                failed += 1
+        else:
+            # Should trigger alert with specific level
+            if len(rate_alerts) > 0 and rate_alerts[0].level == expected_level:
+                print(f"[PASS] {rate:+.2f}m/h: {description} - {expected_level.value.upper()} alert")
+                print(f"   Message: {rate_alerts[0].message}")
+                passed += 1
+            elif len(rate_alerts) == 0:
+                print(f"[FAIL] {rate:+.2f}m/h: {description} - No alert triggered")
+                failed += 1
+            else:
+                print(f"[FAIL] {rate:+.2f}m/h: {description} - Wrong alert level: {rate_alerts[0].level.value}")
+                failed += 1
+
+    print("-" * 70)
+    print(f"Rate-of-Change Tests: {passed} passed, {failed} failed")
+
+    # Cleanup - force garbage collection and wait briefly before removing file
+    gc.collect()
+    time.sleep(0.5)
+    try:
+        os.remove(db_path)
+    except PermissionError:
+        print(f"Warning: Could not remove test database {db_path}")
+
+    return failed == 0
+
+
+def test_combined_alerts():
+    """Test scenario where both zone and rate-of-change alerts trigger"""
+    print("\n" + "="*70)
+    print("TEST 3: Combined Zone + Rate-of-Change Alerts")
+    print("="*70)
+
+    db_path = setup_test_database('combined_tests')
+
+    print("\nScenario: Water rising rapidly from 3.5m to 4.5m over 3 hours")
+    print("  Expected: Both Zone 7 alert AND Critical rate-of-change alert")
+    print("-" * 70)
+
+    # Insert test data
+    conn = sqlite3.connect(db_path)
+    cursor = conn.cursor()
+
+    current_time = datetime.datetime.now()
+    start_time = current_time - datetime.timedelta(hours=3)
+
+    # Water rising from 3.5m to 4.5m over 3 hours (0.33 m/h - Critical rate)
+    cursor.execute("""
+    INSERT INTO water_measurements (timestamp, station_id, water_level, discharge)
+    VALUES (?, 8, 3.5, 350.0)
+    """, (start_time,))
+
+    cursor.execute("""
+    INSERT INTO water_measurements (timestamp, station_id, water_level, discharge)
+    VALUES (?, 8, 4.5, 450.0)
+    """, (current_time,))
+
+    conn.commit()
+    conn.close()
+
+    # Check both types of alerts
+    alerting = WaterLevelAlertSystem()
+    alerting.db_adapter = create_database_adapter('sqlite', connection_string=f'sqlite:///{db_path}')
+    alerting.db_adapter.connect()
+
+    zone_alerts = alerting.check_water_levels()
+    rate_alerts = alerting.check_rate_of_change(lookback_hours=4)
+
+    all_alerts = zone_alerts + rate_alerts
+
+    print(f"\nTotal alerts triggered: {len(all_alerts)}")
+
+    zone_alert_found = False
+    rate_alert_found = False
+
+    for alert in all_alerts:
+        print(f"\n  Alert Type: {alert.alert_type}")
+        print(f"     Severity: {alert.level.value.upper()}")
+        print(f"     Water Level: {alert.water_level:.2f}m")
+        if alert.message:
+            print(f"     Details: {alert.message}")
+
+        if "Zone" in alert.alert_type:
+            zone_alert_found = True
+        if "Rise" in alert.alert_type or "rate" in alert.alert_type.lower():
+            rate_alert_found = True
+
+    print("-" * 70)
+
+    if zone_alert_found and rate_alert_found:
+        print("[PASS] Combined Alert Test - Both alert types triggered")
+        success = True
+    else:
+        print("[FAIL] Combined Alert Test")
+        if not zone_alert_found:
+            print("   Missing: Zone-based alert")
+        if not rate_alert_found:
+            print("   Missing: Rate-of-change alert")
+        success = False
+
+    # Cleanup - force garbage collection and wait briefly before removing file
+    gc.collect()
+    time.sleep(0.5)
+    try:
+        os.remove(db_path)
+    except PermissionError:
+        print(f"Warning: Could not remove test database {db_path}")
+
+    return success
+
+
+def main():
+    """Run all alert tests"""
+    print("\n" + "="*70)
+    print("WATER LEVEL ALERTING SYSTEM - COMPREHENSIVE TESTS")
+    print("="*70)
+
+    results = []
+
+    # Run tests
+    results.append(("Zone-Based Alerts", test_zone_level_alerts()))
+    results.append(("Rate-of-Change Alerts", test_rate_of_change_alerts()))
+    results.append(("Combined Alerts", test_combined_alerts()))
+
+    # Summary
+    print("\n" + "="*70)
+    print("TEST SUMMARY")
+    print("="*70)
+
+    all_passed = True
+    for test_name, passed in results:
+        status = "PASS" if passed else "FAIL"
+        print(f"{test_name}: [{status}]")
+        if not passed:
+            all_passed = False
+
+    print("="*70)
+
+    if all_passed:
+        print("\nAll tests PASSED!")
+        return 0
+    else:
+        print("\nSome tests FAILED!")
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/test_dashboard.py

@@ -0,0 +1,46 @@
+from pathlib import Path
+
+
+DASHBOARD_PATH = Path(__file__).parents[1] / "src" / "static" / "dashboard.html"
+
+
+def test_dashboard_contains_live_map_and_flow_visualization():
+    html = DASHBOARD_PATH.read_text(encoding="utf-8")
+
+    assert "id=\"station-map\"" in html
+    assert "id=\"river-flow\"" in html
+    assert "fetch('/stations')" in html or 'fetch("/stations")' in html
+    assert "fetch('/measurements/latest" in html or 'fetch(\"/measurements/latest' in html
+    assert "leaflet" in html.lower()
+
+
+def test_dashboard_explains_flow_legend_and_refresh():
+    html = DASHBOARD_PATH.read_text(encoding="utf-8")
+
+    assert "Flow status" in html
+    assert "Last updated" in html
+    assert "Refresh" in html
+
+
+def test_dashboard_uses_mapped_river_network_instead_of_station_connections():
+    html = DASHBOARD_PATH.read_text(encoding="utf-8")
+    river_network = DASHBOARD_PATH.with_name("ping-river-network.geojson")
+
+    assert river_network.exists()
+    assert "fetch('/static/ping-river-network.geojson')" in html
+    assert "mainBasin.map" not in html
+
+
+def test_dashboard_loads_additional_thaiwater_sensors():
+    html = DASHBOARD_PATH.read_text(encoding="utf-8")
+
+    assert "fetch('/sensors/thaiwater')" in html
+    assert "Additional ThaiWater sensor" in html
+
+
+def test_dashboard_loads_postgresql_history_chart():
+    html = DASHBOARD_PATH.read_text(encoding="utf-8")
+
+    assert "PostgreSQL history" in html
+    assert "/measurements/history/" in html
+    assert "history-chart" in html

tests/test_flood_forecast.py

@@ -0,0 +1,246 @@
+"""Tests for the flood forecast ML package. Synthetic data only -- no DB/network."""
+
+import datetime
+from typing import Dict, List, Optional
+
+import joblib
+import numpy as np
+import pandas as pd
+import pytest
+
+from src.ml import features, predict, train
+
+
+def make_synth(
+    n_hours: int,
+    stations: List[str],
+    seed: int = 0,
+    start: str = "2020-01-01",
+    pulses: Optional[Dict[str, List[tuple]]] = None,
+    missing_patches: Optional[Dict[str, List[tuple]]] = None,
+) -> pd.DataFrame:
+    """Generate a synthetic long measurement frame with smooth levels, flood pulses,
+    and optional missing patches, for `stations` over `n_hours` hourly steps.
+
+    pulses: {station: [(start_hour, width_hours, peak_add), ...]}
+    missing_patches: {station: [(start_hour, length_hours), ...]}
+    """
+    rng = np.random.default_rng(seed)
+    idx = pd.date_range(start, periods=n_hours, freq="h")
+    rows = []
+    for station in stations:
+        base = 1.5 + 0.1 * np.sin(np.linspace(0, 6 * np.pi, n_hours))
+        noise = rng.normal(0, 0.02, n_hours)
+        level = base + noise
+        for pulse_start, width, peak_add in (pulses or {}).get(station, []):
+            t = np.arange(n_hours)
+            bump = peak_add * np.exp(-0.5 * ((t - (pulse_start + width / 2)) / (width / 4)) ** 2)
+            level = level + bump
+        discharge = 20.0 * level + rng.normal(0, 1.0, n_hours)
+
+        missing = np.zeros(n_hours, dtype=bool)
+        for patch_start, length in (missing_patches or {}).get(station, []):
+            missing[patch_start : patch_start + length] = True
+
+        for i in range(n_hours):
+            if missing[i]:
+                continue
+            rows.append(
+                {
+                    "timestamp": idx[i],
+                    "station_code": station,
+                    "water_level": round(float(level[i]), 3),
+                    "discharge": round(float(discharge[i]), 2),
+                }
+            )
+    return pd.DataFrame(rows)
+
+
+def test_no_future_leakage():
+    stations = ["P.1", "P.20"]
+    df_a = make_synth(200, stations, seed=1, pulses={"P.1": [(150, 10, 3.0)]})
+    grid_a = features.make_hourly_grid(df_a)
+    feat_a = features.build_features(grid_a, "P.1")
+
+    t0 = grid_a.observed.index[120]
+
+    df_b = df_a.copy()
+    future_mask = df_b["timestamp"] > t0
+    df_b.loc[future_mask, "water_level"] = df_b.loc[future_mask, "water_level"] + 50.0
+    df_b.loc[future_mask, "discharge"] = df_b.loc[future_mask, "discharge"] + 500.0
+    grid_b = features.make_hourly_grid(df_b)
+    feat_b = features.build_features(grid_b, "P.1")
+
+    past_a = feat_a.loc[feat_a.index <= t0]
+    past_b = feat_b.loc[feat_b.index <= t0]
+    pd.testing.assert_frame_equal(past_a, past_b)
+
+
+def test_label_alignment():
+    idx = pd.date_range("2020-01-01", periods=12, freq="h")
+    warn_thr, _danger_thr = features.get_thresholds("P.1")
+    peak = warn_thr + 0.3
+    levels = [1.0, 1.0, 1.0, 1.0, 1.0, peak, peak, 1.0, 1.0, 1.0, 1.0, 1.0]
+    df = pd.DataFrame(
+        {
+            "timestamp": idx,
+            "station_code": "P.1",
+            "water_level": levels,
+            "discharge": [20.0 * lvl for lvl in levels],
+        }
+    )
+    grid = features.make_hourly_grid(df)
+    labels = features.build_labels(grid, "P.1", horizons=(6,))
+
+    # Level crosses the warning threshold at t=5. A 6h forward window (t, t+6]
+    # first includes t=5 for t=0 .. t=4 (inclusive), so exceed_warn_6 should be
+    # 1 for t=0..4 and not (necessarily) for later rows in this hand-built series.
+    for t in range(5):
+        assert labels["exceed_warn_6"].iloc[t] == 1.0, f"t={t} expected warn exceedance"
+
+    # max_level_6 at t=0 covers hours 1..6 -> includes the peak.
+    assert labels["max_level_6"].iloc[0] == pytest.approx(peak)
+
+
+def test_label_coverage_gate():
+    n = 40
+    idx = pd.date_range("2020-01-01", periods=n, freq="h")
+    levels = [1.0] * n
+    df = pd.DataFrame(
+        {"timestamp": idx, "station_code": "P.1", "water_level": levels, "discharge": [20.0] * n}
+    )
+    # Drop 70% of a future window (hours 21..26) for the row at t=20, no exceedance in it.
+    df_missing = df[~df["timestamp"].isin(idx[21:26])].copy()
+    grid = features.make_hourly_grid(df_missing)
+    labels = features.build_labels(grid, "P.1", horizons=(6,))
+    t20 = idx[20]
+    assert pd.isna(labels.loc[t20, "exceed_warn_6"])
+
+    # Same sparse window, but WITH an observed exceedance inside it -> must be 1, not NaN.
+    df_with_peak = df_missing.copy()
+    peak_row = pd.DataFrame(
+        [{"timestamp": idx[22], "station_code": "P.1", "water_level": 5.0, "discharge": 100.0}]
+    )
+    df_with_peak = pd.concat([df_with_peak, peak_row], ignore_index=True)
+    grid2 = features.make_hourly_grid(df_with_peak)
+    labels2 = features.build_labels(grid2, "P.1", horizons=(6,))
+    assert labels2.loc[t20, "exceed_warn_6"] == 1.0
+
+
+def test_ffill_and_staleness():
+    n = 20
+    idx = pd.date_range("2020-01-01", periods=n, freq="h")
+    df = pd.DataFrame(
+        {
+            "timestamp": idx,
+            "station_code": "P.1",
+            "water_level": [1.0 + 0.01 * i for i in range(n)],
+            "discharge": [20.0] * n,
+        }
+    )
+    # Small gap: drop hours 5,6 (2h gap).
+    df_small_gap = df[~df["timestamp"].isin(idx[5:7])].copy()
+    grid = features.make_hourly_grid(df_small_gap)
+    feat = features.build_features(grid, "P.1")
+    assert feat.loc[idx[5], "obs_age_h"] == pytest.approx(1.0)
+    assert feat.loc[idx[6], "obs_age_h"] == pytest.approx(2.0)
+
+    # Large gap: drop hours 5..9 (5h gap) -> rows with age>3 dropped (NaN).
+    df_big_gap = df[~df["timestamp"].isin(idx[5:10])].copy()
+    grid2 = features.make_hourly_grid(df_big_gap)
+    feat2 = features.build_features(grid2, "P.1")
+    assert feat2.loc[idx[8], "obs_age_h"] != feat2.loc[idx[8], "obs_age_h"]  # NaN
+    assert feat2.loc[idx[9], "obs_age_h"] != feat2.loc[idx[9], "obs_age_h"]  # NaN
+    assert feat2.loc[idx[7], "obs_age_h"] == pytest.approx(3.0)
+
+
+_FORECAST_KEYS = {
+    "station_code",
+    "horizon_hours",
+    "p_warning",
+    "p_danger",
+    "predicted_max_level",
+    "current_level",
+    "as_of",
+    "model_version",
+    "trained_at",
+    "source",
+    "threshold_warning",
+    "threshold_danger",
+}
+
+
+def _assert_valid_forecast_row(row: dict) -> None:
+    # "stages" is optional: model rows for stations in features.FLOOD_STAGES carry
+    # per-inundation-stage exceedance probabilities (currently P.1 only).
+    assert _FORECAST_KEYS <= set(row.keys())
+    assert set(row.keys()) - _FORECAST_KEYS <= {"stages"}
+    assert 0.0 <= row["p_warning"] <= 1.0
+    assert 0.0 <= row["p_danger"] <= 1.0
+    assert row["p_danger"] <= row["p_warning"]
+    assert row["predicted_max_level"] >= row["current_level"]
+    assert row["source"] in ("model", "heuristic")
+    for stage in row.get("stages", []):
+        assert 0.0 <= stage["p_exceed"] <= 1.0
+        assert stage["level"] > 0
+
+
+def test_train_smoke_and_roundtrip(tmp_path):
+    # Include every station P.1's feature set actually references (its UPSTREAM_LEADS)
+    # so no upstream column is entirely NaN -- HistGradientBoosting's binning step
+    # cannot fit a fully-degenerate column (see train._safe_fit).
+    upstream = [code for code, _lead in features.UPSTREAM_LEADS["P.1"]]
+    data_stations = ["P.1"] + upstream
+    target_stations = ["P.1", "P.20"]
+    n = 700
+    pulses = {station: [(start, 20, 2.0) for start in range(50, n - 50, 110)] for station in data_stations}
+    df = make_synth(n, data_stations, seed=7, pulses=pulses)
+
+    metrics = train.train_all(
+        df, target_stations, models_dir=tmp_path, skip_eval=True, hgb_overrides={"max_iter": 20}
+    )
+    assert metrics["stations"]["P.1"]["status"] == "trained"
+    assert metrics["stations"]["P.20"]["status"] == "trained"
+    assert (tmp_path / "flood_P.1.joblib").exists()
+    assert (tmp_path / "metrics.json").exists()
+
+    readings_by_station = {
+        code: group[["timestamp", "water_level", "discharge"]].to_dict("records")
+        for code, group in df.groupby("station_code")
+        if code in target_stations
+    }
+    now = df["timestamp"].max()
+    forecasts = predict.get_forecasts(readings_by_station, models_dir=tmp_path, now=now)
+
+    assert len(forecasts) > 0
+    for row in forecasts:
+        _assert_valid_forecast_row(row)
+    assert any(row["source"] == "model" for row in forecasts)
+
+
+def test_heuristic_fallback(tmp_path):
+    df = make_synth(50, ["P.1"], seed=3)
+    readings_by_station = {"P.1": df[["timestamp", "water_level", "discharge"]].to_dict("records")}
+    now = df["timestamp"].max()
+
+    forecasts = predict.get_forecasts(readings_by_station, models_dir=tmp_path, now=now)
+
+    assert len(forecasts) == len(predict.DEFAULT_HORIZONS)
+    for row in forecasts:
+        _assert_valid_forecast_row(row)
+        assert row["source"] == "heuristic"
+        assert row["model_version"] == "heuristic-v1"
+        assert row["trained_at"] is None
+
+
+def test_feature_name_stability(tmp_path):
+    upstream = [code for code, _lead in features.UPSTREAM_LEADS["P.1"]]
+    data_stations = ["P.1"] + upstream
+    df = make_synth(300, data_stations, seed=11, pulses={"P.1": [(100, 20, 2.0)]})
+    train.train_all(df, ["P.1"], models_dir=tmp_path, skip_eval=True, hgb_overrides={"max_iter": 10})
+    # Safe: loading the bundle this same test just wrote to tmp_path, not an external file.
+    bundle = joblib.load(tmp_path / "flood_P.1.joblib")
+
+    grid = features.make_hourly_grid(df)
+    fresh_columns = list(features.build_features(grid, "P.1").columns)
+    assert fresh_columns == bundle["feature_names"]

tests/test_integration.py

@@ -165,7 +165,7 @@ def test_logging():
 
 def main():
     """Run all tests"""
-    print("🧪 Running integration tests for Northern Thailand Ping River Monitor v3.1.2")
+    print("🧪 Running integration tests for Northern Thailand Ping River Monitor v3.1.3")
     print("=" * 60)
     
     tests = [

tests/test_matrix_formatting.py

@@ -0,0 +1,98 @@
+"""Assert-based tests for Matrix message formatting.
+
+Matrix clients only render formatting from an HTML ``formatted_body``; Markdown
+in the plain ``body`` shows as literal characters. These tests lock in that the
+notifier emits real HTML plus a clean plain-text fallback, and that untrusted
+station data is HTML-escaped.
+"""
+
+import datetime
+
+from src.alerting import AlertLevel, MatrixNotifier, WaterAlert, markdown_to_matrix_html, strip_markdown
+
+
+class _FakeResponse:
+    def raise_for_status(self):
+        pass
+
+    def json(self):
+        return {"event_id": "$test"}
+
+
+def _notifier_capturing(captured):
+    """A MatrixNotifier whose HTTP PUT records the JSON payload into ``captured``."""
+    notifier = MatrixNotifier("https://hs.example", "token", "!room:hs.example")
+
+    def fake_put(url, headers=None, json=None, timeout=None):
+        captured.update(json)
+        return _FakeResponse()
+
+    notifier.session.put = fake_put
+    return notifier
+
+
+def test_bold_becomes_strong():
+    assert markdown_to_matrix_html("**hi**") == "<strong>hi</strong>"
+
+
+def test_url_is_linkified():
+    out = markdown_to_matrix_html("see https://x.example/z")
+    assert '<a href="https://x.example/z">https://x.example/z</a>' in out
+
+
+def test_newlines_become_br():
+    assert markdown_to_matrix_html("a\nb") == "a<br/>b"
+
+
+def test_html_is_escaped():
+    out = markdown_to_matrix_html("<script> & 'stuff'")
+    assert "&lt;script&gt;" in out
+    assert "&amp;" in out
+    assert "<script>" not in out
+
+
+def test_strip_markdown_removes_bold_markers():
+    assert strip_markdown("**WATER LEVEL ALERT**") == "WATER LEVEL ALERT"
+
+
+def test_send_message_sends_html_and_plain_fallback():
+    captured = {}
+    notifier = _notifier_capturing(captured)
+
+    assert notifier.send_message("**hi** http://x.example") is True
+    assert captured["format"] == "org.matrix.custom.html"
+    assert "<strong>hi</strong>" in captured["formatted_body"]
+    # Plain body has the markdown markers stripped.
+    assert captured["body"] == "hi http://x.example"
+
+
+def test_send_message_plain_when_markdown_disabled():
+    captured = {}
+    notifier = _notifier_capturing(captured)
+
+    assert notifier.send_message("**raw**", markdown=False) is True
+    assert "formatted_body" not in captured
+    assert captured["body"] == "**raw**"
+
+
+def test_send_alert_renders_alert_fields():
+    captured = {}
+    notifier = _notifier_capturing(captured)
+    alert = WaterAlert(
+        station_code="P.1",
+        station_name="สะพานนวรัฐ",
+        alert_type="Zone 7 - Critical",
+        level=AlertLevel.CRITICAL,
+        water_level=4.62,
+        threshold=4.60,
+        discharge=612.0,
+        timestamp=datetime.datetime(2026, 7, 22, 14, 30, 0),
+    )
+
+    assert notifier.send_alert(alert) is True
+    html = captured["formatted_body"]
+    assert "<strong>WATER LEVEL ALERT</strong>" in html
+    assert "สะพานนวรัฐ" in html  # Thai station name preserved
+    assert "<strong>Current Level:</strong>" in html
+    # Plain fallback carries no leftover markdown markers.
+    assert "**" not in captured["body"]

tests/test_postgres_history.py

@@ -0,0 +1,49 @@
+import datetime
+
+from sqlalchemy import create_engine, text
+
+from src.postgres_history import PostgresHistory
+
+
+def test_history_returns_station_series_in_chronological_order(tmp_path):
+    engine = create_engine(f"sqlite:///{tmp_path / 'history.db'}")
+    with engine.begin() as connection:
+        connection.execute(text("CREATE TABLE stations (id INTEGER PRIMARY KEY, station_code TEXT)"))
+        connection.execute(
+            text(
+                "CREATE TABLE water_measurements ("
+                "timestamp DATETIME, station_id INTEGER, water_level REAL, "
+                "discharge REAL, discharge_percent REAL)"
+            )
+        )
+        connection.execute(text("INSERT INTO stations VALUES (1, 'P.1'), (2, 'P.20')"))
+        connection.execute(
+            text(
+                "INSERT INTO water_measurements VALUES "
+                "('2026-08-09 13:00:00', 1, 3.2, 110.0, 40.0),"
+                "('2026-08-09 14:00:00', 1, 3.4, 120.0, 42.0),"
+                "('2026-08-09 14:00:00', 2, 2.1, 30.0, 15.0)"
+            )
+        )
+
+    history = PostgresHistory(engine=engine).station_history(
+        "P.1",
+        start=datetime.datetime(2026, 8, 9, 12),
+        end=datetime.datetime(2026, 8, 9, 15),
+        limit=100,
+    )
+
+    assert [row["timestamp"].hour for row in history] == [13, 14]
+    assert [row["discharge"] for row in history] == [110.0, 120.0]
+    assert all(row["station_code"] == "P.1" for row in history)
+
+
+def test_history_rejects_excessive_limit():
+    history = PostgresHistory.__new__(PostgresHistory)
+
+    try:
+        history.station_history("P.1", datetime.datetime.now(), datetime.datetime.now(), 100001)
+    except ValueError as error:
+        assert "limit" in str(error)
+    else:
+        raise AssertionError("Expected excessive history limit to be rejected")

tests/test_scraper_parsing.py

@@ -0,0 +1,125 @@
+"""Assert-based tests for the RID API response parsing.
+
+The parsing in ``fetch_water_data_for_date`` is the riskiest, previously
+untested code: it maps the API's 1..24 "hourlytime" onto real timestamps
+(hour 24 rolls to next-day midnight) and treats ``"***"``/``None`` discharge as
+missing. These tests mock the HTTP call so no network is touched and stub the
+validator so we assert on the parser's output directly.
+"""
+
+import datetime
+from unittest.mock import MagicMock
+
+import pytest
+
+import src.water_scraper_v3 as scraper_mod
+from src.water_scraper_v3 import EnhancedWaterMonitorScraper as Scraper
+
+TARGET = datetime.datetime(2026, 7, 22)
+
+
+@pytest.fixture
+def make_scraper(monkeypatch):
+    """Return a factory that builds a bare scraper returning the given API rows."""
+
+    def _factory(rows):
+        scraper = Scraper.__new__(Scraper)  # bypass __init__ (no DB/network)
+        scraper.api_url = "https://example.invalid/api"
+        scraper.rate_limiter = MagicMock()
+        scraper.request_tracker = MagicMock()
+        scraper.station_config_path = "/nonexistent/stations.json"
+        scraper.station_mapping = scraper._load_station_mapping()  # bundled defaults
+
+        response = MagicMock()
+        response.json.return_value = {"rows": rows}
+        response.raise_for_status.return_value = None
+        scraper.session = MagicMock()
+        scraper.session.post.return_value = response
+
+        # Isolate parsing from validation.
+        monkeypatch.setattr(
+            scraper_mod.DataValidator,
+            "validate_measurements",
+            staticmethod(lambda m: m),
+        )
+        return scraper
+
+    return _factory
+
+
+def test_parses_water_level_and_discharge(make_scraper):
+    rows = [
+        {
+            "hourlytime": "9.00",
+            "wlvalues1": "3.50",
+            "qvalues1": "120.5",
+            "QPercent1": "45.2",
+        }
+    ]
+    data = make_scraper(rows).fetch_water_data_for_date(TARGET)
+
+    p20 = [d for d in data if d["station_code"] == "P.20"]
+    assert len(p20) == 1
+    m = p20[0]
+    assert m["water_level"] == 3.5
+    assert m["discharge"] == 120.5
+    assert m["discharge_percent"] == 45.2
+    assert m["timestamp"] == datetime.datetime(2026, 7, 22, 9, 0)
+    assert m["station_name_en"] == "Ban Chiang Dao"
+
+
+def test_discharge_asterisks_becomes_none(make_scraper):
+    rows = [{"hourlytime": "10.00", "wlvalues8": "4.20", "qvalues8": "***"}]
+    data = make_scraper(rows).fetch_water_data_for_date(TARGET)
+
+    p1 = [d for d in data if d["station_code"] == "P.1"][0]
+    assert p1["water_level"] == 4.2
+    assert p1["discharge"] is None
+    assert p1["discharge_percent"] is None
+
+
+def test_hour_24_rolls_to_next_day_midnight(make_scraper):
+    rows = [{"hourlytime": "24.00", "wlvalues1": "3.00"}]
+    data = make_scraper(rows).fetch_water_data_for_date(TARGET)
+
+    assert data[0]["timestamp"] == datetime.datetime(2026, 7, 23, 0, 0)
+
+
+def test_hours_1_to_23_stay_same_day(make_scraper):
+    rows = [
+        {"hourlytime": "1.00", "wlvalues1": "3.00"},
+        {"hourlytime": "23.00", "wlvalues1": "3.10"},
+    ]
+    data = make_scraper(rows).fetch_water_data_for_date(TARGET)
+
+    times = sorted(d["timestamp"] for d in data)
+    assert times == [
+        datetime.datetime(2026, 7, 22, 1, 0),
+        datetime.datetime(2026, 7, 22, 23, 0),
+    ]
+
+
+def test_none_water_level_is_skipped(make_scraper):
+    rows = [{"hourlytime": "9.00", "wlvalues1": None, "wlvalues2": "2.5"}]
+    data = make_scraper(rows).fetch_water_data_for_date(TARGET)
+
+    codes = {d["station_code"] for d in data}
+    assert "P.20" not in codes  # station 1 skipped (None water level)
+    assert "P.75" in codes  # station 2 present
+
+
+def test_out_of_range_and_empty_hours_skipped(make_scraper):
+    rows = [
+        {"hourlytime": "25.00", "wlvalues1": "3.0"},
+        {"hourlytime": "0.00", "wlvalues1": "3.0"},
+        {"hourlytime": "", "wlvalues1": "3.0"},
+    ]
+    data = make_scraper(rows).fetch_water_data_for_date(TARGET)
+
+    assert data == []
+
+
+def test_missing_rows_key_returns_empty(make_scraper):
+    scraper = make_scraper([])
+    scraper.session.post.return_value.json.return_value = {"unexpected": True}
+    assert scraper.fetch_water_data_for_date(TARGET) == []

tests/test_station_persistence.py

@@ -0,0 +1,77 @@
+"""Assert-based tests for station-mapping persistence.
+
+Station CRUD must survive restarts: the scraper loads its mapping from a
+runtime-writable JSON file (falling back to bundled defaults) and writes it back
+atomically. These tests exercise that load/save behaviour without constructing a
+full scraper (which would open network/DB connections).
+"""
+
+from src.water_scraper_v3 import EnhancedWaterMonitorScraper as Scraper
+
+
+def _bare_scraper(config_path):
+    """A scraper instance with only the station-config attribute set.
+
+    Bypasses __init__ so no database/HTTP connection is attempted.
+    """
+    scraper = Scraper.__new__(Scraper)
+    scraper.station_config_path = config_path
+    return scraper
+
+
+def test_loads_bundled_defaults_when_runtime_file_absent(tmp_path):
+    scraper = _bare_scraper(str(tmp_path / "does_not_exist.json"))
+    mapping = scraper._load_station_mapping()
+
+    assert len(mapping) == 16
+    assert mapping["8"]["code"] == "P.1"
+    assert mapping["8"]["english_name"] == "Nawarat Bridge"
+
+
+def test_save_then_reload_roundtrips_including_thai(tmp_path):
+    path = str(tmp_path / "stations.json")
+    scraper = _bare_scraper(path)
+    scraper.station_mapping = {
+        "1": {
+            "code": "P.99",
+            "thai_name": "สถานีทดสอบ",
+            "english_name": "Test Station",
+            "latitude": 1.0,
+            "longitude": 2.0,
+            "geohash": None,
+        }
+    }
+
+    assert scraper.save_stations() is True
+
+    reloaded = _bare_scraper(path)._load_station_mapping()
+    assert reloaded == scraper.station_mapping
+    assert reloaded["1"]["thai_name"] == "สถานีทดสอบ"
+
+
+def test_runtime_file_takes_precedence_over_defaults(tmp_path):
+    path = str(tmp_path / "stations.json")
+    writer = _bare_scraper(path)
+    writer.station_mapping = {"1": {"code": "ONLY"}}
+    assert writer.save_stations() is True
+
+    mapping = _bare_scraper(path)._load_station_mapping()
+    assert list(mapping.keys()) == ["1"]
+    assert mapping["1"]["code"] == "ONLY"
+
+
+def test_save_returns_false_without_a_path():
+    scraper = _bare_scraper("")
+    scraper.station_mapping = {}
+    assert scraper.save_stations() is False
+
+
+def test_save_is_atomic_no_tmp_left_behind(tmp_path):
+    path = tmp_path / "stations.json"
+    scraper = _bare_scraper(str(path))
+    scraper.station_mapping = {"1": {"code": "P.1"}}
+
+    assert scraper.save_stations() is True
+    assert path.exists()
+    # The temp file used during the atomic write must not remain.
+    assert not (tmp_path / "stations.json.tmp").exists()

tests/test_thaiwater.py

@@ -0,0 +1,75 @@
+from unittest.mock import MagicMock
+
+from src.thaiwater import ThaiWaterClient
+
+
+SAMPLE_RESPONSE = {
+    "data": {
+        "50": {
+            "type": "FeatureCollection",
+            "features": [
+                {
+                    "geometry": {"type": "Point", "coordinates": [98.635262, 19.638411]},
+                    "properties": {
+                        "id": "123",
+                        "waterlevelDatetime": "2026-08-09T15:00:00+07:00",
+                        "waterlevelMsl": 742.34,
+                        "storagePercent": 38.57,
+                        "diffWlBank": 1.87,
+                        "riverName": "Ping River",
+                        "station": {
+                            "stationCode": "G07003-P.65",
+                            "station": "Ban Muang Pok",
+                        },
+                        "agency": {"agencyShort": "RID"},
+                        "basin": {"basin": "Ping"},
+                    },
+                },
+                {
+                    "geometry": {"type": "Point", "coordinates": [100.1, 18.1]},
+                    "properties": {
+                        "id": "999",
+                        "station": {"stationCode": "N.1", "station": "Nan station"},
+                        "basin": {"basin": "Nan"},
+                    },
+                },
+            ],
+        }
+    }
+}
+
+
+def test_fetch_ping_sensors_normalizes_and_filters_basin():
+    session = MagicMock()
+    response = session.get.return_value
+    response.json.return_value = SAMPLE_RESPONSE
+    response.raise_for_status.return_value = None
+
+    sensors = ThaiWaterClient(session=session, api_key="public-key").fetch_ping_sensors()
+
+    assert sensors == [
+        {
+            "id": "thaiwater:123",
+            "station_code": "P.65",
+            "station_name": "Ban Muang Pok",
+            "latitude": 19.638411,
+            "longitude": 98.635262,
+            "timestamp": "2026-08-09T15:00:00+07:00",
+            "water_level_msl": 742.34,
+            "bank_percent": 38.57,
+            "distance_to_bank": 1.87,
+            "river_name": "Ping River",
+            "agency": "RID",
+            "source": "ThaiWater",
+        }
+    ]
+    session.get.assert_called_once()
+    assert session.get.call_args.kwargs["headers"]["x-api-key"] == "public-key"
+
+
+def test_fetch_ping_sensors_skips_features_without_coordinates():
+    response_data = {"data": {"50": {"features": [{"geometry": None, "properties": {"basin": {"basin": "Ping"}}}]}}}
+    session = MagicMock()
+    session.get.return_value.json.return_value = response_data
+
+    assert ThaiWaterClient(session=session, api_key="key").fetch_ping_sensors() == []