docs: rewrite the README for the Rust service

Phase 4 of docs/RUST_REWRITE.md. Since 2026-10-18 ping-collector (rust/)
collects, forecasts, notifies (ntfy + Matrix) and serves the website/API;
nothing in production ran the Python service code any more.

Removed (~17k lines): the FastAPI app (web_api, schemas, health_check,
metrics, postgres_history, forecast_history, thaiwater), the Python RID
collector and CLI (water_scraper_v3, main, run.py), the notifier and Matrix
alerting (notify, alerting), the multi-backend adapters (SQLite, MySQL,
InfluxDB, VictoriaMetrics) with demo databases, models/validators/exceptions/
logging/rate limiter, ml/skill.py (served by Rust now), the migration tooling
(api_parity, collector_parity, e2e_notify, install_collector.sh,
gen_notify_cases.py, spikes/rust-forecast), Docker/compose/Grafana files,
sql/, the release workflow and the docs for all of that. Their tests went
with them.

Kept in Python: src/ml (training, evaluation harness, export; predict.py as
the reference the Rust forecaster is tested against), the HII and RID-dam
clients/stores for the archive backfills (their live collectors removed),
the backfill scripts, retrain.sh + timer, forecast_parity.py.

- PostgreSQL only: src/config.py is database_url() from POSTGRES_* / .env;
  the stores lost their sqlite/mysql branches (and the 2026-08 column
  migration). The database tests run against a throwaway schema from
  TEST_DATABASE_URL (tests/conftest.py pg_url, skipped without it), as the
  Rust tests do; CI gets a PostgreSQL service. 100 tests (66 without a DB).
- The notifier settings moved from src/notify.py into src/ml/export.py;
  flood_config.json regenerates byte-identical.
- tests/test_dashboard.py checks every path the dashboard fetches against
  ping-collector's router.
- Dependencies: fastapi, uvicorn, pydantic, influxdb, pymysql, psutil,
  schedule, sphinx, pyinstaller, mypy, pytest-asyncio dropped; python-dotenv
  1.1.1 explicit (it came in through uvicorn[standard]).
- Formatting contract extended to scripts/ and tests/ (reformatted once).
- scripts/install.sh is the one installer (fresh host or update, in place in
  a checkout without touching its ownership): binary, units, uv sync, .env
  defaults, removes the retired water-monitor.service and any old
  --alert-check cron job, dry run + forecast check, restart, readiness wait.
- Docs, CLAUDE.md, CONTRIBUTING.md, .env.example, Makefile, unit comments
  and Rust doc comments updated to the Rust service.

.env.example

@@ -1,69 +1,85 @@
-# Northern Thailand Ping River Monitor Configuration
-# Copy this file to .env and customize for your environment
+# Ping River Monitor configuration: copy to .env and fill in.
+#
+# Read by ping-collector (the service; rust/collector/src/config.rs) and by the
+# Python training tools (src/config.py). Under systemd the variables come from
+# EnvironmentFile= (systemd's parser); for manual runs both programs load
+# ./.env with python-dotenv's rules. Unset keys take the defaults shown.
 
-# Database Configuration
-DB_TYPE=sqlite
-# Options: sqlite, mysql, postgresql, influxdb, victoriametrics
+# --- Database (PostgreSQL only) ---------------------------------------------
+# Either a full connection string (URL-encode special characters in the
+# password: : %3A  @ %40  # %23  ? %3F  & %26  / %2F  % %25) ...
+POSTGRES_CONNECTION_STRING=
+# ... or the parts (the password is encoded for you)
+POSTGRES_HOST=localhost
+POSTGRES_PORT=5432
+POSTGRES_DB=water_monitoring
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=
 
-# SQLite Configuration (default)
-WATER_DB_PATH=water_levels.db
+# --- What the service runs ---------------------------------------------------
+# Website and API: the address to listen on (empty = no website)
+WEB_LISTEN=0.0.0.0:8000
+# Forecast + ntfy/Matrix notifications after each collection (false = collect only)
+FORECAST_IN_COLLECTOR=true
+# Trained models (models/flood_<station>.json.gz from src/ml/export.py)
+# MODELS_DIR=models
 
-# VictoriaMetrics Configuration
-VM_HOST=localhost
-VM_PORT=8428
-VM_URL=
-
-# InfluxDB Configuration
-INFLUX_HOST=localhost
-INFLUX_PORT=8086
-INFLUX_DATABASE=ping_river_monitoring
-INFLUX_USERNAME=
-INFLUX_PASSWORD=
-
-# PostgreSQL Configuration
-POSTGRES_CONNECTION_STRING=postgresql://user:password@localhost:5432/ping_river_monitoring
-
-# MySQL Configuration
-MYSQL_CONNECTION_STRING=mysql://user:password@localhost:3306/ping_river_monitoring
-
-# API Configuration
-API_HOST=0.0.0.0
-API_PORT=8000
-API_WORKERS=1
-
-# Data Collection Settings
-SCRAPING_INTERVAL_HOURS=1
-REQUEST_TIMEOUT=30
-MAX_RETRIES=3
-RETRY_DELAY_SECONDS=60
-
-# Data Retention
-DATA_RETENTION_DAYS=365
-
-# Logging Configuration
+# --- Collection ----------------------------------------------------------------
+# RID is polled this many minutes past each hour and retried every
+# RID_RETRY_MINUTES while the current hour is not published yet
+RID_POLL_OFFSET_MINUTES=5
+RID_RETRY_MINUTES=10
+# RID reports with holes re-fetched at startup (0 = off); older: `ping-collector fill-gaps all`
+GAP_FILL_DAYS_ON_START=14
+# ENABLE_HII_COLLECTION=true
+# ENABLE_RESERVOIR_COLLECTION=true
+# ENABLE_RAIN_COLLECTION=true
+# HII_BASIN_CODE=6
+# REQUEST_TIMEOUT=30
+# Runtime station list written by the admin API; falls back to src/data/stations.json
+# STATION_CONFIG_PATH=stations.json
 LOG_LEVEL=INFO
-LOG_FILE=water_monitor.log
 
-# Security (for production)
-SECRET_KEY=your-secret-key-here
-API_KEY=your-api-key-here
+# --- Notifications (docs/NOTIFICATIONS.md) ------------------------------------
+# Public push alerts via self-hosted ntfy; empty NTFY_SERVER = off. Topics:
+# <prefix>-<station>-warning|danger, <prefix>-warning|danger,
+# <prefix>-p1-outlook, <prefix>-status.
+NTFY_SERVER=
+# Where the service POSTs (default NTFY_SERVER): the local ntfy address, so
+# publishing does not depend on DNS or the reverse proxy
+NTFY_PUBLISH_URL=
+NTFY_TOPIC_PREFIX=ping
+NTFY_TOKEN=
+PUBLIC_URL=https://water.buildfor.life/
+# Report RID lagging this many hours while ThaiWater telemetry is current (0 = never)
+RID_STALE_ALERT_HOURS=18
+# Matrix room for a team: the same transitions (docs/MATRIX_QUICK_START.md);
+# off while token or room is empty
+MATRIX_HOMESERVER=https://matrix.org
+MATRIX_ACCESS_TOKEN=
+MATRIX_ROOM_ID=
 
-# Monitoring
-ENABLE_METRICS=true
-ENABLE_HEALTH_CHECKS=true
+# --- Website and API -------------------------------------------------------------
+# X-API-Key for station editing, /config and POST /scrape/trigger (empty = those are off)
+ADMIN_API_KEY=
+# Comma-separated origins allowed cross-site with credentials (empty: any origin, no credentials)
+CORS_ALLOW_ORIGINS=
+# ThaiWater API key for /sensors/thaiwater (extra Ping-basin sensors on the map)
+THAIWATER_API_KEY=
+# Self-hosted Umami: server-side tracking of API calls
+# UMAMI_API_URL=https://stats.buildfor.life/api/send
+# UMAMI_WEBSITE_ID=00b2be73-8f5f-4400-9029-3be852eb08f7
+# UMAMI_TRACK_API=true
+# Response cache TTLs (seconds)
+# LATEST_CACHE_TTL_SECONDS=45
+# HII_CACHE_TTL_SECONDS=120
+# HEALTH_CACHE_TTL_SECONDS=30
+# A stored forecast older than this is served marked X-Data-Stale: true
+# FORECAST_MAX_AGE_SECONDS=7200
+# STATIC_DIR=src/static
+# Only for `ping-collector serve` running separately from `run`: the status file
+# SCRAPING_STATUS_FILE=collector-status.json
 
-# Geographic Settings
-TIMEZONE=Asia/Bangkok
-DEFAULT_LATITUDE=18.7875
-DEFAULT_LONGITUDE=99.0045
-
-# External Services
-NOTIFICATION_EMAIL=
-SMTP_SERVER=
-SMTP_PORT=587
-SMTP_USERNAME=
-SMTP_PASSWORD=
-
-# Development Settings
-DEBUG=false
-DEVELOPMENT_MODE=false
+# --- Python tools ----------------------------------------------------------------
+# Training/evaluation read the database above; this overrides it for them only
+# FLOOD_ML_DB_URL=

.gitea/workflows/ci.yml

@@ -1,341 +1,85 @@
-name: CI/CD Pipeline - Northern Thailand Ping River Monitor
+name: CI
+
+# The Python side (src/ml training/evaluation stack, archive backfills), on
+# every push and PR to master:
+#   1. formatting contract (black + isort, config in pyproject.toml)
+#   2. flake8 hard-error gate (syntax, undefined names)
+#   3. the pytest suite: synthetic data, no network; the database tests run
+#      against a throwaway PostgreSQL service (TEST_DATABASE_URL, as in rust.yml)
+# The service itself (ping-collector) is checked by rust.yml. Production is a
+# systemd unit deployed by `git pull` + scripts/install.sh on the server.
 
 on:
   push:
-    branches: [ main, develop ]
+    branches: [master, develop]
   pull_request:
-    branches: [ main ]
+    branches: [master]
   schedule:
-    # Run tests daily at 2 AM UTC
-    - cron: '0 2 * * *'
+    # daily, catches dependency drift
+    - cron: "0 2 * * *"
+  workflow_dispatch:
 
 env:
-  PYTHON_VERSION: '3.11'
-  REGISTRY: git.b4l.co.th
-  IMAGE_NAME: b4l/northern-thailand-ping-river-monitor
-  # GitHub token for better rate limits and authentication
-  GH_TOKEN: ${{ secrets.GH_TOKEN }}
+  PYTHON_VERSION: "3.11" # pandas 2.0.3 ships no 3.12 wheels; psycopg2-binary 2.9.9 breaks on 3.13
 
 jobs:
-  # Test job
+  lint:
+    name: Format & lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: pip
+          cache-dependency-path: requirements-dev.txt
+
+      - name: Install tools
+        run: |
+          python -m pip install --upgrade pip --root-user-action=ignore
+          pip install --root-user-action=ignore black==26.5.1 isort==5.12.0 flake8==6.1.0
+
+      - name: black
+        run: black --check --diff src/ scripts/ tests/ rust/forecast/testdata rust/collector/testdata
+
+      - name: isort
+        run: isort --check-only --diff src/ scripts/ tests/
+
+      - name: flake8 (errors only)
+        run: flake8 src/ scripts/ tests/ --count --select=E9,F63,F7,F82 --show-source --statistics
+
   test:
-    name: Test Suite
+    name: Test suite
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ['3.9', '3.10', '3.11', '3.12']
-    
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-      with:
-        token: ${{ secrets.GITEA_TOKEN }}
-      
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python-version }}
-        
-    - name: Cache pip dependencies
-      uses: actions/cache@v3
-      with:
-        path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
-        restore-keys: |
-          ${{ runner.os }}-pip-
-          
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip --root-user-action=ignore
-        pip install --root-user-action=ignore -r requirements.txt
-        pip install --root-user-action=ignore -r requirements-dev.txt
-        
-    - name: Lint with flake8
-      run: |
-        flake8 src/ --count --select=E9,F63,F7,F82 --show-source --statistics
-        flake8 src/ --count --exit-zero --max-complexity=10 --max-line-length=100 --statistics
-        
-    - name: Type check with mypy
-      run: |
-        mypy src/ --ignore-missing-imports
-        
-    - name: Format check with black
-      run: |
-        black --check src/ *.py
-        
-    - name: Import sort check
-      run: |
-        isort --check-only src/ *.py
-        
-    - name: Run integration tests
-      run: |
-        python tests/test_integration.py
-        
-    - name: Run station management tests
-      run: |
-        python tests/test_station_management.py
-        
-    - name: Test application startup
-      run: |
-        timeout 10s python run.py --test || true
-        
-    - name: Security scan with bandit
-      run: |
-        bandit -r src/ -f json -o bandit-report.json || true
-        
-    - name: Upload test artifacts
-      uses: actions/upload-artifact@v3
-      if: always()
-      with:
-        name: test-results-${{ matrix.python-version }}
-        path: |
-          bandit-report.json
-          *.log
-
-  # Code quality job
-  code-quality:
-    name: Code Quality
-    runs-on: ubuntu-latest
-    
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-      with:
-        token: ${{ secrets.GITEA_TOKEN }}
-      
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ env.PYTHON_VERSION }}
-        
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip --root-user-action=ignore
-        pip install --root-user-action=ignore -r requirements-dev.txt
-        
-    - name: Run safety check
-      run: |
-        safety check -r requirements.txt --json --output safety-report.json || true
-        
-    - name: Run bandit security scan
-      run: |
-        bandit -r src/ -f json -o bandit-report.json || true
-        
-    - name: Upload security reports
-      uses: actions/upload-artifact@v3
-      with:
-        name: security-reports
-        path: |
-          safety-report.json
-          bandit-report.json
-
-  # Build Docker image
-  build:
-    name: Build Docker Image
-    runs-on: ubuntu-latest
-    needs: test
-    
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-      with:
-        token: ${{ secrets.GITEA_TOKEN }}
-      
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v3
-      
-    - name: Log in to Container Registry
-      uses: docker/login-action@v3
-      with:
-        registry: ${{ env.REGISTRY }}
-        username: ${{ github.actor }}
-        password: ${{ secrets.GITEA_TOKEN }}
-        
-    - name: Extract metadata
-      id: meta
-      uses: docker/metadata-action@v5
-      with:
-        images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-        tags: |
-          type=ref,event=branch
-          type=ref,event=pr
-          type=sha,prefix={{branch}}-
-          type=raw,value=latest,enable={{is_default_branch}}
-          
-    - name: Build and push Docker image
-      uses: docker/build-push-action@v5
-      with:
-        context: .
-        platforms: linux/amd64,linux/arm64
-        push: true
-        tags: ${{ steps.meta.outputs.tags }}
-        labels: ${{ steps.meta.outputs.labels }}
-        cache-from: type=gha
-        cache-to: type=gha,mode=max
-      env:
-        GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
-        
-    - name: Test Docker image
-      run: |
-        docker run --rm ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }} python run.py --test
-
-  # Integration test with services
-  integration-test:
-    name: Integration Test with Services
-    runs-on: ubuntu-latest
-    needs: build
-    
     services:
-      victoriametrics:
-        image: victoriametrics/victoria-metrics:latest
-        ports:
-          - 8428:8428
+      postgres:
+        image: postgres:16
+        env:
+          POSTGRES_PASSWORD: ping
+          POSTGRES_DB: ping
         options: >-
-          --health-cmd "wget --quiet --tries=1 --spider http://localhost:8428/health"
-          --health-interval 30s
-          --health-timeout 10s
-          --health-retries 3
-          
+          --health-cmd pg_isready
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 20
+    env:
+      TEST_DATABASE_URL: postgres://postgres:ping@postgres:5432/ping
     steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-      with:
-        token: ${{ secrets.GITEA_TOKEN }}
-      
-    - name: Wait for VictoriaMetrics
-      run: |
-        timeout 60s bash -c 'until curl -f http://localhost:8428/health; do sleep 2; done'
-        
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ env.PYTHON_VERSION }}
-        
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip --root-user-action=ignore
-        pip install --root-user-action=ignore -r requirements.txt
-        
-    - name: Test with VictoriaMetrics
-      env:
-        DB_TYPE: victoriametrics
-        VM_HOST: localhost
-        VM_PORT: 8428
-      run: |
-        python run.py --test
-        
-    - name: Start API server
-      env:
-        DB_TYPE: victoriametrics
-        VM_HOST: localhost
-        VM_PORT: 8428
-      run: |
-        python run.py --web-api &
-        sleep 10
-        
-    - name: Test API endpoints
-      run: |
-        curl -f http://localhost:8000/health
-        curl -f http://localhost:8000/stations
-        curl -f http://localhost:8000/metrics
+      - uses: actions/checkout@v4
 
-  # Deploy to staging (only on develop branch)
-  deploy-staging:
-    name: Deploy to Staging
-    runs-on: ubuntu-latest
-    needs: [test, build, integration-test]
-    if: github.ref == 'refs/heads/develop'
-    environment:
-      name: staging
-      url: https://staging.ping-river-monitor.b4l.co.th
-      
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-      with:
-        token: ${{ secrets.GITEA_TOKEN }}
-      
-    - name: Deploy to staging
-      run: |
-        echo "Deploying to staging environment..."
-        # Add your staging deployment commands here
-        # Example: kubectl, docker-compose, or webhook call
-        
-    - name: Health check staging
-      run: |
-        sleep 30
-        curl -f https://staging.ping-river-monitor.b4l.co.th/health
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: pip
+          cache-dependency-path: |
+            requirements.txt
+            requirements-dev.txt
 
-  # Deploy to production (only on main branch, manual approval)
-  deploy-production:
-    name: Deploy to Production
-    runs-on: ubuntu-latest
-    needs: [test, build, integration-test]
-    if: github.ref == 'refs/heads/main'
-    environment:
-      name: production
-      url: https://ping-river-monitor.b4l.co.th
-      
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-      with:
-        token: ${{ secrets.GITEA_TOKEN }}
-      
-    - name: Deploy to production
-      run: |
-        echo "Deploying to production environment..."
-        # Add your production deployment commands here
-        
-    - name: Health check production
-      run: |
-        sleep 30
-        curl -f https://ping-river-monitor.b4l.co.th/health
-        
-    - name: Notify deployment
-      run: |
-        echo "✅ Production deployment successful!"
-        echo "🌐 URL: https://ping-river-monitor.b4l.co.th"
-        echo "📊 Grafana: https://grafana.ping-river-monitor.b4l.co.th"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip --root-user-action=ignore
+          pip install --root-user-action=ignore -r requirements.txt pytest==9.1.1
 
-  # Performance test (only on main branch)
-  performance-test:
-    name: Performance Test
-    runs-on: ubuntu-latest
-    needs: deploy-production
-    if: github.ref == 'refs/heads/main'
-    
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-      with:
-        token: ${{ secrets.GITEA_TOKEN }}
-      
-    - name: Install Apache Bench
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y apache2-utils
-        
-    - name: Performance test API endpoints
-      run: |
-        # Test health endpoint
-        ab -n 100 -c 10 https://ping-river-monitor.b4l.co.th/health
-        
-        # Test stations endpoint
-        ab -n 50 -c 5 https://ping-river-monitor.b4l.co.th/stations
-        
-        # Test metrics endpoint
-        ab -n 50 -c 5 https://ping-river-monitor.b4l.co.th/metrics
-
-  # Cleanup old artifacts
-  cleanup:
-    name: Cleanup
-    runs-on: ubuntu-latest
-    if: always()
-    needs: [test, build, integration-test]
-    
-    steps:
-    - name: Clean up old Docker images
-      run: |
-        echo "Cleaning up old Docker images..."
-        # Add cleanup commands for old images/artifacts
+      - name: pytest
+        run: pytest -q -p no:cacheprovider -rs

.gitea/workflows/docs.yml

@@ -1,368 +1,55 @@
-name: Documentation
+name: Docs
+
+# Every relative link / image path in the shipped Markdown (README, CLAUDE,
+# CONTRIBUTING, rust/README, docs/) resolves inside the repo. External URLs
+# are NOT fetched: localhost examples and rate-limited hosts made that gate
+# permanently red, and a 200 proves nothing about a doc anyway. The API
+# reference (/docs, /openapi.json) is generated by ping-collector from its
+# route table and tested in rust/collector/src/web/tests.rs.
 
 on:
   push:
-    branches: [ main, develop ]
+    branches: [master, develop]
     paths:
-      - 'docs/**'
-      - 'README.md'
-      - 'CONTRIBUTING.md'
-      - 'src/**/*.py'
+      - "**.md"
+      - ".gitea/workflows/docs.yml"
   pull_request:
     paths:
-      - 'docs/**'
-      - 'README.md'
-      - 'CONTRIBUTING.md'
+      - "**.md"
   workflow_dispatch:
 
-env:
-  PYTHON_VERSION: '3.11'
-
 jobs:
-  # Validate documentation
-  validate-docs:
-    name: Validate Documentation
+  docs:
+    name: Validate documentation
     runs-on: ubuntu-latest
-    
     steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-      with:
-        token: ${{ secrets.GITEA_TOKEN }}
-      
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ env.PYTHON_VERSION }}
-        
-    - name: Install documentation tools
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
-        pip install sphinx sphinx-rtd-theme sphinx-autodoc-typehints
-        pip install markdown-link-check || true
-        
-    - name: Check markdown links
-      run: |
-        echo "🔗 Checking markdown links..."
-        find . -name "*.md" -not -path "./.git/*" -not -path "./node_modules/*" | while read file; do
-          echo "Checking $file"
-          # Basic link validation (you can enhance this)
-          grep -o 'http[s]*://[^)]*' "$file" | while read url; do
-            if curl -s --head "$url" | head -n 1 | grep -q "200 OK"; then
-              echo "✅ $url"
-            else
-              echo "❌ $url (in $file)"
-            fi
-          done
-        done
-        
-    - name: Validate README structure
-      run: |
-        echo "📋 Validating README structure..."
-        
-        required_sections=(
-          "# Northern Thailand Ping River Monitor"
-          "## Features"
-          "## Quick Start"
-          "## Installation"
-          "## Usage"
-          "## API Endpoints"
-          "## Docker"
-          "## Contributing"
-          "## License"
-        )
-        
-        for section in "${required_sections[@]}"; do
-          if grep -q "$section" README.md; then
-            echo "✅ Found: $section"
-          else
-            echo "❌ Missing: $section"
-          fi
-        done
-        
-    - name: Check documentation completeness
-      run: |
-        echo "📚 Checking documentation completeness..."
-        
-        # Check if all Python modules have docstrings
-        python -c "
-        import ast
-        import os
-        
-        def check_docstrings(filepath):
-            with open(filepath, 'r', encoding='utf-8') as f:
-                tree = ast.parse(f.read())
-            
-            missing_docstrings = []
-            
-            for node in ast.walk(tree):
-                if isinstance(node, (ast.FunctionDef, ast.ClassDef, ast.AsyncFunctionDef)):
-                    if not ast.get_docstring(node):
-                        missing_docstrings.append(f'{node.name} in {filepath}')
-            
-            return missing_docstrings
-        
-        all_missing = []
-        for root, dirs, files in os.walk('src'):
-            for file in files:
-                if file.endswith('.py') and not file.startswith('__'):
-                    filepath = os.path.join(root, file)
-                    missing = check_docstrings(filepath)
-                    all_missing.extend(missing)
-        
-        if all_missing:
-            print('⚠️ Missing docstrings:')
-            for item in all_missing[:10]:  # Show first 10
-                print(f'  - {item}')
-            if len(all_missing) > 10:
-                print(f'  ... and {len(all_missing) - 10} more')
-        else:
-            print('✅ All functions and classes have docstrings')
-        "
+      - uses: actions/checkout@v4
 
-  # Generate API documentation
-  generate-api-docs:
-    name: Generate API Documentation
-    runs-on: ubuntu-latest
-    
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-      with:
-        token: ${{ secrets.GITEA_TOKEN }}
-      
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ env.PYTHON_VERSION }}
-        
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
-        
-    - name: Generate OpenAPI spec
-      run: |
-        echo "📝 Generating OpenAPI specification..."
-        python -c "
-        import json
-        import sys
-        sys.path.insert(0, 'src')
-        
-        try:
-            from web_api import app
-            openapi_spec = app.openapi()
-            
-            with open('openapi.json', 'w') as f:
-                json.dump(openapi_spec, f, indent=2)
-            
-            print('✅ OpenAPI spec generated: openapi.json')
-        except Exception as e:
-            print(f'❌ Failed to generate OpenAPI spec: {e}')
-        "
-        
-    - name: Generate API documentation
-      run: |
-        echo "📖 Generating API documentation..."
-        
-        # Create API documentation from OpenAPI spec
-        if [ -f openapi.json ]; then
-          cat > api-docs.md << 'EOF'
-        # API Documentation
-        
-        This document describes the REST API endpoints for the Northern Thailand Ping River Monitor.
-        
-        ## Base URL
-        
-        - Production: `https://ping-river-monitor.b4l.co.th`
-        - Staging: `https://staging.ping-river-monitor.b4l.co.th`
-        - Development: `http://localhost:8000`
-        
-        ## Authentication
-        
-        Currently, the API does not require authentication. This may change in future versions.
-        
-        ## Endpoints
-        
-        EOF
-          
-          # Extract endpoints from OpenAPI spec
-          python -c "
-        import json
-        
-        with open('openapi.json', 'r') as f:
-            spec = json.load(f)
-        
-        for path, methods in spec.get('paths', {}).items():
-            for method, details in methods.items():
-                print(f'### {method.upper()} {path}')
-                print()
-                print(details.get('summary', 'No description available'))
-                print()
-                if 'parameters' in details:
-                    print('**Parameters:**')
-                    for param in details['parameters']:
-                        print(f'- `{param[\"name\"]}` ({param.get(\"in\", \"query\")}): {param.get(\"description\", \"No description\")}')
-                    print()
-                print('---')
-                print()
-          " >> api-docs.md
-          
-          echo "✅ API documentation generated: api-docs.md"
-        fi
-        
-    - name: Upload documentation artifacts
-      uses: actions/upload-artifact@v3
-      with:
-        name: documentation-${{ github.run_number }}
-        path: |
-          openapi.json
-          api-docs.md
-
-  # Build Sphinx documentation
-  build-sphinx-docs:
-    name: Build Sphinx Documentation
-    runs-on: ubuntu-latest
-    
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-      with:
-        token: ${{ secrets.GITEA_TOKEN }}
-      
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ env.PYTHON_VERSION }}
-        
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
-        pip install sphinx sphinx-rtd-theme sphinx-autodoc-typehints
-        
-    - name: Create Sphinx configuration
-      run: |
-        mkdir -p docs/sphinx
-        
-        cat > docs/sphinx/conf.py << 'EOF'
-        import os
-        import sys
-        sys.path.insert(0, os.path.abspath('../../src'))
-        
-        project = 'Northern Thailand Ping River Monitor'
-        copyright = '2025, Ping River Monitor Team'
-        author = 'Ping River Monitor Team'
-        version = '3.1.3'
-        release = '3.1.3'
-        
-        extensions = [
-            'sphinx.ext.autodoc',
-            'sphinx.ext.viewcode',
-            'sphinx.ext.napoleon',
-            'sphinx_autodoc_typehints',
-        ]
-        
-        templates_path = ['_templates']
-        exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
-        
-        html_theme = 'sphinx_rtd_theme'
-        html_static_path = ['_static']
-        
-        autodoc_default_options = {
-            'members': True,
-            'member-order': 'bysource',
-            'special-members': '__init__',
-            'undoc-members': True,
-            'exclude-members': '__weakref__'
-        }
-        EOF
-        
-        cat > docs/sphinx/index.rst << 'EOF'
-        Northern Thailand Ping River Monitor Documentation
-        ================================================
-        
-        .. toctree::
-           :maxdepth: 2
-           :caption: Contents:
-        
-           modules
-        
-        Indices and tables
-        ==================
-        
-        * :ref:`genindex`
-        * :ref:`modindex`
-        * :ref:`search`
-        EOF
-        
-    - name: Generate module documentation
-      run: |
-        cd docs/sphinx
-        sphinx-apidoc -o . ../../src
-        
-    - name: Build documentation
-      run: |
-        cd docs/sphinx
-        sphinx-build -b html . _build/html
-        
-    - name: Upload Sphinx documentation
-      uses: actions/upload-artifact@v3
-      with:
-        name: sphinx-docs-${{ github.run_number }}
-        path: docs/sphinx/_build/html/
-
-  # Documentation summary
-  docs-summary:
-    name: Documentation Summary
-    runs-on: ubuntu-latest
-    needs: [validate-docs, generate-api-docs, build-sphinx-docs]
-    if: always()
-    
-    steps:
-    - name: Generate documentation summary
-      run: |
-        echo "# 📚 Documentation Build Summary" > docs-summary.md
-        echo "" >> docs-summary.md
-        echo "**Build Date:** $(date -u)" >> docs-summary.md
-        echo "**Repository:** ${{ github.repository }}" >> docs-summary.md
-        echo "**Commit:** ${{ github.sha }}" >> docs-summary.md
-        echo "" >> docs-summary.md
-        
-        echo "## 📊 Results" >> docs-summary.md
-        echo "" >> docs-summary.md
-        
-        if [ "${{ needs.validate-docs.result }}" = "success" ]; then
-          echo "- ✅ **Documentation Validation**: Passed" >> docs-summary.md
-        else
-          echo "- ❌ **Documentation Validation**: Failed" >> docs-summary.md
-        fi
-        
-        if [ "${{ needs.generate-api-docs.result }}" = "success" ]; then
-          echo "- ✅ **API Documentation**: Generated" >> docs-summary.md
-        else
-          echo "- ❌ **API Documentation**: Failed" >> docs-summary.md
-        fi
-        
-        if [ "${{ needs.build-sphinx-docs.result }}" = "success" ]; then
-          echo "- ✅ **Sphinx Documentation**: Built" >> docs-summary.md
-        else
-          echo "- ❌ **Sphinx Documentation**: Failed" >> docs-summary.md
-        fi
-        
-        echo "" >> docs-summary.md
-        echo "## 🔗 Available Documentation" >> docs-summary.md
-        echo "" >> docs-summary.md
-        echo "- [README.md](../README.md)" >> docs-summary.md
-        echo "- [API Documentation](../docs/)" >> docs-summary.md
-        echo "- [Contributing Guide](../CONTRIBUTING.md)" >> docs-summary.md
-        echo "- [Deployment Checklist](../DEPLOYMENT_CHECKLIST.md)" >> docs-summary.md
-        
-        cat docs-summary.md
-        
-    - name: Upload documentation summary
-      uses: actions/upload-artifact@v3
-      with:
-        name: docs-summary-${{ github.run_number }}
-        path: docs-summary.md
+      - name: Relative links and images resolve
+        run: |
+          python3 - <<'PY'
+          import re, sys, pathlib
+          root = pathlib.Path(".")
+          files = [root / "README.md", root / "CLAUDE.md", root / "CONTRIBUTING.md",
+                   root / "rust" / "README.md", *root.glob("docs/**/*.md")]
+          link = re.compile(r"!?\[[^\]]*\]\(([^)\s]+)(?:\s+\"[^\"]*\")?\)")
+          bad = []
+          for md in files:
+              if not md.exists():
+                  continue
+              for m in link.finditer(md.read_text(encoding="utf-8")):
+                  target = m.group(1)
+                  if target.startswith(("http://", "https://", "mailto:", "#")):
+                      continue
+                  path = target.split("#", 1)[0]
+                  if not path:
+                      continue
+                  resolved = (md.parent / path).resolve()
+                  if not resolved.exists():
+                      bad.append(f"{md}: {target}")
+          if bad:
+              print("Broken relative links:")
+              print("\n".join("  " + b for b in bad))
+              sys.exit(1)
+          print(f"checked {len(files)} files, all relative links resolve")
+          PY

.gitea/workflows/release.yml

@@ -1,284 +0,0 @@
-name: Release - Northern Thailand Ping River Monitor
-
-on:
-  push:
-    tags:
-      - 'v*.*.*'
-  workflow_dispatch:
-    inputs:
-      version:
-        description: 'Release version (e.g., v3.1.3)'
-        required: true
-        type: string
-
-env:
-  PYTHON_VERSION: '3.11'
-  REGISTRY: git.b4l.co.th
-  IMAGE_NAME: b4l/northern-thailand-ping-river-monitor
-  # GitHub token for better rate limits and authentication
-  GH_TOKEN: ${{ secrets.GH_TOKEN }}
-
-jobs:
-  # Create release
-  create-release:
-    name: Create Release
-    runs-on: ubuntu-latest
-    outputs:
-      version: ${{ steps.version.outputs.version }}
-      
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-      with:
-        token: ${{ secrets.GITEA_TOKEN }}
-        fetch-depth: 0
-        
-    - name: Get version
-      id: version
-      run: |
-        if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
-          echo "version=${{ github.event.inputs.version }}" >> $GITHUB_OUTPUT
-        else
-          echo "version=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
-        fi
-        
-    - name: Generate changelog
-      id: changelog
-      run: |
-        # Generate changelog from git commits
-        echo "## Changes" > CHANGELOG.md
-        git log --pretty=format:"- %s" $(git describe --tags --abbrev=0 HEAD^)..HEAD >> CHANGELOG.md || echo "- Initial release" >> CHANGELOG.md
-        echo "" >> CHANGELOG.md
-        echo "## Docker Images" >> CHANGELOG.md
-        echo "- \`${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.version.outputs.version }}\`" >> CHANGELOG.md
-        echo "- \`${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest\`" >> CHANGELOG.md
-        
-    - name: Create Release
-      uses: actions/create-release@v1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITEA_TOKEN }}
-      with:
-        tag_name: ${{ steps.version.outputs.version }}
-        release_name: Northern Thailand Ping River Monitor ${{ steps.version.outputs.version }}
-        body_path: CHANGELOG.md
-        draft: false
-        prerelease: false
-
-  # Build and test for release
-  test-release:
-    name: Test Release Build
-    runs-on: ubuntu-latest
-    needs: create-release
-    strategy:
-      matrix:
-        python-version: ['3.9', '3.10', '3.11', '3.12']
-        
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-      with:
-        token: ${{ secrets.GITEA_TOKEN }}
-      
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python-version }}
-        
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip --root-user-action=ignore
-        pip install --root-user-action=ignore -r requirements.txt
-        pip install --root-user-action=ignore -r requirements-dev.txt
-        
-    - name: Run full test suite
-      run: |
-        python tests/test_integration.py
-        python tests/test_station_management.py
-        python run.py --test
-        
-    - name: Build Python package
-      run: |
-        pip install --root-user-action=ignore build
-        python -m build
-        
-    - name: Upload Python package
-      uses: actions/upload-artifact@v3
-      with:
-        name: python-package-${{ matrix.python-version }}
-        path: dist/
-
-  # Build release Docker images
-  build-release:
-    name: Build Release Images
-    runs-on: ubuntu-latest
-    needs: [create-release, test-release]
-    
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-      with:
-        token: ${{ secrets.GITEA_TOKEN }}
-      
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v3
-      
-    - name: Log in to Container Registry
-      uses: docker/login-action@v3
-      with: 
-        registry: ${{ env.REGISTRY }}
-        username: ${{ vars.WORKER_USERNAME}}
-        password: ${{ secrets.CI_BOT_TOKEN }}
-        
-    - name: Build and push release images
-      uses: docker/build-push-action@v5
-      with:
-        context: .
-        platforms: linux/amd64,linux/arm64
-        push: true
-        tags: |
-          ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.create-release.outputs.version }}
-          ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
-        labels: |
-          org.opencontainers.image.title=Northern Thailand Ping River Monitor
-          org.opencontainers.image.description=Real-time water level monitoring for Ping River Basin
-          org.opencontainers.image.version=${{ needs.create-release.outputs.version }}
-          org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
-          org.opencontainers.image.revision=${{ github.sha }}
-        cache-from: type=gha
-        cache-to: type=gha,mode=max
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITEA_TOKEN }}
-
-  # Security scan for release
-  security-scan:
-    name: Security Scan
-    runs-on: ubuntu-latest
-    needs: build-release
-    
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-      with:
-        token: ${{ secrets.GITEA_TOKEN}}
-      
-
-
-  # Test release deployment locally
-  deploy-release:
-    name: Test Release Deployment
-    runs-on: ubuntu-latest
-    needs: [create-release, build-release, security-scan]
-    environment:
-      name: testing
-      url: http://localhost:8080
-      
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-      with:
-        token: ${{ secrets.GITEA_TOKEN }}
-      
-    - name: Log in to Container Registry
-      uses: docker/login-action@v3
-      with: 
-        registry: ${{ env.REGISTRY }}
-        username: ${{ vars.WORKER_USERNAME}}
-        password: ${{ secrets.CI_BOT_TOKEN }}
-        
-    - name: Deploy to production (Local Test)
-      run: |
-        echo "🚀 Testing ${{ needs.create-release.outputs.version }} deployment locally..."
-        
-        # Pull the built image
-        docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.create-release.outputs.version }}
-        
-        # Stop any existing containers
-        docker stop ping-river-monitor-test || true
-        docker rm ping-river-monitor-test || true
-        
-        # Start the container for testing
-        docker run -d \
-          --name ping-river-monitor-test \
-          -p 8080:8000 \
-          -e LOG_LEVEL=INFO \
-          -e DB_TYPE=sqlite \
-          ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.create-release.outputs.version }}
-        
-        echo "✅ Container started for testing"
-        
-    - name: Health check after deployment
-      run: |
-        echo "⏳ Waiting for application to start..."
-        sleep 30
-        
-        echo "🔍 Running health checks against local container..."
-        
-        # Wait for the application to be ready
-        for i in {1..12}; do
-          if curl -f http://localhost:8080/health; then
-            echo "✅ Health endpoint responding"
-            break
-          else
-            echo "⏳ Waiting for health endpoint... (attempt $i/12)"
-            sleep 10
-          fi
-        done
-        
-        # Test API endpoints
-        echo "🧪 Testing API endpoints..."
-        curl -f http://localhost:8080/health || exit 1
-        curl -f http://localhost:8080/docs || exit 1
-        curl -f http://localhost:8080/stations || exit 1
-        curl -f http://localhost:8080/metrics || exit 1
-        
-        echo "✅ All health checks passed!"
-        
-    - name: Container logs and cleanup
-      if: always()
-      run: |
-        echo "📋 Container logs:"
-        docker logs ping-river-monitor-test || true
-        
-        echo "🧹 Cleaning up test container..."
-        docker stop ping-river-monitor-test || true
-        docker rm ping-river-monitor-test || true
-        
-        echo "📊 Deployment Test Summary:"
-        echo "Version: ${{ needs.create-release.outputs.version }}"
-        echo "Image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.create-release.outputs.version }}"
-        echo "Status: Container tested successfully"
-        echo "Ready for production deployment"
-
-
-
-  # Notify stakeholders
-  notify:
-    name: Notify Release
-    runs-on: ubuntu-latest
-    needs: [create-release, deploy-release]
-    if: always()
-    
-    steps:
-    - name: Notify success
-      if: needs.deploy-release.result == 'success'
-      run: |
-        echo "🎉 Release ${{ needs.create-release.outputs.version }} tested successfully!"
-        echo "🧪 Local Test: Passed all health checks"
-        echo "� GDocker Image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.create-release.outputs.version }}"
-        echo "✅ Ready for production deployment"
-        
-        # Add notification to Slack, Discord, email, etc.
-        # curl -X POST -H 'Content-type: application/json' \
-        #   --data '{"text":"🎉 Northern Thailand Ping River Monitor ${{ needs.create-release.outputs.version }} tested and ready for deployment!"}' \
-        #   ${{ secrets.SLACK_WEBHOOK_URL }}
-        
-    - name: Notify failure
-      if: needs.deploy-release.result == 'failure'
-      run: |
-        echo "❌ Release ${{ needs.create-release.outputs.version }} testing failed!"
-        echo "Please check the logs and fix issues before production deployment."
-        
-        # Add failure notification
-        # curl -X POST -H 'Content-type: application/json' \
-        #   --data '{"text":"❌ Northern Thailand Ping River Monitor ${{ needs.create-release.outputs.version }} testing failed!"}' \
-        #   ${{ secrets.SLACK_WEBHOOK_URL }}

.gitea/workflows/rust.yml

@@ -0,0 +1,72 @@
+name: Rust
+
+# rust/ workspace (ping-collector, ping-forecast): format, clippy, tests
+# (the Python parity fixtures, and the database tests against a throwaway
+# PostgreSQL service), release build.
+# The Linux binary is uploaded as an artifact for scripts/install_collector.sh.
+
+on:
+  push:
+    branches: [master, develop]
+    paths:
+      - "rust/**"
+      - "src/data/stations.json"
+      - ".gitea/workflows/rust.yml"
+  pull_request:
+    branches: [master]
+    paths:
+      - "rust/**"
+      - "src/data/stations.json"
+      - ".gitea/workflows/rust.yml"
+  workflow_dispatch:
+
+jobs:
+  rust:
+    name: fmt, clippy, test, build
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:16
+        env:
+          POSTGRES_PASSWORD: ping
+          POSTGRES_DB: ping
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 20
+    env:
+      CARGO_TERM_COLOR: always
+      TEST_DATABASE_URL: postgres://postgres:ping@postgres:5432/ping
+    defaults:
+      run:
+        working-directory: rust
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: System packages (OpenSSL headers for native-tls)
+        run: |
+          SUDO=$(command -v sudo || true)
+          $SUDO apt-get update -qq
+          $SUDO apt-get install -y -qq pkg-config libssl-dev
+
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          components: rustfmt, clippy
+
+      - name: fmt
+        run: cargo fmt --all --check
+
+      - name: clippy
+        run: cargo clippy --all-targets --locked -- -D warnings
+
+      - name: test
+        run: cargo test --locked
+
+      - name: release build
+        run: cargo build --release --locked
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: ping-collector-linux-x86_64
+          path: rust/target/release/ping-collector

.gitea/workflows/security.yml

@@ -1,293 +1,109 @@
-name: Security & Dependency Updates
+name: Security
+
+# Two gates that can actually fail, plus one report:
+#   - pip-audit against requirements.txt: any known vulnerability in a runtime
+#     dependency fails the job (dev-only tools are reported, not gated)
+#   - bandit on src/: HIGH severity findings fail; medium/low are listed.
+#     B104 (bind 0.0.0.0) is skipped: the service is meant to listen on all
+#     interfaces behind Cloudflare/Caddy.
+#   - pip-licenses report as an artifact (informational; the project is MIT
+#     and its runtime deps are MIT/BSD/Apache/PSF)
+# The old file ran safety/bandit/semgrep with `|| true` and could not go red.
 
 on:
   schedule:
-    # Run security scans daily at 3 AM UTC
-    - cron: "0 3 * * *"
+    - cron: "0 3 * * 1" # weekly, Monday 03:00 UTC
   workflow_dispatch:
   push:
     paths:
       - "requirements*.txt"
-      - "Dockerfile"
+      - "pyproject.toml"
+      - "uv.lock"
+      - "src/**/*.py"
       - ".gitea/workflows/security.yml"
+  pull_request:
+    paths:
+      - "requirements*.txt"
+      - "pyproject.toml"
+      - "src/**/*.py"
 
 env:
   PYTHON_VERSION: "3.11"
-  # GitHub token for better rate limits and authentication
-  GH_TOKEN: ${{ secrets.GH_TOKEN }}
 
 jobs:
-  # Dependency vulnerability scan
-  dependency-scan:
-    name: Dependency Security Scan
+  dependencies:
+    name: Dependency vulnerabilities
     runs-on: ubuntu-latest
-
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          token: ${{ secrets.GITEA_TOKEN }}
+      - uses: actions/checkout@v4
 
-      - name: Set up Python
-        uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
 
-      - name: Install dependencies
+      - name: Install pip-audit
         run: |
           python -m pip install --upgrade pip --root-user-action=ignore
-          pip install --root-user-action=ignore safety bandit semgrep
+          pip install --root-user-action=ignore pip-audit
 
-      - name: Run Safety check
-        run: |
-          safety check -r requirements.txt --json --output safety-report.json || true
-          safety check -r requirements-dev.txt --json --output safety-dev-report.json || true
+      - name: Runtime dependencies (gate)
+        run: pip-audit -r requirements.txt --strict --desc on
 
-      - name: Run Bandit security scan
-        run: |
-          bandit -r src/ -f json -o bandit-report.json || true
+      - name: Dev dependencies (report only)
+        run: pip-audit -r requirements-dev.txt --desc on || echo "::warning::dev-only dependency advisories above"
 
-      - name: Run Semgrep security scan
-        run: |
-          semgrep --config=auto src/ --json --output=semgrep-report.json || true
-
-      - name: Upload security reports
-        uses: actions/upload-artifact@v3
-        with:
-          name: security-reports-${{ github.run_number }}
-          path: |
-            safety-report.json
-            safety-dev-report.json
-            bandit-report.json
-            semgrep-report.json
-
-      - name: Check for critical vulnerabilities
-        run: |
-          echo "Checking for critical vulnerabilities..."
-
-          # Check Safety results
-          if [ -f safety-report.json ]; then
-            critical_count=$(jq '.vulnerabilities | length' safety-report.json 2>/dev/null || echo "0")
-            if [ "$critical_count" -gt 0 ]; then
-              echo "Found $critical_count dependency vulnerabilities"
-              jq '.vulnerabilities[] | "- \(.package_name) \(.installed_version): \(.vulnerability_id)"' safety-report.json
-            else
-              echo "No dependency vulnerabilities found"
-            fi
-          fi
-
-          # Check Bandit results
-          if [ -f bandit-report.json ]; then
-            high_severity=$(jq '.results[] | select(.issue_severity == "HIGH") | length' bandit-report.json 2>/dev/null | wc -l)
-            if [ "$high_severity" -gt 0 ]; then
-              echo "Found $high_severity high-severity security issues"
-            else
-              echo "No high-severity security issues found"
-            fi
-          fi
-
-  # License compliance check
-  license-check:
-    name: License Compliance
+  code:
+    name: Static analysis
     runs-on: ubuntu-latest
-
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          token: ${{ secrets.GITEA_TOKEN }}
+      - uses: actions/checkout@v4
 
-      - name: Set up Python
-        uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
 
-      - name: Install pip-licenses
+      - name: Install bandit
         run: |
           python -m pip install --upgrade pip --root-user-action=ignore
-          pip install --root-user-action=ignore pip-licenses
-          pip install --root-user-action=ignore -r requirements.txt
+          pip install --root-user-action=ignore bandit
 
-      - name: Check licenses
+      - name: bandit (HIGH fails; medium/low listed)
         run: |
-          echo "Checking dependency licenses..."
+          bandit -r src/ -q --skip B104 -ll -ii || true
+          bandit -r src/ -q --skip B104 --severity-level high --confidence-level medium
+
+  licenses:
+    name: License report
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: pip
+          cache-dependency-path: requirements.txt
+
+      # A fresh venv, not the runner's site-packages: the report must list the
+      # project's runtime deps, not whatever the runner image or a previous
+      # workflow happened to leave installed (semgrep once showed up here).
+      - name: Install into a clean venv
+        run: |
+          python -m venv .lic && . .lic/bin/activate
+          pip install --upgrade pip --root-user-action=ignore
+          pip install --root-user-action=ignore -r requirements.txt pip-licenses
+
+      - name: Report
+        run: |
+          . .lic/bin/activate
+          pip-licenses --format=markdown --with-urls --output-file=licenses.md
           pip-licenses --format=json --output-file=licenses.json
-          pip-licenses --format=markdown --output-file=licenses.md
+          echo "Copyleft licenses among runtime deps (informational; LGPL is fine to link from MIT):"
+          pip-licenses --format=plain --ignore-packages pip-licenses | grep -iE 'GPL|AGPL|LGPL' || echo "  none"
 
-          # Check for problematic licenses
-          problematic_licenses=("GPL" "AGPL" "LGPL")
-
-          for license in "${problematic_licenses[@]}"; do
-            if grep -i "$license" licenses.json; then
-              echo "Found potentially problematic license: $license"
-            fi
-          done
-
-          echo "License check completed"
-
-      - name: Upload license report
-        uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v3
         with:
-          name: license-report-${{ github.run_number }}
+          name: licenses-${{ github.run_number }}
           path: |
-            licenses.json
             licenses.md
-
-  # Dependency update check
-  dependency-update:
-    name: Check for Dependency Updates
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          token: ${{ secrets.GITEA_TOKEN }}
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
-
-      - name: Install pip-check-updates equivalent
-        run: |
-          python -m pip install --upgrade pip --root-user-action=ignore
-          pip install --root-user-action=ignore pip-review
-
-      - name: Check for outdated packages
-        run: |
-          echo "Checking for outdated packages..."
-          pip install --root-user-action=ignore -r requirements.txt
-          pip list --outdated --format=json > outdated-packages.json || true
-
-          if [ -s outdated-packages.json ]; then
-            echo "Outdated packages found:"
-            cat outdated-packages.json | jq -r '.[] | "- \(.name): \(.version) -> \(.latest_version)"'
-          else
-            echo "All packages are up to date"
-          fi
-
-      - name: Upload dependency reports
-        uses: actions/upload-artifact@v3
-        with:
-          name: dependency-reports-${{ github.run_number }}
-          path: |
-            outdated-packages.json
-
-  # Code quality metrics
-  code-quality:
-    name: Code Quality Metrics
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          token: ${{ secrets.GITEA_TOKEN }}
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
-
-      - name: Install quality tools
-        run: |
-          python -m pip install --upgrade pip --root-user-action=ignore
-          pip install --root-user-action=ignore radon xenon vulture
-          pip install --root-user-action=ignore -r requirements.txt
-
-      - name: Calculate code complexity
-        run: |
-          echo "Calculating code complexity..."
-          radon cc src/ --json > complexity-report.json
-          radon mi src/ --json > maintainability-report.json
-
-          echo "Complexity Summary:"
-          radon cc src/ --average
-
-          echo "Maintainability Summary:"
-          radon mi src/
-
-      - name: Find dead code
-        run: |
-          echo "Checking for dead code..."
-          vulture src/ --json > dead-code-report.json || true
-
-      - name: Check for code smells
-        run: |
-          echo "Checking for code smells..."
-          xenon --max-absolute B --max-modules A --max-average A src/ || true
-
-      - name: Upload quality reports
-        uses: actions/upload-artifact@v3
-        with:
-          name: code-quality-reports-${{ github.run_number }}
-          path: |
-            complexity-report.json
-            maintainability-report.json
-            dead-code-report.json
-
-  # Security summary
-  security-summary:
-    name: Security Summary
-    runs-on: ubuntu-latest
-    needs: [dependency-scan, license-check, code-quality]
-    if: always()
-
-    steps:
-      - name: Download all artifacts
-        uses: actions/download-artifact@v3
-
-      - name: Generate security summary
-        run: |
-          echo "# Security Scan Summary" > security-summary.md
-          echo "" >> security-summary.md
-          echo "**Scan Date:** $(date -u)" >> security-summary.md
-          echo "**Repository:** ${{ github.repository }}" >> security-summary.md
-          echo "**Commit:** ${{ github.sha }}" >> security-summary.md
-          echo "" >> security-summary.md
-
-          echo "## Results" >> security-summary.md
-          echo "" >> security-summary.md
-
-          # Dependency scan results
-          if [ -f security-reports-*/safety-report.json ]; then
-            vuln_count=$(jq '.vulnerabilities | length' security-reports-*/safety-report.json 2>/dev/null || echo "0")
-            if [ "$vuln_count" -eq 0 ]; then
-              echo "- Dependency Scan: No vulnerabilities found" >> security-summary.md
-            else
-              echo "- Dependency Scan: $vuln_count vulnerabilities found" >> security-summary.md
-            fi
-          else
-            echo "- Dependency Scan: Results not available" >> security-summary.md
-          fi
-
-          # Docker scan results (removed Trivy)
-          echo "- Docker Scan: Skipped (Trivy removed)" >> security-summary.md
-
-          # License check results
-          if [ -f license-report-*/licenses.json ]; then
-            echo "- License Check: Completed" >> security-summary.md
-          else
-            echo "- License Check: Results not available" >> security-summary.md
-          fi
-
-          # Code quality results
-          if [ -f code-quality-reports-*/complexity-report.json ]; then
-            echo "- Code Quality: Analyzed" >> security-summary.md
-          else
-            echo "- Code Quality: Results not available" >> security-summary.md
-          fi
-
-          echo "" >> security-summary.md
-          echo "## Detailed Reports" >> security-summary.md
-          echo "" >> security-summary.md
-          echo "Detailed reports are available in the workflow artifacts." >> security-summary.md
-
-          cat security-summary.md
-
-      - name: Upload security summary
-        uses: actions/upload-artifact@v3
-        with:
-          name: security-summary-${{ github.run_number }}
-          path: security-summary.md
+            licenses.json

.gitignore

@@ -134,4 +134,43 @@ cython_debug/
 
 # Docker volumes
 vm_data/
-grafana_data/
+grafana_data/
+# Runtime station config (persisted CRUD); bundled default lives in src/data/
+/stations.json
+
+# Ruflo local secrets and runtime data
+.env.*.local
+.claude-flow/data/
+.claude-flow/logs/
+.claude-flow/sessions/
+
+# Trained flood-forecast model artifacts (produced on the server, ~100 MB; see docs/FLOOD_FORECASTING.md)
+models/*.joblib
+models/*.json.gz
+models/cache/
+models/metrics.json
+# scripts/retrain.sh working dirs (staging + one rollback generation)
+models/.staging/
+models/.previous/
+
+# Playwright MCP browser artifacts (screenshots/snapshots from agent sessions)
+.playwright-mcp/
+
+# Agent tooling state (whole dirs; the entries above only covered subpaths)
+.claude/
+.claude-flow/
+.swarm/
+# local MCP server wiring, not project config
+.mcp.json
+# CLAUDE.md is intentionally NOT ignored — track it if you want the agent
+# conventions shared with collaborators; it is untracked today.
+
+# Model evaluation output (regenerate with scripts/evaluate_variants.py)
+models/eval_*.json
+
+# Editor/merge leftovers and stray shell-redirect artifacts. The repo root once
+# collected 56 zero-byte files named after fragments of shell commands.
+*.orig
+*.rej
+*.bak
+*~

.gitlab-ci.yml

@@ -1,129 +0,0 @@
-# GitLab CI/CD Pipeline for Northern Thailand Ping River Monitor
-
-stages:
-  - test
-  - build
-  - deploy
-
-variables:
-  PYTHON_VERSION: "3.11"
-  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
-
-cache:
-  paths:
-    - .cache/pip
-    - venv/
-
-# Test stage
-test:
-  stage: test
-  image: python:${PYTHON_VERSION}-slim
-  before_script:
-    - apt-get update && apt-get install -y build-essential
-    - python -m venv venv
-    - source venv/bin/activate
-    - pip install --upgrade pip
-    - pip install -r requirements-dev.txt
-  script:
-    - python test_integration.py
-    - python test_station_management.py
-    - flake8 src/ --max-line-length=100
-    - mypy src/
-  coverage: '/TOTAL.*\s+(\d+%)$/'
-  artifacts:
-    reports:
-      coverage_report:
-        coverage_format: cobertura
-        path: coverage.xml
-    paths:
-      - htmlcov/
-    expire_in: 1 week
-
-# Code quality
-code_quality:
-  stage: test
-  image: python:${PYTHON_VERSION}-slim
-  before_script:
-    - python -m venv venv
-    - source venv/bin/activate
-    - pip install black isort flake8 mypy
-  script:
-    - black --check src/ *.py
-    - isort --check-only src/ *.py
-    - flake8 src/ --max-line-length=100
-    - mypy src/
-  allow_failure: true
-
-# Security scan
-security_scan:
-  stage: test
-  image: python:${PYTHON_VERSION}-slim
-  before_script:
-    - pip install safety bandit
-  script:
-    - safety check -r requirements.txt
-    - bandit -r src/
-  allow_failure: true
-
-# Build Docker image
-build:
-  stage: build
-  image: docker:latest
-  services:
-    - docker:dind
-  before_script:
-    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-  script:
-    - docker build -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA .
-    - docker build -t $CI_REGISTRY_IMAGE:latest .
-    - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
-    - docker push $CI_REGISTRY_IMAGE:latest
-  only:
-    - main
-    - develop
-
-# Deploy to staging
-deploy_staging:
-  stage: deploy
-  image: alpine:latest
-  before_script:
-    - apk add --no-cache curl
-  script:
-    - echo "Deploying to staging environment"
-    - curl -X POST "$STAGING_WEBHOOK_URL" -H "Content-Type: application/json" -d '{"image":"'$CI_REGISTRY_IMAGE:$CI_COMMIT_SHA'"}'
-  environment:
-    name: staging
-    url: https://staging.ping-river-monitor.example.com
-  only:
-    - develop
-
-# Deploy to production
-deploy_production:
-  stage: deploy
-  image: alpine:latest
-  before_script:
-    - apk add --no-cache curl
-  script:
-    - echo "Deploying to production environment"
-    - curl -X POST "$PRODUCTION_WEBHOOK_URL" -H "Content-Type: application/json" -d '{"image":"'$CI_REGISTRY_IMAGE:$CI_COMMIT_SHA'"}'
-  environment:
-    name: production
-    url: https://ping-river-monitor.example.com
-  when: manual
-  only:
-    - main
-
-# Health check after deployment
-health_check:
-  stage: deploy
-  image: alpine:latest
-  before_script:
-    - apk add --no-cache curl jq
-  script:
-    - sleep 30  # Wait for deployment
-    - curl -f $HEALTH_CHECK_URL/health
-    - curl -s $HEALTH_CHECK_URL/metrics | jq .
-  dependencies:
-    - deploy_production
-  only:
-    - main

.pre-commit-config.yaml

@@ -0,0 +1,38 @@
+# Pre-commit hooks for Northern Thailand Ping River Monitor
+# See https://pre-commit.com for more information
+
+repos:
+  # General file checks
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-json
+      - id: check-toml
+      - id: check-added-large-files
+        args: ['--maxkb=1000']
+      - id: check-merge-conflict
+      - id: check-case-conflict
+      - id: mixed-line-ending
+
+  # Python code formatting with Black
+  - repo: https://github.com/psf/black
+    rev: 26.5.1
+    hooks:
+      - id: black
+        language_version: python3
+
+  # Import sorting with isort
+  - repo: https://github.com/pycqa/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+
+  # Linting with flake8
+  - repo: https://github.com/pycqa/flake8
+    rev: 6.1.0
+    hooks:
+      - id: flake8
+        args: ['--max-line-length=100', '--extend-ignore=E203,W503']

CLAUDE.md

@@ -0,0 +1,71 @@
+# CLAUDE.md
+
+Guidance for AI coding agents working in this repository.
+
+## What this is
+
+Flood monitoring and forecasting for the Ping River, Chiang Mai. Public dashboard and
+API at https://water.buildfor.life/ (never publish the server's private/mesh IP).
+Production: one systemd unit, `ping-collector.service` (the Rust binary in `rust/`),
+on a small VPS in `/opt/thailand-water-monitor` (a git checkout), user `water-monitor`;
+plus the monthly `water-monitor-retrain.timer` (Python, `.venv`, uv-managed). Updated by
+`git pull`, `cargo build --release`, `sudo bash scripts/install.sh`.
+
+## Rules
+
+- Python 3.11 only. `uv sync --python 3.11`; run everything as `uv run ...`.
+- `make format` (black 88 / isort black profile, config in pyproject.toml; cargo fmt)
+  before committing; CI fails on formatting. `make test` must stay green. Tests use
+  synthetic data, never the network; database tests (both languages) run against a
+  throwaway PostgreSQL schema from `TEST_DATABASE_URL` and skip without it.
+- PostgreSQL only. No other database backends.
+- Timestamps everywhere are Asia/Bangkok wall-clock with no offset. The dashboard
+  parses them with `parseTs()` and renders with `timeZone: TZ`; keep it that way.
+- Model changes go through the rolling-origin harness (`scripts/evaluate_variants.py`)
+  and are judged on first-alert LEAD and false alarms, not MAE. Record results, positive
+  or negative, in `docs/FLOOD_FORECASTING.md` section 5. Do not change what is deployed
+  (`rise_rain` / hgb-v3) without a harness result that beats it on lead.
+- `train_all()` must never silently produce a gauge-only (v2) model; the guard that
+  raises `RainUnavailableError` stays.
+- No `git add -A`: zero-byte shell-accident files (`#`, `$(wc`, ...) have been committed
+  before. Stage files by name.
+- Do not add Co-Authored-By trailers.
+- The dashboard is a single file, `src/static/dashboard.html`, EN + TH via the `t()`
+  table: every user-visible string needs both languages. Every path it fetches must be
+  a route in `rust/collector/src/web/mod.rs` (tests/test_dashboard.py checks).
+- Rust (`rust/`, see rust/README.md): `cargo fmt --all`, `cargo clippy --all-targets
+  -- -D warnings`, `cargo test`; minimum version 1.88
+  (`CARGO_TARGET_DIR=target/msrv cargo +1.88 check --all-targets --locked`). Read
+  `.env` only through `rust/collector/src/envfile.rs` (python-dotenv semantics, never
+  dotenvy: the Python tools read the same file).
+- The forecast is served by Rust (`rust/forecast`) from models trained in Python;
+  `src/ml/predict.py` is the reference it is tested against. A change to
+  `src/ml/features.py`, `predict.py`, the HII fill or the dam frame must be ported in
+  the same change: regenerate `rust/forecast/testdata/gen_forecast_fixtures.py` and keep
+  `cargo test -p ping-forecast` green. Changing thresholds, stages, leads or the
+  notifier settings (`src/ml/export.py`): `uv run python -m src.ml.export --config`
+  (a test fails otherwise) and rebuild the collector.
+- Notifier texts are pinned by `rust/collector/testdata/notify_python_cases.json.gz`;
+  a deliberate wording change updates that fixture in the same commit.
+- API routes live in `rust/collector/src/web`. A route's parameters are declared once
+  (`web/params.rs`); that list validates and documents (/docs). Keep response formats
+  stable: external clients use them.
+- The Python backfill stores (`src/hii_collector.py`, `src/rid_reservoir.py`,
+  `src/ml/rain.py`) write the same tables as `rust/collector/src/db.rs`: a table or
+  upsert change goes into both.
+
+## Where things are
+
+- `rust/collector` (`ping-collector`): collection (`rid.rs`, `hii.rs`, `dams.rs`,
+  `rain.rs`), forecast run (`forecasting.rs`), ntfy/Matrix (`notify.rs`), website/API
+  (`web/`), tables (`db.rs`). `rust/forecast`: inference.
+- `src/ml/`: features/train/evaluate/predict/export, `rain.py` (Open-Meteo), `dam.py`,
+  `hii_rain.py`. `src/hii_backfill.py`, `src/hii_collector.py`, `src/rid_reservoir.py`:
+  archive backfills (`scripts/backfill_*.py`).
+- `scripts/install.sh`: install/update on the server. `scripts/retrain.sh` +
+  `water-monitor-retrain.timer`: monthly retrain with staged promote.
+  `scripts/forecast_parity.py`: Rust vs Python forecast on a live database.
+  `scripts/dev_proxy.py`: serve the working-copy dashboard against the live API.
+- `docs/FLOOD_FORECASTING.md` is the authoritative model write-up; `docs/DATA_SOURCES.md`
+  the source catalog; `docs/NOTIFICATIONS.md` the alerts; `docs/RUST_REWRITE.md` the
+  history of the Python-to-Rust move.

CONTRIBUTING.md

@@ -1,277 +1,51 @@
-# Contributing to Northern Thailand Ping River Monitor
+# Contributing
 
-Thank you for your interest in contributing to the Northern Thailand Ping River Monitor! This document provides guidelines and information for contributors.
+Issues and merge requests on
+[git.b4l.co.th](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor).
+The repository has two halves: the service in Rust (`rust/`, see
+[rust/README.md](rust/README.md)) and the model training/evaluation stack in Python
+(`src/ml`, see [docs/FLOOD_FORECASTING.md](docs/FLOOD_FORECASTING.md)).
 
-## 🚀 Getting Started
+## Setup
 
-### Prerequisites
-- Python 3.9 or higher
-- Git
-- Basic knowledge of water monitoring systems
-- Familiarity with FastAPI and time-series databases (optional)
+- Python 3.11 and [uv](https://docs.astral.sh/uv/) (3.12+ has no pandas 2.0.3 wheels)
+- Rust >= 1.88 from [rustup](https://rustup.rs), plus `pkg-config` and `libssl-dev` on Linux
+- Docker (or any PostgreSQL 16) for the database tests
 
-### Development Setup
+```bash
+git clone https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor.git
+cd Northern-Thailand-Ping-River-Monitor
+make install                     # uv sync + pre-commit hooks
+docker run -d --name ping-pg -e POSTGRES_PASSWORD=ping -e POSTGRES_DB=ping -p 55432:5432 postgres:16
+export TEST_DATABASE_URL=postgres://postgres:ping@localhost:55432/ping
+make check                       # format check, flake8, clippy, pytest, cargo test, Rust 1.88
+```
 
-1. **Clone the repository:**
-   ```bash
-   git clone https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor.git
-   cd Northern-Thailand-Ping-River-Monitor
-   ```
+Without `TEST_DATABASE_URL` the database tests are skipped, the rest still run.
 
-2. **Create a virtual environment:**
-   ```bash
-   python -m venv venv
-   source venv/bin/activate  # On Windows: venv\Scripts\activate
-   ```
+## Before a merge request
 
-3. **Install dependencies:**
-   ```bash
-   pip install -r requirements.txt
-   ```
+- `make format`, then `make check` green. CI runs the same (`.gitea/workflows/`).
+- One logical change per commit, imperative subject (`fix(collector): ...`,
+  `feat(web): ...`, `docs: ...`).
+- Dashboard changes (`src/static/dashboard.html`): every visible string in English
+  and Thai (`t()` table); a screenshot in the merge request.
+- API changes: keep existing fields and formats; external clients read them. New
+  parameters go into the route's declaration in `rust/collector/src/web/params.rs`,
+  which also documents them at `/docs`.
+- Model changes: evaluate with `scripts/evaluate_variants.py` (rolling origin, judged
+  on first-alert lead and false alarms) and record the result in
+  `docs/FLOOD_FORECASTING.md` section 5, whether it helped or not. A feature change
+  must be ported to `rust/forecast` in the same change (see CLAUDE.md for the
+  fixture and config regeneration steps).
+- Never commit credentials, `.env`, trained models (`models/*.joblib`,
+  `models/*.json.gz`) or the server's private addresses.
 
-4. **Run tests:**
-   ```bash
-   python test_integration.py
-   ```
+## Useful commands
 
-5. **Start the development server:**
-   ```bash
-   python run.py --web-api
-   ```
-
-## 🔧 Development Guidelines
-
-### Code Style
-- Follow PEP 8 Python style guidelines
-- Use type hints for all function parameters and return values
-- Write descriptive docstrings for all classes and functions
-- Keep functions focused and under 50 lines when possible
-
-### Testing
-- Write tests for new features
-- Ensure all existing tests pass before submitting
-- Test both success and error cases
-- Include integration tests for API endpoints
-
-### Documentation
-- Update README.md for new features
-- Add docstrings to all new functions and classes
-- Update API documentation for new endpoints
-- Include examples in documentation
-
-## 📝 Contribution Process
-
-### 1. Issue Creation
-- Check existing issues before creating new ones
-- Use clear, descriptive titles
-- Provide detailed descriptions with steps to reproduce (for bugs)
-- Include system information and logs when relevant
-
-### 2. Pull Request Process
-1. **Fork the repository**
-2. **Create a feature branch:**
-   ```bash
-   git checkout -b feature/your-feature-name
-   ```
-3. **Make your changes**
-4. **Test thoroughly:**
-   ```bash
-   python test_integration.py
-   python test_station_management.py
-   ```
-5. **Commit with clear messages:**
-   ```bash
-   git commit -m "Add: New station validation feature"
-   ```
-6. **Push to your fork:**
-   ```bash
-   git push origin feature/your-feature-name
-   ```
-7. **Create a pull request**
-
-### Pull Request Guidelines
-- Use clear, descriptive titles
-- Provide detailed descriptions of changes
-- Reference related issues
-- Include screenshots for UI changes
-- Ensure all tests pass
-- Update documentation as needed
-
-## 🎯 Areas for Contribution
-
-### High Priority
-- **Additional Database Support** - MongoDB, TimescaleDB
-- **Authentication System** - User management and API keys
-- **Real-time Alerts** - Email/SMS notifications for critical conditions
-- **Mobile App** - React Native or Flutter interface
-- **Advanced Analytics** - Trend analysis and forecasting
-
-### Medium Priority
-- **Bulk Station Operations** - Import/export multiple stations
-- **Map Interface** - Visual station management
-- **Data Export** - CSV, Excel format support
-- **Performance Optimization** - Query optimization and caching
-- **Internationalization** - Multi-language support
-
-### Low Priority
-- **Additional Visualizations** - Custom chart types
-- **Report Generation** - Automated PDF reports
-- **Integration APIs** - Third-party service connections
-- **Advanced Logging** - Structured logging with ELK stack
-
-## 🐛 Bug Reports
-
-When reporting bugs, please include:
-
-1. **Environment Information:**
-   - Operating system and version
-   - Python version
-   - Database type and version
-   - Browser (for web interface issues)
-
-2. **Steps to Reproduce:**
-   - Clear, numbered steps
-   - Expected vs actual behavior
-   - Screenshots or error messages
-
-3. **Additional Context:**
-   - Log files (sanitized of sensitive data)
-   - Configuration details
-   - Recent changes to the system
-
-## 💡 Feature Requests
-
-For feature requests, please provide:
-
-1. **Use Case:** Why is this feature needed?
-2. **Description:** What should the feature do?
-3. **Acceptance Criteria:** How do we know it's complete?
-4. **Priority:** How important is this feature?
-5. **Alternatives:** Are there workarounds available?
-
-## 🔒 Security
-
-### Reporting Security Issues
-- **DO NOT** create public issues for security vulnerabilities
-- Email security issues to: [security contact]
-- Include detailed information about the vulnerability
-- Allow time for fixes before public disclosure
-
-### Security Guidelines
-- Never commit sensitive data (passwords, API keys)
-- Use environment variables for configuration
-- Validate all user inputs
-- Follow OWASP security guidelines
-- Keep dependencies updated
-
-## 📊 Database Contributions
-
-### Adding New Database Support
-1. Create adapter in `src/database_adapters.py`
-2. Implement all required methods
-3. Add configuration options in `src/config.py`
-4. Update documentation
-5. Add integration tests
-
-### Database Schema Changes
-- Provide migration scripts
-- Test with existing data
-- Document breaking changes
-- Update all database adapters
-
-## 🌐 API Contributions
-
-### Adding New Endpoints
-1. Define Pydantic models in `src/models.py`
-2. Implement endpoint in `src/web_api.py`
-3. Add input validation
-4. Include error handling
-5. Write comprehensive tests
-6. Update API documentation
-
-### API Guidelines
-- Use RESTful conventions
-- Include proper HTTP status codes
-- Provide clear error messages
-- Support pagination for large datasets
-- Include rate limiting considerations
-
-## 📈 Performance Guidelines
-
-### Optimization Areas
-- Database query efficiency
-- API response times
-- Memory usage
-- Concurrent request handling
-- Data processing speed
-
-### Monitoring
-- Use built-in metrics collection
-- Monitor response times
-- Track error rates
-- Watch resource usage
-- Set up alerts for issues
-
-## 🧪 Testing Guidelines
-
-### Test Types
-- **Unit Tests:** Individual function testing
-- **Integration Tests:** Component interaction testing
-- **API Tests:** Endpoint functionality testing
-- **Performance Tests:** Load and stress testing
-
-### Test Coverage
-- Aim for >80% code coverage
-- Test both success and failure paths
-- Include edge cases
-- Test with different database backends
-
-## 📚 Documentation Standards
-
-### Code Documentation
-- Use clear, descriptive variable names
-- Write comprehensive docstrings
-- Include type hints
-- Add inline comments for complex logic
-
-### User Documentation
-- Keep README.md updated
-- Provide clear installation instructions
-- Include usage examples
-- Document configuration options
-
-## 🤝 Community Guidelines
-
-### Communication
-- Be respectful and professional
-- Help newcomers get started
-- Share knowledge and experiences
-- Provide constructive feedback
-
-### Code Reviews
-- Review code thoroughly
-- Provide specific, actionable feedback
-- Be open to suggestions
-- Focus on code quality and maintainability
-
-## 📞 Getting Help
-
-### Resources
-- **Documentation:** README.md and docs/
-- **API Reference:** http://localhost:8000/docs
-- **Issues:** GitHub Issues section
-- **Discussions:** GitHub Discussions
-
-### Contact
-- **General Questions:** Create a GitHub issue
-- **Security Issues:** [security contact]
-- **Maintainer:** [maintainer contact]
-
-## 🎉 Recognition
-
-Contributors will be recognized in:
-- README.md contributors section
-- Release notes for significant contributions
-- Special recognition for major features
-
-Thank you for contributing to the Northern Thailand Ping River Monitor! Your contributions help improve water resource monitoring and management in the region.
+```bash
+make build && make serve          # the website/API from your database on 127.0.0.1:8001
+uv run python scripts/dev_proxy.py  # the working-copy dashboard against the live API
+uv run python scripts/evaluate_variants.py --stations P.1
+uv run python -m src.ml.export --config   # after changing thresholds or notifier settings
+```

DEPLOYMENT_CHECKLIST.md

@@ -1,268 +0,0 @@
-# 🚀 Deployment Checklist - Northern Thailand Ping River Monitor
-
-## ✅ Pre-Deployment Checklist
-
-### **Code Quality**
-- [ ] All tests pass (`make test`)
-- [ ] Code formatting applied (`make format`)
-- [ ] Linting checks pass (`make lint`)
-- [ ] No security vulnerabilities (`safety check`)
-- [ ] Documentation updated
-- [ ] Version number updated in `setup.py` and `src/__init__.py`
-
-### **Configuration**
-- [ ] Environment variables configured (`.env` file)
-- [ ] Database connection tested
-- [ ] API endpoints tested
-- [ ] Log levels appropriate for environment
-- [ ] Security settings configured (API keys, secrets)
-- [ ] Resource limits set (memory, CPU)
-
-### **Dependencies**
-- [ ] All required packages in `requirements.txt`
-- [ ] No unused dependencies
-- [ ] Security updates applied
-- [ ] Compatible Python version (3.9+)
-
-## 🐳 Docker Deployment
-
-### **Pre-Docker Checklist**
-- [ ] Dockerfile tested locally
-- [ ] Docker Compose configuration verified
-- [ ] Volume mounts configured correctly
-- [ ] Network settings configured
-- [ ] Health checks working
-- [ ] Resource limits set
-
-### **Docker Commands**
-```bash
-# Build and test locally
-make docker-build
-docker run --rm ping-river-monitor python run.py --test
-
-# Deploy with Docker Compose
-make docker-run
-
-# Verify deployment
-make health-check
-```
-
-### **Post-Docker Checklist**
-- [ ] All services running (`docker-compose ps`)
-- [ ] Health checks passing
-- [ ] Logs showing normal operation
-- [ ] API accessible (`curl http://localhost:8000/health`)
-- [ ] Database connectivity verified
-- [ ] Grafana dashboards loading
-
-## 🌐 Production Deployment
-
-### **Infrastructure Requirements**
-- [ ] Server specifications adequate (CPU, RAM, Storage)
-- [ ] Network connectivity to external APIs
-- [ ] SSL certificates configured (if HTTPS)
-- [ ] Firewall rules configured
-- [ ] Backup strategy implemented
-- [ ] Monitoring alerts configured
-
-### **Security Checklist**
-- [ ] API keys secured (environment variables)
-- [ ] Database credentials secured
-- [ ] HTTPS enabled for web interface
-- [ ] Input validation enabled
-- [ ] Rate limiting configured
-- [ ] Log sanitization enabled
-
-### **Performance Checklist**
-- [ ] Database indexes created
-- [ ] Connection pooling configured
-- [ ] Caching enabled where appropriate
-- [ ] Resource monitoring enabled
-- [ ] Performance baselines established
-
-## 📊 Monitoring Setup
-
-### **Health Monitoring**
-- [ ] Health check endpoints responding
-- [ ] Database health monitoring
-- [ ] API response time monitoring
-- [ ] Memory usage monitoring
-- [ ] Disk space monitoring
-
-### **Alerting**
-- [ ] Critical error alerts configured
-- [ ] Performance degradation alerts
-- [ ] Database connectivity alerts
-- [ ] Disk space alerts
-- [ ] API availability alerts
-
-### **Logging**
-- [ ] Log rotation configured
-- [ ] Log levels appropriate
-- [ ] Structured logging enabled
-- [ ] Log aggregation configured (if applicable)
-- [ ] Log retention policy set
-
-## 🔄 CI/CD Pipeline
-
-### **GitLab CI/CD**
-- [ ] `.gitlab-ci.yml` configured
-- [ ] Pipeline variables set
-- [ ] Test stage passing
-- [ ] Build stage creating artifacts
-- [ ] Deploy stage configured
-- [ ] Rollback procedure documented
-
-### **Pipeline Stages**
-- [ ] **Test**: Unit tests, integration tests, linting
-- [ ] **Build**: Docker image creation, artifact generation
-- [ ] **Deploy**: Staging deployment, production deployment
-- [ ] **Verify**: Health checks, smoke tests
-
-## 🗄️ Database Setup
-
-### **Database Configuration**
-- [ ] Database server running and accessible
-- [ ] Database created with correct permissions
-- [ ] Connection string configured
-- [ ] Migration scripts run (if applicable)
-- [ ] Backup strategy implemented
-- [ ] Performance tuning applied
-
-### **Database-Specific Checklist**
-
-#### **SQLite**
-- [ ] Database file permissions set correctly
-- [ ] WAL mode enabled for better concurrency
-- [ ] Regular backup scheduled
-
-#### **MySQL/PostgreSQL**
-- [ ] User accounts created with minimal privileges
-- [ ] Connection pooling configured
-- [ ] Query performance optimized
-- [ ] Replication configured (if applicable)
-
-#### **InfluxDB**
-- [ ] Retention policies configured
-- [ ] Continuous queries set up (if needed)
-- [ ] Backup strategy implemented
-
-#### **VictoriaMetrics**
-- [ ] Storage configuration optimized
-- [ ] Retention period set
-- [ ] Resource limits configured
-
-## 🌐 Web Interface
-
-### **API Deployment**
-- [ ] FastAPI server running
-- [ ] All endpoints responding correctly
-- [ ] API documentation accessible (`/docs`)
-- [ ] CORS configured correctly
-- [ ] Rate limiting working
-- [ ] Authentication configured (if applicable)
-
-### **Frontend Integration**
-- [ ] Grafana dashboards configured
-- [ ] Data sources connected
-- [ ] Visualizations working
-- [ ] Alerts configured
-- [ ] User access configured
-
-## 📈 Performance Verification
-
-### **Load Testing**
-- [ ] API endpoints tested under load
-- [ ] Database performance under load
-- [ ] Memory usage under load
-- [ ] Response times acceptable
-- [ ] Error rates acceptable
-
-### **Capacity Planning**
-- [ ] Expected data volume calculated
-- [ ] Storage growth projected
-- [ ] Scaling strategy documented
-- [ ] Resource monitoring thresholds set
-
-## 🔧 Operational Procedures
-
-### **Maintenance**
-- [ ] Update procedure documented
-- [ ] Backup and restore procedures tested
-- [ ] Rollback procedure documented
-- [ ] Monitoring runbooks created
-- [ ] Incident response procedures documented
-
-### **Documentation**
-- [ ] Deployment guide updated
-- [ ] API documentation current
-- [ ] Configuration documentation complete
-- [ ] Troubleshooting guide available
-- [ ] Contact information updated
-
-## ✅ Post-Deployment Verification
-
-### **Functional Testing**
-- [ ] Data collection working
-- [ ] API endpoints responding
-- [ ] Database writes successful
-- [ ] Web interface accessible
-- [ ] Station management working
-
-### **Integration Testing**
-- [ ] External API connectivity
-- [ ] Database integration
-- [ ] Monitoring integration
-- [ ] Alert system working
-- [ ] Backup system working
-
-### **Performance Testing**
-- [ ] Response times acceptable
-- [ ] Memory usage normal
-- [ ] CPU usage normal
-- [ ] Disk I/O normal
-- [ ] Network usage normal
-
-## 🚨 Rollback Plan
-
-### **Rollback Triggers**
-- [ ] Critical errors in production
-- [ ] Performance degradation
-- [ ] Data corruption
-- [ ] Security vulnerabilities
-- [ ] Service unavailability
-
-### **Rollback Procedure**
-1. [ ] Stop current deployment
-2. [ ] Restore previous Docker images
-3. [ ] Restore database backup (if needed)
-4. [ ] Verify system functionality
-5. [ ] Update monitoring and alerts
-6. [ ] Document incident and lessons learned
-
-## 📞 Support Information
-
-### **Emergency Contacts**
-- [ ] System administrator contact
-- [ ] Database administrator contact
-- [ ] Network administrator contact
-- [ ] Application developer contact
-
-### **Documentation Links**
-- [ ] Deployment guide
-- [ ] API documentation
-- [ ] Troubleshooting guide
-- [ ] Configuration reference
-- [ ] Monitoring dashboards
-
----
-
-**Deployment Date**: ___________  
-**Deployed By**: ___________  
-**Version**: v3.1.3  
-**Environment**: ___________  
-
-**Sign-off**:
-- [ ] Technical Lead: ___________
-- [ ] Operations Team: ___________
-- [ ] Security Team: ___________

Dockerfile

@@ -1,58 +0,0 @@
-# Multi-stage Docker build for Northern Thailand Ping River Monitor
-
-# Build stage
-FROM python:3.11-slim as builder
-
-# Set working directory
-WORKDIR /app
-
-# Install build dependencies
-RUN apt-get update && apt-get install -y \
-    build-essential \
-    gcc \
-    && rm -rf /var/lib/apt/lists/*
-
-# Copy requirements and install Python dependencies
-COPY requirements.txt .
-RUN pip install --no-cache-dir --user -r requirements.txt
-
-# Production stage
-FROM python:3.11-slim
-
-# Set working directory
-WORKDIR /app
-
-# Install runtime dependencies and create user
-RUN apt-get update && apt-get install -y \
-    wget \
-    curl \
-    && rm -rf /var/lib/apt/lists/* \
-    && groupadd -r appuser && useradd -r -g appuser appuser \
-    && mkdir -p /home/appuser/.local
-
-# Copy Python packages from builder stage
-COPY --from=builder /root/.local /home/appuser/.local
-
-# Copy application code
-COPY . .
-
-# Create logs directory and set permissions
-RUN mkdir -p logs && chown -R appuser:appuser /app /home/appuser/.local
-
-# Set environment variables
-ENV PYTHONUNBUFFERED=1
-ENV TZ=Asia/Bangkok
-ENV PATH=/home/appuser/.local/bin:$PATH
-
-# Switch to non-root user
-USER appuser
-
-# Health check for API mode
-HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
-    CMD python -c "import requests; requests.get('http://localhost:8000/health', timeout=5)" 2>/dev/null || exit 1
-
-# Expose port for API
-EXPOSE 8000
-
-# Default command
-CMD ["python", "run.py", "--web-api"]

FINAL_CHECKLIST.md

@@ -1,193 +0,0 @@
-# Final GitHub Publication Checklist ✅
-
-This checklist ensures the Thailand Water Level Monitor project is ready for GitHub publication.
-
-## 🎯 **Project Preparation Complete**
-
-### ✅ **Core Repository Files**
-- [x] **README.md** - Comprehensive project documentation with badges and quick start
-- [x] **LICENSE** - MIT License for open source distribution
-- [x] **CONTRIBUTING.md** - Detailed contributor guidelines
-- [x] **.gitignore** - Comprehensive ignore rules for all file types
-- [x] **requirements.txt** - All Python dependencies listed and tested
-
-### ✅ **Source Code Organization**
-- [x] **src/** directory created with clean separation
-- [x] **scripts/** directory for utility scripts and system files
-- [x] **docs/** directory with comprehensive documentation
-- [x] **grafana/** directory with visualization configuration
-- [x] All temporary files removed (*.db, *.log, __pycache__)
-
-### ✅ **Documentation Quality**
-- [x] **Installation guides** for all platforms and databases
-- [x] **Configuration examples** for 5 different database types
-- [x] **Troubleshooting guides** for common deployment issues
-- [x] **Migration guides** for updating existing systems
-- [x] **API references** documenting Thai government data sources
-- [x] **Notable documents** section with official resources
-
-### ✅ **Production Readiness**
-- [x] **Docker support** with Dockerfile and docker-compose
-- [x] **Systemd service** configuration for Linux deployment
-- [x] **Multi-database support** (SQLite, PostgreSQL, MySQL, InfluxDB, VictoriaMetrics)
-- [x] **Geolocation support** for Grafana geomap visualization
-- [x] **Migration scripts** for safe database schema updates
-- [x] **HTTPS configuration** guide for secure deployment
-
-### ✅ **Code Quality**
-- [x] **Modular architecture** with clean separation of concerns
-- [x] **Error handling** and comprehensive logging
-- [x] **Configuration management** via environment variables
-- [x] **Database abstraction** layer for multiple backends
-- [x] **Testing utilities** (demo_databases.py)
-
-### ✅ **Features Verified**
-- [x] **Real-time data collection** from 16 Thai water stations
-- [x] **15-minute scheduling** with intelligent retry logic
-- [x] **Gap filling** for missing historical data
-- [x] **Data validation** and error recovery
-- [x] **Geolocation integration** with sample coordinates
-- [x] **Grafana dashboards** with pre-built visualizations
-
-## 🚀 **Ready for GitHub Publication**
-
-### **Repository Information**
-- **Name**: `thailand-water-monitor`
-- **Description**: "Real-time water level monitoring system for Thailand's Royal Irrigation Department stations with Grafana visualization"
-- **Topics**: `water-monitoring`, `thailand`, `grafana`, `timeseries`, `python`, `iot`, `environmental-monitoring`
-- **License**: MIT
-- **Language**: Python
-
-### **Repository Settings**
-- [x] Enable Issues for bug reports and feature requests
-- [x] Enable Discussions for community support
-- [x] Enable Wiki for extended documentation
-- [x] Set up GitHub Pages for documentation hosting
-- [x] Configure branch protection for main branch
-
-### **Initial Release (v1.0.0)**
-- **Release Title**: "Thailand Water Level Monitor v1.0.0 - Complete Monitoring Solution"
-- **Release Notes**: 
-  - Complete real-time monitoring system
-  - Multi-database backend support
-  - Grafana geomap integration
-  - Production-ready deployment
-  - Comprehensive documentation
-
-## 📊 **Project Statistics**
-
-### **Code Metrics**
-- **Total Files**: 25+ files
-- **Python Source Files**: 4 main modules
-- **Documentation Files**: 12 comprehensive guides
-- **Configuration Files**: 6 deployment configurations
-- **Lines of Code**: ~2,000+ lines of Python
-- **Documentation**: ~15,000+ words
-
-### **Feature Coverage**
-- **Database Backends**: 5 different types supported
-- **Monitoring Stations**: 16 across Thailand
-- **Data Collection**: Every 15 minutes
-- **Data Points**: ~300 measurements per collection cycle
-- **Geolocation**: GPS coordinates and geohash support
-- **Visualization**: Pre-built Grafana dashboards
-
-### **Documentation Coverage**
-- **Installation**: Complete setup for all platforms
-- **Configuration**: All database types documented
-- **Deployment**: Docker, systemd, manual options
-- **Troubleshooting**: Common issues and solutions
-- **Migration**: Safe upgrade procedures
-- **API**: External data source documentation
-
-## 🌟 **Key Selling Points**
-
-### **For Water Management Professionals**
-- Real-time monitoring of 16 stations across Thailand
-- Historical data analysis and trend visualization
-- Alert capabilities for critical water levels
-- Integration with official Thai government data sources
-
-### **For Developers**
-- Clean, modular Python codebase
-- Multiple database backend options
-- Docker containerization for easy deployment
-- Comprehensive API documentation
-
-### **For System Administrators**
-- Production-ready deployment configurations
-- Systemd service integration
-- HTTPS and security configuration
-- Monitoring and logging capabilities
-
-### **For Data Scientists**
-- Time-series data with geolocation
-- Grafana visualization and analysis tools
-- Historical data gap filling
-- Export capabilities for further analysis
-
-## 🎯 **Post-Publication Roadmap**
-
-### **Immediate (Week 1)**
-- [ ] Create GitHub repository and upload files
-- [ ] Set up initial release v1.0.0
-- [ ] Configure repository settings and templates
-- [ ] Create project documentation website
-
-### **Short-term (Month 1)**
-- [ ] Add GitHub Actions for CI/CD
-- [ ] Create issue and PR templates
-- [ ] Set up automated testing
-- [ ] Add code quality badges
-
-### **Medium-term (Quarter 1)**
-- [ ] Community feedback integration
-- [ ] Additional database backends
-- [ ] Mobile app development
-- [ ] Advanced alerting system
-
-### **Long-term (Year 1)**
-- [ ] Predictive analytics features
-- [ ] Machine learning integration
-- [ ] Multi-country expansion
-- [ ] Commercial support options
-
-## 🏆 **Success Metrics**
-
-### **Community Engagement**
-- GitHub stars and forks
-- Issue reports and feature requests
-- Community contributions
-- Documentation feedback
-
-### **Technical Adoption**
-- Download and deployment statistics
-- Database backend usage patterns
-- Performance benchmarks
-- User success stories
-
-### **Impact Measurement**
-- Water management improvements
-- Early warning system effectiveness
-- Data accessibility improvements
-- Research and academic usage
-
----
-
-## ✅ **FINAL VERIFICATION**
-
-**All checklist items completed successfully!**
-
-The Thailand Water Level Monitor project is now:
-- ✅ **Professionally organized** with clean structure
-- ✅ **Comprehensively documented** with guides for all use cases
-- ✅ **Production ready** with multiple deployment options
-- ✅ **Community friendly** with contribution guidelines
-- ✅ **Feature complete** with real-time monitoring capabilities
-
-**🚀 Ready for GitHub publication and community engagement!** 🌊
-
----
-
-*Last updated: July 30, 2025*
-*Project status: Ready for publication*

GITEA_SETUP_SUMMARY.md

@@ -1,233 +0,0 @@
-# 🎉 Gitea Actions Setup Complete!
-
-## 🚀 **What's Been Created**
-
-Your **Northern Thailand Ping River Monitor** now has a complete CI/CD pipeline with Gitea Actions! Here's what's been set up:
-
-### **🔄 Gitea Actions Workflows**
-
-```
-.gitea/workflows/
-├── ci.yml          # Main CI/CD pipeline
-├── release.yml     # Automated releases
-├── security.yml    # Security & dependency scanning
-└── docs.yml        # Documentation generation
-```
-
-### **📊 Workflow Features**
-
-#### **1. CI/CD Pipeline (`ci.yml`)**
-- ✅ **Multi-Python Testing** (3.9, 3.10, 3.11, 3.12)
-- ✅ **Code Quality Checks** (flake8, mypy, black, isort)
-- ✅ **Docker Multi-Arch Builds** (amd64, arm64)
-- ✅ **Integration Testing** with VictoriaMetrics
-- ✅ **Automated Staging Deployment** (develop branch)
-- ✅ **Manual Production Deployment** (main branch)
-- ✅ **Performance Testing** after deployment
-
-#### **2. Release Management (`release.yml`)**
-- 🏷️ **Tag-Based Releases** (`v*.*.*` pattern)
-- 📝 **Automatic Changelog Generation**
-- 🐳 **Multi-Architecture Docker Images**
-- 🔒 **Security Scanning** before release
-- ✅ **Comprehensive Validation** after deployment
-
-#### **3. Security Monitoring (`security.yml`)**
-- 🔒 **Daily Security Scans** (3 AM UTC)
-- 📦 **Dependency Vulnerability Detection**
-- 🐳 **Docker Image Security Scanning**
-- 📄 **License Compliance Checking**
-- 📊 **Code Quality Metrics**
-- 🔄 **Automated Update Notifications**
-
-#### **4. Documentation (`docs.yml`)**
-- 📚 **API Documentation Generation**
-- 🔗 **Link Validation**
-- 📖 **Sphinx Documentation Building**
-- ✅ **Documentation Completeness Checking**
-
-## 🔧 **Setup Instructions**
-
-### **1. Configure Repository Secrets**
-
-In your Gitea repository settings, add these secrets:
-
-```bash
-# Required
-GITEA_TOKEN          # For container registry access
-
-# Optional (for notifications)
-SLACK_WEBHOOK_URL    # Slack notifications
-STAGING_WEBHOOK_URL  # Staging deployment webhook
-PRODUCTION_WEBHOOK_URL # Production deployment webhook
-```
-
-### **2. Enable Actions**
-
-1. Go to your repository settings in Gitea
-2. Enable "Actions" if not already enabled
-3. Configure runners if using self-hosted runners
-
-### **3. Push to Repository**
-
-```bash
-# Initialize and push
-git init
-git remote add origin https://git.b4l.co.th/grabowski/Northern-Thailand-Ping-River-Monitor.git
-git add .
-git commit -m "Initial commit with Gitea Actions workflows"
-git push -u origin main
-```
-
-## 🎯 **Workflow Triggers**
-
-### **Automatic Triggers**
-- **Push to main/develop** → CI/CD Pipeline
-- **Pull Request to main** → Testing & Validation
-- **Daily at 2 AM UTC** → CI/CD Health Check
-- **Daily at 3 AM UTC** → Security Scanning
-- **Git Tag `v*.*.*`** → Release Pipeline
-- **Documentation Changes** → Documentation Build
-
-### **Manual Triggers**
-- **Manual Dispatch** → Any workflow can be triggered manually
-- **Release Creation** → Manual release with custom version
-
-## 📊 **Monitoring & Status**
-
-### **Status Badges**
-Your README now includes comprehensive status badges:
-- CI/CD Pipeline Status
-- Security Scan Status  
-- Documentation Build Status
-- Python Version Support
-- FastAPI Version
-- Docker Ready
-- License Information
-- Current Version
-
-### **Workflow Artifacts**
-Each workflow generates useful artifacts:
-- **Test Results** and coverage reports
-- **Security Scan Reports** (JSON format)
-- **Docker Images** (multi-architecture)
-- **Documentation** (HTML and PDF)
-- **Performance Reports**
-
-## 🚀 **Usage Examples**
-
-### **Development Workflow**
-```bash
-# Create feature branch
-git checkout -b feature/new-station-type
-# Make changes
-git add .
-git commit -m "Add support for new station type"
-git push origin feature/new-station-type
-# Create PR in Gitea → Triggers testing
-```
-
-### **Release Workflow**
-```bash
-# Create and push release tag
-git tag v3.1.1
-git push origin v3.1.1
-# → Triggers automated release pipeline
-```
-
-### **Security Monitoring**
-- **Daily scans** run automatically
-- **Security reports** available in Actions artifacts
-- **Notifications** sent for critical vulnerabilities
-
-## 🔍 **Validation Commands**
-
-Test your setup locally:
-
-```bash
-# Validate workflow syntax
-make validate-workflows
-
-# Test workflow components
-make workflow-test
-
-# Run full test suite
-make test
-
-# Build Docker image
-make docker-build
-```
-
-## 📈 **Performance & Optimization**
-
-### **Caching Strategy**
-- **Pip dependencies** cached across runs
-- **Docker layers** cached for faster builds
-- **Workflow artifacts** retained for analysis
-
-### **Parallel Execution**
-- **Matrix builds** for multiple Python versions
-- **Independent jobs** for security and testing
-- **Conditional execution** to skip unnecessary steps
-
-### **Resource Management**
-- **Appropriate timeouts** prevent hanging workflows
-- **Artifact cleanup** manages storage usage
-- **Efficient Docker builds** with multi-stage approach
-
-## 🔒 **Security Best Practices**
-
-### **Implemented Security**
-- ✅ **Secret management** via Gitea repository secrets
-- ✅ **Multi-stage Docker builds** for minimal attack surface
-- ✅ **Non-root containers** for better security
-- ✅ **Vulnerability scanning** before deployment
-- ✅ **Dependency monitoring** with automated alerts
-
-### **Security Scanning Coverage**
-- **Python dependencies** (Safety, Bandit)
-- **Docker images** (Trivy)
-- **Code quality** (Semgrep)
-- **License compliance** (pip-licenses)
-
-## 📚 **Documentation**
-
-### **Available Documentation**
-- [Gitea Workflows Guide](docs/GITEA_WORKFLOWS.md) - Detailed workflow documentation
-- [Contributing Guide](CONTRIBUTING.md) - How to contribute
-- [Deployment Checklist](DEPLOYMENT_CHECKLIST.md) - Production deployment
-- [Project Structure](docs/PROJECT_STRUCTURE.md) - Architecture overview
-
-### **Generated Documentation**
-- **API Documentation** - Auto-generated from OpenAPI spec
-- **Code Documentation** - Sphinx-generated from docstrings
-- **Security Reports** - Automated vulnerability reports
-
-## 🎉 **Ready for Production!**
-
-Your repository is now equipped with:
-
-- 🔄 **Enterprise-grade CI/CD pipeline**
-- 🔒 **Comprehensive security monitoring**
-- 📊 **Automated quality assurance**
-- 🚀 **Streamlined release management**
-- 📚 **Automated documentation**
-- 🐳 **Multi-architecture Docker support**
-- 📈 **Performance monitoring**
-- 🔍 **Comprehensive testing**
-
-## 🚀 **Next Steps**
-
-1. **Push to Gitea** and watch the workflows run
-2. **Configure deployment environments** (staging/production)
-3. **Set up monitoring dashboards** for workflow metrics
-4. **Configure notifications** for team collaboration
-5. **Create your first release** with `git tag v3.1.3`
-
-Your **Northern Thailand Ping River Monitor** is now ready for professional development and deployment! 🎊
-
----
-
-**Workflow Version**: v3.1.3  
-**Setup Date**: 2025-08-12  
-**Repository**: https://git.b4l.co.th/grabowski/Northern-Thailand-Ping-River-Monitor

GITHUB_PUBLICATION_SUMMARY.md

@@ -1,203 +0,0 @@
-# GitHub Publication Summary
-
-This document summarizes the Thailand Water Level Monitor project preparation for GitHub publication.
-
-## 📁 **Final Project Structure**
-
-```
-thailand-water-monitor/
-├── 📄 README.md                           # Main project documentation
-├── 📄 LICENSE                             # MIT License
-├── 📄 CONTRIBUTING.md                     # Contributor guidelines
-├── 📄 requirements.txt                    # Python dependencies
-├── 📄 .gitignore                          # Git ignore rules
-├── 📄 Dockerfile                          # Container definition
-├── 📄 docker-compose.victoriametrics.yml # Complete stack deployment
-│
-├── 📂 src/                                # Source Code
-│   ├── 🐍 water_scraper_v3.py           # Main application
-│   ├── 🐍 database_adapters.py          # Multi-database support
-│   ├── 🐍 config.py                     # Configuration management
-│   └── 🐍 demo_databases.py             # Database testing utility
-│
-├── 📂 scripts/                           # Utility Scripts
-│   ├── 🐍 migrate_geolocation.py        # Database migration script
-│   └── ⚙️ water-monitor.service          # Systemd service file
-│
-├── 📂 docs/                              # Documentation
-│   ├── 📖 DATABASE_DEPLOYMENT_GUIDE.md   # Complete setup guide
-│   ├── 📖 ENHANCED_SCHEDULER_GUIDE.md    # 15-minute scheduling
-│   ├── 📖 GEOLOCATION_GUIDE.md          # Grafana geomap integration
-│   ├── 📖 GAP_FILLING_GUIDE.md          # Data integrity management
-│   ├── 📖 MIGRATION_QUICKSTART.md       # Quick migration guide
-│   ├── 📖 VICTORIAMETRICS_SETUP.md      # High-performance deployment
-│   ├── 📖 HTTPS_CONFIGURATION.md        # Secure deployment
-│   ├── 📖 DEBIAN_TROUBLESHOOTING.md     # Linux deployment issues
-│   ├── 📖 PROJECT_STATUS.md             # Development status
-│   └── 📂 references/
-│       └── 📖 NOTABLE_DOCUMENTS.md       # Official Thai government resources
-│
-└── 📂 grafana/                          # Grafana Configuration
-    ├── 📂 dashboards/
-    │   └── 📊 water-monitoring-dashboard.json
-    └── 📂 provisioning/
-        ├── 📂 dashboards/
-        │   └── ⚙️ dashboard.yml
-        └── 📂 datasources/
-            └── ⚙️ victoriametrics.yml
-```
-
-## ✅ **GitHub Readiness Checklist**
-
-### **Core Files**
-- ✅ **README.md** - Comprehensive project documentation with badges, features, quick start
-- ✅ **LICENSE** - MIT License for open source distribution
-- ✅ **CONTRIBUTING.md** - Detailed contributor guidelines and development setup
-- ✅ **.gitignore** - Comprehensive ignore rules for Python, databases, logs, IDE files
-- ✅ **requirements.txt** - All Python dependencies listed
-
-### **Source Code Organization**
-- ✅ **src/** directory - Clean separation of source code
-- ✅ **scripts/** directory - Utility scripts and system files
-- ✅ **docs/** directory - Comprehensive documentation
-- ✅ **grafana/** directory - Visualization configuration
-
-### **Documentation Quality**
-- ✅ **Installation guides** - Multiple deployment options
-- ✅ **Configuration examples** - All database types covered
-- ✅ **Troubleshooting guides** - Common issues and solutions
-- ✅ **Migration guides** - Updating existing systems
-- ✅ **API references** - External data sources documented
-
-### **Production Readiness**
-- ✅ **Docker support** - Containerization ready
-- ✅ **Systemd service** - Linux service configuration
-- ✅ **Multi-database support** - 5 different database options
-- ✅ **Geolocation support** - Grafana geomap integration
-- ✅ **Migration scripts** - Safe database updates
-
-## 🌟 **Key Features for GitHub**
-
-### **Real-time Monitoring**
-- 16 water stations across Thailand
-- 15-minute data collection frequency
-- Automatic gap filling and data validation
-- Multi-database backend support
-
-### **Visualization Ready**
-- Pre-built Grafana dashboards
-- Geomap integration with coordinates
-- Real-time alerts and notifications
-- Historical trend analysis
-
-### **Production Deployment**
-- Docker containerization
-- VictoriaMetrics high-performance backend
-- HTTPS and security configuration
-- Comprehensive logging and monitoring
-
-### **Developer Friendly**
-- Clean, modular code structure
-- Comprehensive documentation
-- Multiple database adapters
-- Easy local development setup
-
-## 📊 **Project Statistics**
-
-### **Code Metrics**
-- **Python Files**: 4 main source files
-- **Documentation**: 10+ comprehensive guides
-- **Database Support**: 5 different backends
-- **Monitoring Stations**: 16 across Thailand
-- **Data Points**: ~300 every 15 minutes
-
-### **Documentation Coverage**
-- **Installation**: Complete setup guides for all platforms
-- **Configuration**: All database types documented
-- **Deployment**: Docker, systemd, and manual options
-- **Troubleshooting**: Common issues and solutions
-- **Migration**: Safe upgrade procedures
-
-### **Features Implemented**
-- ✅ Real-time data collection
-- ✅ Multi-database support
-- ✅ Geolocation integration
-- ✅ Gap filling and data validation
-- ✅ Grafana visualization
-- ✅ Docker deployment
-- ✅ Production monitoring
-- ✅ Migration tools
-
-## 🚀 **Ready for GitHub Publication**
-
-### **Repository Setup**
-1. **Create GitHub repository** - "thailand-water-monitor"
-2. **Upload all files** - Complete project structure
-3. **Configure repository settings**:
-   - Add description: "Real-time water level monitoring for Thailand's RID stations"
-   - Add topics: `water-monitoring`, `thailand`, `grafana`, `timeseries`, `python`
-   - Enable Issues and Discussions
-   - Set up GitHub Pages for documentation
-
-### **Initial Release**
-- **Version**: v1.0.0
-- **Release Notes**: Complete feature set with multi-database support
-- **Assets**: Include sample configuration files
-- **Documentation**: Link to comprehensive guides
-
-### **Community Features**
-- **Issues Template**: Bug reports and feature requests
-- **Pull Request Template**: Contribution guidelines
-- **Discussions**: Community support and questions
-- **Wiki**: Extended documentation and tutorials
-
-## 🎯 **Post-Publication Tasks**
-
-### **Community Building**
-- Create detailed issue templates
-- Set up GitHub Actions for CI/CD
-- Add code quality badges
-- Create project roadmap
-
-### **Documentation Enhancement**
-- Add video tutorials
-- Create API documentation
-- Add performance benchmarks
-- Create deployment examples
-
-### **Feature Development**
-- Mobile app integration
-- Additional database backends
-- Advanced alerting system
-- Predictive analytics
-
-## 📞 **Support Channels**
-
-- **GitHub Issues**: Bug reports and feature requests
-- **GitHub Discussions**: Community support and questions
-- **Documentation**: Comprehensive guides in docs/ directory
-- **Examples**: Working configurations and deployments
-
-## 🏆 **Project Highlights**
-
-### **Technical Excellence**
-- Clean, modular architecture
-- Comprehensive error handling
-- Production-ready deployment
-- Multi-database abstraction
-
-### **Documentation Quality**
-- Step-by-step installation guides
-- Troubleshooting for common issues
-- Migration procedures for updates
-- API and configuration references
-
-### **Community Ready**
-- Open source MIT license
-- Contributor guidelines
-- Development setup instructions
-- Code quality standards
-
----
-
-**The Thailand Water Level Monitor project is now fully prepared for GitHub publication with a professional structure, comprehensive documentation, and production-ready features.** 🌊

GITHUB_TOKEN_SETUP.md

@@ -1,114 +0,0 @@
-# 🔑 GitHub Token Setup Guide
-
-## 🎯 **Why You Need This**
-
-The Gitea Actions workflows use Trivy for security scanning, which needs to download vulnerability databases from GitHub. Without a GitHub token, you'll hit rate limits and the security scans will fail.
-
-## 🚀 **Quick Setup (5 minutes)**
-
-### **Step 1: Create GitHub Personal Access Token**
-
-1. **Go to GitHub**: https://github.com/settings/tokens
-2. **Click "Generate new token"** → "Generate new token (classic)"
-3. **Configure the token**:
-   - **Note**: `B4L Ping River Monitor - Gitea Actions`
-   - **Expiration**: `90 days` (or longer)
-   - **Scopes**: Select `public_repo` (for public repositories)
-4. **Click "Generate token"**
-5. **Copy the token** (you won't see it again!)
-
-### **Step 2: Add Token to Gitea Repository**
-
-1. **Go to your repository**: https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor
-2. **Click "Settings"** (in the repository)
-3. **Click "Secrets"** in the left sidebar
-4. **Click "Add Secret"**
-5. **Configure the secret**:
-   - **Name**: `GITHUB_TOKEN`
-   - **Value**: Paste the token you copied from GitHub
-6. **Click "Add Secret"**
-
-### **Step 3: Verify It's Working**
-
-1. **Trigger a workflow** by pushing a commit or manually running the security workflow
-2. **Check the Actions tab** in your repository
-3. **Look for the message**: `✅ GITHUB_TOKEN is configured`
-
-## 🔒 **Security Best Practices**
-
-### **Token Permissions**
-- **Minimum required**: `public_repo` scope
-- **Never use**: `repo` scope unless you need private repo access
-- **Avoid**: Admin or write permissions
-
-### **Token Management**
-- **Set expiration**: Don't create tokens that never expire
-- **Regular rotation**: Update tokens every 90 days
-- **Monitor usage**: Check GitHub token usage in settings
-
-### **Repository Security**
-- **Only trusted contributors**: Should have access to repository secrets
-- **Audit regularly**: Review who has access to secrets
-- **Use organization secrets**: For multiple repositories
-
-## 🧪 **Testing the Setup**
-
-### **Manual Test**
-```bash
-# Trigger the security workflow manually
-# Go to: Repository → Actions → Security & Dependency Updates → Run workflow
-```
-
-### **Automatic Test**
-```bash
-# Push any change to trigger workflows
-git commit --allow-empty -m "Test GitHub token setup"
-git push
-```
-
-### **Check Workflow Logs**
-1. Go to Actions tab in your repository
-2. Click on the latest "Security & Dependency Updates" run
-3. Click on "Docker Security Scan" job
-4. Look for: `✅ GITHUB_TOKEN is configured`
-
-## ❌ **Troubleshooting**
-
-### **"GITHUB_TOKEN not configured" message**
-- **Problem**: Token not added to repository secrets
-- **Solution**: Follow Step 2 above, ensure exact name `GITHUB_TOKEN`
-
-### **"Bad credentials" error**
-- **Problem**: Token is invalid or expired
-- **Solution**: Generate a new token and update the secret
-
-### **Rate limit errors**
-- **Problem**: Token doesn't have correct permissions
-- **Solution**: Ensure token has `public_repo` scope
-
-### **Trivy still failing**
-- **Problem**: Network issues or GitHub API problems
-- **Solution**: Wait and retry, or check GitHub status page
-
-## 🎉 **Success Indicators**
-
-When everything is working correctly, you'll see:
-
-✅ **In workflow logs**: `✅ GITHUB_TOKEN is configured`  
-✅ **Security scans**: Complete without authentication errors  
-✅ **Trivy reports**: Generated and uploaded as artifacts  
-✅ **No rate limit errors**: In the workflow execution  
-
-## 📚 **Additional Resources**
-
-- [GitHub Personal Access Tokens Documentation](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token)
-- [Gitea Secrets Documentation](https://docs.gitea.io/en-us/usage/actions/#secrets)
-- [Trivy Action Documentation](https://github.com/aquasecurity/trivy-action)
-
----
-
-**Setup Time**: ~5 minutes  
-**Token Validity**: 90 days (recommended)  
-**Security Level**: High (read-only public repo access)  
-
-Your workflows will now run smoothly with proper GitHub API authentication! 🚀

Makefile

@@ -1,146 +1,70 @@
-# Northern Thailand Ping River Monitor - Makefile
+# Ping River Monitor. The service is Rust (rust/, ping-collector); Python is
+# the model training/evaluation stack and the archive backfills (src/).
+# Database tests (both sides) need TEST_DATABASE_URL, e.g.
+#   docker run -d --name ping-pg -e POSTGRES_PASSWORD=ping -e POSTGRES_DB=ping -p 55432:5432 postgres:16
+#   export TEST_DATABASE_URL=postgres://postgres:ping@localhost:55432/ping
 
-.PHONY: help install install-dev test test-coverage lint format clean run run-api docker-build docker-run docs
+PY_SRC := src/ scripts/ tests/ rust/forecast/testdata rust/collector/testdata
+CARGO := cargo
+MANIFEST := --manifest-path rust/Cargo.toml
+
+.PHONY: help install format lint test test-py test-rust build check serve train evaluate export clean
 
-# Default target
 help:
-	@echo "Northern Thailand Ping River Monitor - Available Commands:"
-	@echo ""
-	@echo "Setup:"
-	@echo "  install      Install production dependencies"
-	@echo "  install-dev  Install development dependencies"
-	@echo ""
-	@echo "Development:"
-	@echo "  test         Run all tests"
-	@echo "  test-cov     Run tests with coverage report"
-	@echo "  lint         Run code linting"
-	@echo "  format       Format code with black and isort"
-	@echo "  clean        Clean up temporary files"
-	@echo ""
-	@echo "Running:"
-	@echo "  run          Run the monitor in continuous mode"
-	@echo "  run-api      Run the web API server"
-	@echo "  run-test     Run a single test cycle"
-	@echo ""
-	@echo "Docker:"
-	@echo "  docker-build Build Docker image"
-	@echo "  docker-run   Run with Docker Compose"
-	@echo "  docker-stop  Stop Docker services"
-	@echo ""
-	@echo "Documentation:"
-	@echo "  docs         Generate documentation"
+	@echo "make install    uv sync (Python 3.11) + pre-commit hooks"
+	@echo "make format     black + isort (Python), cargo fmt (Rust)"
+	@echo "make lint       the CI checks without tests (format, flake8, clippy)"
+	@echo "make test       pytest + cargo test"
+	@echo "make build      release build of ping-collector (rust/target/release)"
+	@echo "make check      lint + test + the Rust 1.88 minimum-version check"
+	@echo "make serve      website/API from the local database on 127.0.0.1:8001"
+	@echo "make train      retrain every station's model into models/"
+	@echo "make evaluate   rolling-origin harness on P.1 (see scripts/evaluate_variants.py)"
+	@echo "make export     export trained models + flood_config.json for the Rust forecaster"
 
-# Installation
 install:
-	pip install -r requirements.txt
-
-install-dev:
-	pip install -r requirements-dev.txt
-	pre-commit install
-
-# Testing
-test:
-	python test_integration.py
-	python test_station_management.py
-
-test-cov:
-	pytest --cov=src --cov-report=html --cov-report=term
-
-# Code quality
-lint:
-	flake8 src/ --max-line-length=100
-	mypy src/
+	uv sync --python 3.11
+	uv run pre-commit install
 
 format:
-	black src/ *.py
-	isort src/ *.py
+	uv run black $(PY_SRC)
+	uv run isort src/ scripts/ tests/
+	$(CARGO) fmt --all $(MANIFEST)
+
+lint:
+	uv run black --check $(PY_SRC)
+	uv run isort --check-only src/ scripts/ tests/
+	uv run flake8 src/ scripts/ tests/ --count --select=E9,F63,F7,F82 --show-source --statistics
+	$(CARGO) fmt --all --check $(MANIFEST)
+	$(CARGO) clippy --all-targets --locked $(MANIFEST) -- -D warnings
+
+test: test-py test-rust
+
+test-py:
+	uv run pytest -q
+
+test-rust:
+	$(CARGO) test --locked $(MANIFEST)
+
+build:
+	$(CARGO) build --release --locked $(MANIFEST)
+
+check: lint test
+	CARGO_TARGET_DIR=rust/target/msrv $(CARGO) +1.88 check --all-targets --locked $(MANIFEST)
+
+serve:
+	rust/target/release/ping-collector serve --listen 127.0.0.1:8001 --no-analytics
+
+train:
+	uv run python scripts/train_flood_model.py --stations all
+
+evaluate:
+	uv run python scripts/evaluate_variants.py --stations P.1
+
+export:
+	uv run python -m src.ml.export
+	uv run python -m src.ml.export --config
 
-# Cleanup
 clean:
-	find . -type f -name "*.pyc" -delete
-	find . -type d -name "__pycache__" -delete
-	find . -type f -name "*.log" -delete
-	rm -rf .pytest_cache/
-	rm -rf .mypy_cache/
-	rm -rf htmlcov/
-	rm -rf dist/
-	rm -rf build/
-	rm -rf *.egg-info/
-
-# Running
-run:
-	python run.py
-
-run-api:
-	python run.py --web-api
-
-run-test:
-	python run.py --test
-
-run-status:
-	python run.py --status
-
-# Docker
-docker-build:
-	docker build -t ping-river-monitor .
-
-docker-run:
-	docker-compose -f docker-compose.victoriametrics.yml up -d
-
-docker-stop:
-	docker-compose -f docker-compose.victoriametrics.yml down
-
-docker-logs:
-	docker-compose -f docker-compose.victoriametrics.yml logs -f
-
-# Documentation
-docs:
-	cd docs && make html
-
-# Database management
-db-migrate:
-	python scripts/migrate_geolocation.py
-
-# Monitoring
-health-check:
-	curl -f http://localhost:8000/health || exit 1
-
-metrics:
-	curl -s http://localhost:8000/metrics | jq .
-
-# Development helpers
-dev-setup: install-dev
-	cp .env.example .env
-	@echo "Development environment set up!"
-	@echo "Edit .env file with your configuration"
-
-# Production deployment
-deploy-check:
-	python run.py --test
-	@echo "Deployment check passed!"
-
-# Git helpers
-git-setup:
-	git remote add origin https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor.git
-	@echo "Git remote configured!"
-
-# Quick start
-quick-start: install dev-setup run-test
-	@echo "Quick start completed!"
-	@echo "Run 'make run-api' to start the web interface"
-
-# Gitea Actions
-validate-workflows:
-	@echo "Validating Gitea Actions workflows..."
-	@for file in .gitea/workflows/*.yml; do \
-		echo "Checking $$file..."; \
-		python -c "import yaml; yaml.safe_load(open('$$file', encoding='utf-8'))" || exit 1; \
-	done
-	@echo "✅ All workflows are valid"
-
-workflow-test:
-	@echo "Testing workflow components locally..."
-	make test
-	make lint
-	make format
-	@echo "✅ Workflow test completed"
+	find . -type d -name "__pycache__" -prune -exec rm -rf {} +
+	rm -rf .pytest_cache/ htmlcov/ dist/ build/ *.egg-info/

README.md

@@ -1,491 +1,198 @@
-# Northern Thailand Ping River Monitor 🏔️
+# Northern Thailand Ping River Monitor
 
-A comprehensive real-time water level monitoring system for the Ping River Basin in Northern Thailand, covering Royal Irrigation Department (RID) stations from Chiang Dao to Nakhon Sawan with advanced data collection, storage, and visualization capabilities.
+Live water levels, discharge, rainfall and machine-learning flood forecasts for the
+Ping River basin around Chiang Mai. Collects hourly gauge data from public sources,
+keeps the full history in PostgreSQL, serves a bilingual dashboard and an open REST
+API, forecasts the flood risk per gauge 6/12/24 hours ahead, and pushes alerts when
+a gauge crosses its warning or danger level.
 
-[![CI/CD](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/actions/workflows/ci.yml/badge.svg)](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/actions) [![Security](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/actions/workflows/security.yml/badge.svg)](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/actions) [![Documentation](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/actions/workflows/docs.yml/badge.svg)](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/actions) [![Python](https://img.shields.io/badge/Python-3.9+-blue.svg)](https://python.org) [![FastAPI](https://img.shields.io/badge/FastAPI-0.104+-green.svg)](https://fastapi.tiangolo.com) [![Docker](https://img.shields.io/badge/Docker-Ready-blue.svg)](https://docker.com) [![License](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE) [![Version](https://img.shields.io/badge/Version-v3.1.3-blue.svg)](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/releases)
+**Live: [water.buildfor.life](https://water.buildfor.life/)** · API reference at
+[/docs](https://water.buildfor.life/docs) · built by [buildfor.life](https://buildfor.life)
+after the [October 2024 flood](https://buildfor.life/blog/chiang-mai-flood-2024/) —
+background in [Teaching a Model to See the Ping River Rise 13 Hours Early](https://buildfor.life/blog/ping-river-monitor/).
 
-## 🌟 Features
+[![CI](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/actions/workflows/ci.yml/badge.svg)](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/actions)
+[![Rust](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/actions/workflows/rust.yml/badge.svg)](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/actions)
+[![Security](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/actions/workflows/security.yml/badge.svg)](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/actions)
+[![Docs](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/actions/workflows/docs.yml/badge.svg)](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/actions)
+[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE)
 
-### 📊 **Real-time Data Collection**
-- **16 Monitoring Stations** across Thailand
-- **15-minute Collection Frequency** with intelligent scheduling
-- **Automatic Gap Filling** for missing historical data
-- **Data Validation** and error recovery mechanisms
-- **Rate Limiting** to prevent API abuse
+## What it does
 
-### 🌐 **Web API Interface (NEW!)**
-- **FastAPI-powered REST API** with interactive documentation
-- **Station Management** - Add, update, and remove monitoring stations
-- **Real-time health monitoring** and system status
-- **Manual data collection triggers** via web interface
-- **Comprehensive metrics** and performance monitoring
-- **CORS support** for web applications
+- **Collects** hourly water level and discharge from 16 Royal Irrigation Department
+  (RID) telemetry gauges, Chiang Dao to the southern basin, since 2018-08; hourly
+  rainfall and water level from 400+ ThaiWater/HII stations; Open-Meteo catchment
+  rainfall (archive + 48 h forecast); daily state of the large RID dams, Mae Ngat
+  above all. Every source and its quirks: [docs/DATA_SOURCES.md](docs/DATA_SOURCES.md).
+- **Fills gaps.** The raw RID grid had readings for ~56 % of hours; a full-history
+  re-fetch plus HII cross-fill brought it to ~93 %. RID's report often arrives
+  hours late in the evening, so late readings are always stored and holes are
+  re-fetched. `GET /api/stats` reports the current coverage.
+- **Forecasts.** Per gauge and horizon, a gradient-boosted model predicts the rise
+  within 6/12/24 h and the probability of crossing the station's warning and danger
+  levels. Trained on the monitor's own history plus catchment rain; evaluated
+  rolling-origin, event by event. On the October 2024 record flood, trained only on
+  data through August 2024, the first alert came **13 hours before** P.1 crossed
+  3.70 m. Everything about the model, including what did not work:
+  [docs/FLOOD_FORECASTING.md](docs/FLOOD_FORECASTING.md).
+- **Shows it.** A Leaflet map with the river drawn as OSM geometry and styled by live
+  discharge, rain gauges, the Chiang Mai inundation zones, per-station history, the
+  forecast card, a replay of the 2024 flood, English/Thai, light/dark.
+- **Notifies.** Public push alerts over a self-hosted [ntfy](https://ntfy.sh): one
+  message when a gauge crosses its warning or danger level, one all-clear on the
+  way down, an opt-in early-warning topic from the model, nothing in between.
+  Subscribe from the free app, no account. A Matrix room can get the same
+  messages for a team. [docs/NOTIFICATIONS.md](docs/NOTIFICATIONS.md).
 
-### 🗄️ **Multi-Database Support**
-- **VictoriaMetrics** (Recommended) - High-performance time-series
-- **InfluxDB** - Purpose-built time-series database
-- **PostgreSQL + TimescaleDB** - Relational with time-series optimization
-- **MySQL** - Traditional relational database
-- **SQLite** - Local development and testing
+## How it runs
 
-### 🗺️ **Geolocation Support**
-- **Grafana Geomap** integration ready
-- **GPS coordinates** and geohash support
-- **Interactive mapping** of water stations
+```text
+ RID gauges · ThaiWater/HII · RID dams · Open-Meteo
+                    │ hourly
+                    ▼
+ ┌──────────────────────────────────────┐        ┌───────────────────────────┐
+ │ ping-collector  (Rust, one process)  │        │ monthly retrain (Python)  │
+ │  collect → forecast → notify         │◄───────│  scikit-learn, exports    │
+ │  website + API on :8000              │ models │  models/flood_*.json.gz   │
+ └──────┬──────────────┬────────────────┘        └─────────────┬─────────────┘
+        │              │                                       │
+        ▼              ▼                                       │
+  ntfy topics,    PostgreSQL ◄──────────── history ────────────┘
+  Matrix room     (gauges, rain, dams, forecasts, alert state)
+```
 
-### 📈 **Visualization & Monitoring**
-- **Pre-built Grafana dashboards**
-- **Real-time alerts** and notifications
-- **Historical trend analysis**
-- **Built-in metrics collection** (counters, gauges, histograms)
-- **Health checks** for database, API, and system resources
+- **The service** is one Rust binary, `ping-collector` ([rust/README.md](rust/README.md)),
+  run by systemd. It polls RID at :05 past each hour and retries every 10 minutes
+  while the report is behind, fetches HII, the dams and the rain once an hour,
+  forecasts after every cycle that brought new readings, sends the notifications,
+  and serves the dashboard and API, in under 100 MB of memory with every model
+  loaded.
+- **The models** are trained in Python (`src/ml`, scikit-learn
+  HistGradientBoosting) by a monthly timer and exported for the Rust forecaster,
+  which reproduces Python's features bit for bit and reloads new models without
+  a restart.
+- Until October 2026 all of this was a Python/FastAPI app; how and why it moved:
+  [docs/RUST_REWRITE.md](docs/RUST_REWRITE.md).
 
-### 🚀 **Production Ready**
-- **Docker containerization** with multi-service support
-- **Systemd service** configuration
-- **HTTPS support** with SSL certificates
-- **Comprehensive logging** with rotation and colored output
-- **Type safety** with Pydantic models and type hints
-- **Custom exception handling** for better error management
+## API
 
-## 🚀 Quick Start
+Read-only, no key, JSON. Base URL `https://water.buildfor.life`; timestamps are
+Asia/Bangkok wall-clock without an offset suffix.
 
-### Prerequisites
-- Python 3.9 or higher
-- Internet connection for data fetching
-- Database server (optional - SQLite works out of the box)
+| Endpoint | Returns |
+| --- | --- |
+| `GET /stations` | The 16 RID gauges: code, Thai/English names, coordinates |
+| `GET /measurements/latest?limit=N` | Newest reading per station |
+| `GET /measurements/history/{code}?hours=N` | Hourly history; or `?start=YYYY-MM-DD&end=YYYY-MM-DD`; `limit` ≤ 100000 |
+| `GET /forecast` | Current flood-risk forecast, every station × horizon, with thresholds and P.1 inundation-stage probabilities |
+| `GET /api/forecast/history/{code}?hours=N&horizon=24` | Forecasts as issued, for auditing lead time after the fact |
+| `GET /api/forecast/skill?station_code=P.1` | Issued forecasts vs what happened, per deployed model version |
+| `GET /api/hii/rainfall/latest`, `/api/hii/waterlevel/latest` | Latest ThaiWater/HII gauge readings |
+| `GET /api/hii/rainfall/catchment?days=N` | HII gauge catchment-mean rain next to the Open-Meteo series the model uses |
+| `GET /api/notifications` | ntfy server and topic names for the subscribe panel |
+| `GET /api/stats` | Row counts per source, date range, coverage |
+| `GET /health` | Database and data freshness (RID, ThaiWater as backup), the RID API, memory |
+| `GET /scraping/status` | Collection runs, the last forecast run, service version |
 
-### Installation
+Interactive reference with schemas: [water.buildfor.life/docs](https://water.buildfor.life/docs)
+(also `/redoc`, `/openapi.json`). Responses are cached briefly server-side; poll no
+faster than once a minute — the data changes hourly. Station editing
+(`POST/PUT/DELETE /stations`), `/config` and `POST /scrape/trigger` need the
+server's `ADMIN_API_KEY` in an `X-API-Key` header.
+
+## Run it locally
+
+Needs Rust >= 1.88 ([rustup](https://rustup.rs); on Linux also `pkg-config` and
+`libssl-dev`) and PostgreSQL. Python 3.11 with [uv](https://docs.astral.sh/uv/)
+only for training models.
 
 ```bash
-# Clone the repository
 git clone https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor.git
 cd Northern-Thailand-Ping-River-Monitor
-
-# Quick setup with Make
-make dev-setup
-
-# Or manual setup:
-python -m venv venv
-source venv/bin/activate  # Windows: venv\Scripts\activate
-pip install -r requirements.txt
+docker run -d --name ping-pg -e POSTGRES_PASSWORD=ping -e POSTGRES_DB=ping -p 55432:5432 postgres:16
 cp .env.example .env
+#   POSTGRES_CONNECTION_STRING=postgresql://postgres:ping@localhost:55432/ping
+#   WEB_LISTEN=127.0.0.1:8000
+cargo build --release --manifest-path rust/Cargo.toml
+rust/target/release/ping-collector check-db         # create the tables
+rust/target/release/ping-collector fill-gaps 30     # the last 30 days from RID
+rust/target/release/ping-collector run              # collect, forecast, serve http://127.0.0.1:8000
 ```
 
-### Basic Usage
+Notifications stay off until `NTFY_SERVER` or `MATRIX_*` is set. Without trained
+models the forecast is the persistence heuristic, marked `source: "heuristic"`.
+Training needs years of history (the server's database, or `FLOOD_ML_DB_URL`) and
+the Open-Meteo archive:
 
 ```bash
-# Test run with SQLite (default)
-make run-test
-# or: python run.py --test
-
-# Run continuous monitoring
-make run
-# or: python run.py
-
-# Start web API server (NEW!)
-make run-api
-# or: python run.py --web-api
-
-# Run all tests
-make test
-
-# Demo different databases
-python src/demo_databases.py
+uv sync --python 3.11
+uv run python scripts/train_flood_model.py --stations all    # writes models/flood_*
+uv run python scripts/evaluate_variants.py --stations P.1     # the rolling-origin harness
 ```
 
-### 🌐 Web API Interface (NEW!)
+`make help` lists the rest (`make check` runs every CI gate locally).
 
-The system now includes a comprehensive FastAPI web interface:
+## Deployment
+
+Production is one systemd unit on a small VPS behind Cloudflare; the app directory
+`/opt/thailand-water-monitor` is a git checkout. To update:
 
 ```bash
-# Start the web API
-python run.py --web-api
-
-# Access the API at:
-# - Dashboard: http://localhost:8000
-# - Interactive docs: http://localhost:8000/docs
-# - Health check: http://localhost:8000/health
-# - Latest data: http://localhost:8000/measurements/latest
+cd /opt/thailand-water-monitor && git pull
+cargo build --release --locked --manifest-path rust/Cargo.toml
+sudo bash scripts/install.sh
 ```
 
-**Key API Endpoints:**
-- `GET /` - Web dashboard
-- `GET /health` - System health status
-- `GET /metrics` - Application metrics
-- `GET /stations` - List all monitoring stations
-- `POST /stations` - Add new monitoring station
-- `PUT /stations/{id}` - Update station information
-- `DELETE /stations/{id}` - Remove monitoring station
-- `GET /measurements/latest` - Latest measurements
-- `GET /measurements/station/{code}` - Station-specific data
-- `POST /scrape/trigger` - Trigger manual data collection
-
-## 📊 Station Information
-
-The system monitors **16 water stations** along the Ping River Basin in Northern Thailand:
-
-| Station | Thai Name | English Name | Location |
-|---------|-----------|--------------|----------|
-| P.1 | สะพานนวรัฐ | Nawarat Bridge | Nakhon Sawan |
-| P.5 | สะพานท่านาง | Tha Nang Bridge | - |
-| P.20 | บ้านเชียงดาว | Ban Chiang Dao | Chiang Mai |
-| P.21 | บ้านริมใต้ | Ban Rim Tai | - |
-| P.4A | บ้านแม่แตง | Ban Mae Taeng | Chiang Mai |
-| P.67 | บ้านแม่แต | Ban Tae | - |
-| P.75 | บ้านช่อแล | Ban Chai Lat | - |
-| P.76 | บ้านแม่อีไฮ | Banb Mae I Hai | - |
-| P.77 | บ้านสบแม่สะป๊วด | Baan Sop Mae Sapuord | - |
-| P.81 | บ้านโป่ง | Ban Pong | - |
-| P.82 | บ้านสบวิน | Ban Sob win | - |
-| P.84 | บ้านพันตน | Ban Panton | - |
-| P.85 | บ้านหล่ายแก้ว | Baan Lai Kaew | - |
-| P.87 | บ้านป่าซาง | Ban Pa Sang | - |
-| P.92 | บ้านเมืองกึ๊ด | Ban Muang Aut | - |
-| P.103 | สะพานวงแหวนรอบ 3 | Ring Bridge 3 | Bangkok |
-
-### Data Metrics
-- **Water Level**: Measured in meters (m)
-- **Discharge**: Flow rate in cubic meters per second (cms)
-- **Discharge Percentage**: Relative to station capacity
-- **Timestamp**: Thai time (UTC+7) with Buddhist calendar support
-
-## 🗄️ Database Configuration
-
-### VictoriaMetrics (Recommended)
-
-**High-performance time-series database with excellent compression and query speed.**
-
-```bash
-# Environment variables
-export DB_TYPE=victoriametrics
-export VM_HOST=localhost
-export VM_PORT=8428
-
-# Quick start with Docker
-docker run -d \
-  --name victoriametrics \
-  -p 8428:8428 \
-  -v victoria-metrics-data:/victoria-metrics-data \
-  victoriametrics/victoria-metrics:latest \
-  --storageDataPath=/victoria-metrics-data \
-  --retentionPeriod=2y \
-  --httpListenAddr=:8428
-```
-
-### Complete Stack with Grafana
-
-```bash
-# Start the complete monitoring stack
-docker-compose -f docker-compose.victoriametrics.yml up -d
-
-# Access Grafana at http://localhost:3000
-# Username: admin, Password: admin_password
-```
-
-### Other Database Options
-
-<details>
-<summary>InfluxDB Configuration</summary>
-
-```bash
-export DB_TYPE=influxdb
-export INFLUX_HOST=localhost
-export INFLUX_PORT=8086
-export INFLUX_DATABASE=water_monitoring
-export INFLUX_USERNAME=water_user
-export INFLUX_PASSWORD=your_password
-```
-</details>
-
-<details>
-<summary>PostgreSQL Configuration</summary>
-
-```bash
-export DB_TYPE=postgresql
-export POSTGRES_CONNECTION_STRING=postgresql://user:password@localhost:5432/water_monitoring
-```
-</details>
-
-<details>
-<summary>MySQL Configuration</summary>
-
-```bash
-export DB_TYPE=mysql
-export MYSQL_CONNECTION_STRING=mysql://user:password@localhost:3306/water_monitoring
-```
-</details>
-
-## 📈 Grafana Dashboards
-
-### Pre-built Dashboard Features
-- **Real-time water levels** across all stations
-- **Historical trends** and patterns
-- **Discharge monitoring** with percentage indicators
-- **Station status** and health monitoring
-- **Geomap visualization** of station locations
-- **Alert thresholds** for critical water levels
-
-### Sample Queries
-
-**VictoriaMetrics/Prometheus:**
-```promql
-# Current water levels
-water_level
-
-# High discharge alerts
-water_discharge_percent > 80
-
-# Station-specific data
-water_level{station_code="P.1"}
-```
-
-**SQL Databases:**
-```sql
--- Latest readings from all stations
-SELECT s.station_code, s.english_name, m.water_level, m.discharge
-FROM stations s 
-JOIN water_measurements m ON s.id = m.station_id
-WHERE m.timestamp = (SELECT MAX(timestamp) FROM water_measurements WHERE station_id = s.id);
-```
-
-## 🚀 Production Deployment
-
-### Docker Deployment
-
-```bash
-# Build the image
-docker build -t thailand-water-monitor .
-
-# Run with environment variables
-docker run -d \
-  --name water-monitor \
-  -e DB_TYPE=victoriametrics \
-  -e VM_HOST=victoriametrics \
-  thailand-water-monitor
-```
-
-### Systemd Service (Linux)
-
-```bash
-# Copy service file
-sudo cp scripts/water-monitor.service /etc/systemd/system/
-
-# Enable and start
-sudo systemctl enable water-monitor.service
-sudo systemctl start water-monitor.service
-```
-
-### Migration for Existing Systems
-
-If you have an existing installation, use the migration script to add geolocation support:
-
-```bash
-# Stop the service
-sudo systemctl stop water-monitor
-
-# Run migration
-python scripts/migrate_geolocation.py
-
-# Restart the service
-sudo systemctl start water-monitor
-```
-
-## 🔧 Command Line Tools
-
-### Main Application
-```bash
-python src/water_scraper_v3.py                    # Run continuous monitoring
-python src/water_scraper_v3.py --test             # Single test cycle
-python src/water_scraper_v3.py --help             # Show help
-```
-
-### Data Management
-```bash
-python src/water_scraper_v3.py --check-gaps 7     # Check for missing data (7 days)
-python src/water_scraper_v3.py --fill-gaps 7      # Fill missing data gaps
-python src/water_scraper_v3.py --update-data 2    # Update existing data (2 days)
-```
-
-### Database Testing
-```bash
-python src/demo_databases.py                      # SQLite demo
-python src/demo_databases.py victoriametrics      # VictoriaMetrics demo
-python src/demo_databases.py all                  # Test all databases
-```
-
-## 📚 Documentation
-
-### Core Documentation
-- **[Installation Guide](docs/DATABASE_DEPLOYMENT_GUIDE.md)** - Complete setup instructions
-- **[Scheduler Guide](docs/ENHANCED_SCHEDULER_GUIDE.md)** - 15-minute scheduling system
-- **[Geolocation Guide](docs/GEOLOCATION_GUIDE.md)** - Grafana geomap integration
-- **[Gap Filling Guide](docs/GAP_FILLING_GUIDE.md)** - Data integrity management
-
-### Deployment Guides
-- **[VictoriaMetrics Setup](docs/VICTORIAMETRICS_SETUP.md)** - High-performance deployment
-- **[HTTPS Configuration](docs/HTTPS_CONFIGURATION.md)** - Secure deployment
-- **[Debian Troubleshooting](docs/DEBIAN_TROUBLESHOOTING.md)** - Linux deployment issues
-
-### References
-- **[Notable Documents](docs/references/NOTABLE_DOCUMENTS.md)** - Official Thai government resources
-- **[Migration Guide](docs/MIGRATION_QUICKSTART.md)** - Updating existing systems
-
-## 🔍 Troubleshooting
-
-### Common Issues
-
-**Database Connection Errors:**
-```bash
-# Check database status
-python src/demo_databases.py
-
-# Test specific database
-python src/demo_databases.py victoriametrics
-```
-
-**Missing Data:**
-```bash
-# Check for gaps
-python src/water_scraper_v3.py --check-gaps 7
-
-# Fill missing data
-python src/water_scraper_v3.py --fill-gaps 7
-```
-
-**Service Issues:**
-```bash
-# Check service status
-sudo systemctl status water-monitor
-
-# View logs
-sudo journalctl -u water-monitor -f
-```
-
-### Health Checks
-
-```bash
-# VictoriaMetrics health
-curl http://localhost:8428/health
-
-# Check latest data
-curl "http://localhost:8428/api/v1/query?query=water_level"
-
-# Application logs
-tail -f water_monitor.log
-```
-
-## 🌐 API Integration
-
-### VictoriaMetrics API Examples
-
-```bash
-# Query current water levels
-curl "http://localhost:8428/api/v1/query?query=water_level"
-
-# Query discharge rates for last hour
-curl "http://localhost:8428/api/v1/query_range?query=water_discharge&start=$(date -d '1 hour ago' +%s)&end=$(date +%s)&step=300"
-
-# Query specific station
-curl "http://localhost:8428/api/v1/query?query=water_level{station_code=\"P.1\"}"
-
-# High discharge alerts
-curl "http://localhost:8428/api/v1/query?query=water_discharge_percent>80"
-```
-
-## 📊 Performance
-
-### System Requirements
-- **CPU**: 1-2 cores (minimal load)
-- **RAM**: 512MB - 2GB (depending on database)
-- **Storage**: 1GB+ (for historical data)
-- **Network**: Stable internet connection
-
-### Performance Metrics
-- **Data Collection**: ~300 data points every 15 minutes
-- **Database Write Speed**: 1000+ points/second (VictoriaMetrics)
-- **Query Response**: <100ms for recent data
-- **Storage Efficiency**: 70x compression vs. raw data
-
-## 🤝 Contributing
-
-Contributions are welcome! Please:
-
-1. Fork the repository
-2. Create a feature branch
-3. Make your changes
-4. Add tests if applicable
-5. Submit a pull request
-
-### Development Setup
-
-```bash
-# Clone your fork
-git clone https://github.com/your-username/thailand-water-monitor.git
-cd thailand-water-monitor
-
-# Install development dependencies
-pip install -r requirements.txt
-pip install pytest black flake8
-
-# Run tests
-pytest
-
-# Format code
-black src/
-```
-
-## 📄 License
-
-This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
-
-## 🙏 Acknowledgments
-
-- **Royal Irrigation Department (RID)** of Thailand for providing the data API
-- **VictoriaMetrics** team for the excellent time-series database
-- **Grafana** team for the visualization platform
-- **Python community** for the amazing libraries and tools
-
-## 📞 Support
-
-- **Issues**: [GitHub Issues](https://github.com/your-username/thailand-water-monitor/issues)
-- **Discussions**: [GitHub Discussions](https://github.com/your-username/thailand-water-monitor/discussions)
-- **Documentation**: [Project Wiki](https://github.com/your-username/thailand-water-monitor/wiki)
-
----
-
-## 📁 Project Structure
+`scripts/install.sh` (idempotent, also for a fresh host) installs the binary and
+`ping-collector.service`, syncs the Python environment for the monthly
+`water-monitor-retrain.timer`, runs a dry run of every source and a forecast check
+against the Python reference, restarts the service and waits for the website. The
+retrain trains into `models/.staging`, refuses to promote anything that is not a
+rain-enabled (`hgb-v3+`) set covering the expected stations, and renames the
+bundles into place; the service picks them up on its next forecast. Details and
+the operations runbook: [docs/FLOOD_FORECASTING.md](docs/FLOOD_FORECASTING.md)
+sections 6–8. Configuration: [.env.example](.env.example).
+
+## Repository layout
 
 ```
-Northern-Thailand-Ping-River-Monitor/
-├── src/                    # Main application code
-├── tests/                  # Test suite
-├── docs/                   # Documentation
-├── grafana/               # Grafana dashboards
-├── scripts/               # Utility scripts
-├── docker-compose.yml     # Docker deployment
-├── Makefile              # Development tasks
-└── requirements.txt      # Dependencies
+rust/collector/      ping-collector: collection, forecast run, ntfy/Matrix, website + API (src/web)
+rust/forecast/       forecast inference from the exported models; parity tests against Python
+src/static/          the dashboard (single file, EN/TH) and its map data
+src/data/            the bundled station list
+src/ml/              features, training, evaluation harness, export; predict.py is the reference
+src/*.py             HII and RID-dam clients/stores for the archive backfills
+scripts/             install.sh, retrain.sh, train/evaluate/backfill tools, systemd units
+tests/               pytest (synthetic data; database tests need TEST_DATABASE_URL)
+docs/                model write-up, data sources, notifications, Matrix, rewrite history
+models/              trained bundles (gitignored) and evaluation results
 ```
 
-See [docs/PROJECT_STRUCTURE.md](docs/PROJECT_STRUCTURE.md) for detailed architecture information.
+## Documentation
 
-## 🔄 CI/CD & Automation
+- [docs/FLOOD_FORECASTING.md](docs/FLOOD_FORECASTING.md) — the model: data, features, evaluation, measured performance, negatives, deployment, retraining
+- [docs/DATA_SOURCES.md](docs/DATA_SOURCES.md) — every ingested and candidate source, endpoints, quirks
+- [docs/NOTIFICATIONS.md](docs/NOTIFICATIONS.md) — public push alerts: topics, semantics, ntfy deployment
+- [docs/MATRIX_QUICK_START.md](docs/MATRIX_QUICK_START.md) — the same alerts in a Matrix room
+- [rust/README.md](rust/README.md) — the service: commands, configuration, develop, deploy
+- [docs/RUST_REWRITE.md](docs/RUST_REWRITE.md) — why and how it moved from Python to Rust
+- [docs/references/NOTABLE_DOCUMENTS.md](docs/references/NOTABLE_DOCUMENTS.md) — official Thai government resources
+- Public overview: [buildfor.life/docs/tooling/ping-river-monitor](https://buildfor.life/docs/tooling/ping-river-monitor/)
 
-The project includes comprehensive Gitea Actions workflows:
+## Contributing
 
-- **🧪 CI/CD Pipeline** - Automated testing, building, and deployment
-- **🔒 Security Scanning** - Daily vulnerability and dependency checks  
-- **📚 Documentation** - Automated API docs and validation
-- **🚀 Release Management** - Automated releases with multi-arch Docker builds
+`make format` before committing, `make check` must stay green; see
+[CONTRIBUTING.md](CONTRIBUTING.md). Issues and merge requests on
+[git.b4l.co.th](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor).
 
-See [docs/GITEA_WORKFLOWS.md](docs/GITEA_WORKFLOWS.md) for detailed workflow documentation.
+## Data sources and thanks
 
-## 🔗 Repository
+Royal Irrigation Department (RID) gauge telemetry and reservoir reports;
+Hydro-Informatics Institute (HII) / ThaiWater open API; Open-Meteo; OpenStreetMap
+contributors for the river geometry; Chiang Mai Municipality for the inundation map
+the P.1 stages are keyed to. All instruments are theirs; we aggregate, store, fill
+gaps and forecast.
 
-- **Main Repository**: https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor
-- **Issues**: https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/issues
-- **Actions**: https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/actions
-- **Documentation**: [docs/](docs/)
+## License
 
-**Made with ❤️ for water resource monitoring in Northern Thailand's Ping River Basin**
+MIT — see [LICENSE](LICENSE).

docker-compose.victoriametrics.yml

@@ -1,106 +0,0 @@
-version: '3.8'
-
-services:
-  victoriametrics:
-    image: victoriametrics/victoria-metrics:latest
-    container_name: ping_river_victoriametrics
-    ports:
-      - "8428:8428"
-    volumes:
-      - vm_data:/victoria-metrics-data
-      - ./vm-config:/etc/victoriametrics
-    command:
-      - '--storageDataPath=/victoria-metrics-data'
-      - '--retentionPeriod=2y'
-      - '--httpListenAddr=:8428'
-      - '--maxConcurrentInserts=32'
-      - '--search.maxQueryDuration=60s'
-      - '--search.maxConcurrentRequests=16'
-      - '--dedup.minScrapeInterval=30s'
-      - '--memory.allowedPercent=80'
-      - '--loggerLevel=INFO'
-      - '--loggerFormat=json'
-      - '--search.maxSeries=1000000'
-      - '--search.maxPointsPerTimeseries=100000'
-    restart: unless-stopped
-    environment:
-      - TZ=Asia/Bangkok
-    healthcheck:
-      test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost:8428/health"]
-      interval: 30s
-      timeout: 10s
-      retries: 3
-      start_period: 40s
-    labels:
-      - "com.victoriametrics.service=ping-river-monitoring"
-
-  grafana:
-    image: grafana/grafana:latest
-    container_name: ping_river_grafana
-    ports:
-      - "3000:3000"
-    volumes:
-      - grafana_data:/var/lib/grafana
-      - ./grafana/provisioning:/etc/grafana/provisioning
-      - ./grafana/dashboards:/var/lib/grafana/dashboards
-    environment:
-      - GF_SECURITY_ADMIN_PASSWORD=admin_password
-      - GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-worldmap-panel
-      - GF_USERS_ALLOW_SIGN_UP=false
-      - GF_SECURITY_ALLOW_EMBEDDING=true
-      - GF_AUTH_ANONYMOUS_ENABLED=false
-      - TZ=Asia/Bangkok
-    restart: unless-stopped
-    depends_on:
-      victoriametrics:
-        condition: service_healthy
-    labels:
-      - "com.victoriametrics.service=ping-river-monitoring"
-
-  ping-river-monitor:
-    build: .
-    container_name: ping_river_monitor
-    environment:
-      - DB_TYPE=victoriametrics
-      - VM_HOST=victoriametrics
-      - VM_PORT=8428
-      - TZ=Asia/Bangkok
-    restart: unless-stopped
-    depends_on:
-      victoriametrics:
-        condition: service_healthy
-    volumes:
-      - ./logs:/app/logs
-    labels:
-      - "com.victoriametrics.service=ping-river-monitoring"
-
-  ping-river-api:
-    build: .
-    container_name: ping_river_api
-    ports:
-      - "8000:8000"
-    environment:
-      - DB_TYPE=victoriametrics
-      - VM_HOST=victoriametrics
-      - VM_PORT=8428
-      - TZ=Asia/Bangkok
-    restart: unless-stopped
-    depends_on:
-      victoriametrics:
-        condition: service_healthy
-    volumes:
-      - ./logs:/app/logs
-    command: ["python", "-m", "src.main", "--web-api"]
-    labels:
-      - "com.victoriametrics.service=ping-river-monitoring"
-
-volumes:
-  vm_data:
-    driver: local
-  grafana_data:
-    driver: local
-
-networks:
-  default:
-    name: ping-river-monitoring
-    driver: bridge

docs/DATABASE_DEPLOYMENT_GUIDE.md

@@ -1,447 +0,0 @@
-# Database Deployment Guide for Thailand Water Monitor
-
-This guide covers deployment options for storing water monitoring data in production environments.
-
-## 🏆 Recommendation Summary
-
-| Database | Best For | Performance | Complexity | Cost |
-|----------|----------|-------------|------------|------|
-| **InfluxDB** | Time-series data, dashboards | ⭐⭐⭐⭐⭐ | ⭐⭐⭐ | ⭐⭐⭐ |
-| **VictoriaMetrics** | High-performance metrics | ⭐⭐⭐⭐⭐ | ⭐⭐⭐⭐ | ⭐⭐⭐⭐ |
-| **PostgreSQL** | Complex queries, reliability | ⭐⭐⭐⭐ | ⭐⭐ | ⭐⭐⭐⭐ |
-| **MySQL** | Familiar, existing infrastructure | ⭐⭐⭐ | ⭐⭐ | ⭐⭐⭐⭐ |
-
-## 1. InfluxDB Deployment (Recommended for Time-Series)
-
-### Why InfluxDB?
-- **Purpose-built** for time-series data
-- **Excellent compression** (10:1 typical ratio)
-- **Built-in retention policies** and downsampling
-- **Great Grafana integration** for dashboards
-- **High write throughput** (100k+ points/second)
-
-### Docker Deployment
-
-```yaml
-# docker-compose.yml
-version: '3.8'
-services:
-  influxdb:
-    image: influxdb:1.8
-    container_name: water_influxdb
-    ports:
-      - "8086:8086"
-    volumes:
-      - influxdb_data:/var/lib/influxdb
-      - ./influxdb.conf:/etc/influxdb/influxdb.conf:ro
-    environment:
-      - INFLUXDB_DB=water_monitoring
-      - INFLUXDB_ADMIN_USER=admin
-      - INFLUXDB_ADMIN_PASSWORD=your_secure_password
-      - INFLUXDB_USER=water_user
-      - INFLUXDB_USER_PASSWORD=water_password
-    restart: unless-stopped
-
-  grafana:
-    image: grafana/grafana:latest
-    container_name: water_grafana
-    ports:
-      - "3000:3000"
-    volumes:
-      - grafana_data:/var/lib/grafana
-    environment:
-      - GF_SECURITY_ADMIN_PASSWORD=admin_password
-    restart: unless-stopped
-
-volumes:
-  influxdb_data:
-  grafana_data:
-```
-
-### Environment Variables
-```bash
-# .env file
-DB_TYPE=influxdb
-INFLUX_HOST=localhost
-INFLUX_PORT=8086
-INFLUX_DATABASE=water_monitoring
-INFLUX_USERNAME=water_user
-INFLUX_PASSWORD=water_password
-```
-
-### InfluxDB Configuration
-```toml
-# influxdb.conf
-[meta]
-  dir = "/var/lib/influxdb/meta"
-
-[data]
-  dir = "/var/lib/influxdb/data"
-  wal-dir = "/var/lib/influxdb/wal"
-  
-  # Optimize for time-series data
-  cache-max-memory-size = "1g"
-  cache-snapshot-memory-size = "25m"
-  cache-snapshot-write-cold-duration = "10m"
-  
-  # Retention and compression
-  compact-full-write-cold-duration = "4h"
-  max-series-per-database = 1000000
-  max-values-per-tag = 100000
-
-[coordinator]
-  write-timeout = "10s"
-  max-concurrent-queries = 0
-  query-timeout = "0s"
-
-[retention]
-  enabled = true
-  check-interval = "30m"
-
-[http]
-  enabled = true
-  bind-address = ":8086"
-  auth-enabled = true
-  max-body-size = "25000000"
-  max-concurrent-requests = 0
-  max-enqueued-requests = 0
-```
-
-### Production Setup Commands
-```bash
-# Start services
-docker-compose up -d
-
-# Create retention policies
-docker exec -it water_influxdb influx -username admin -password your_secure_password -execute "
-CREATE RETENTION POLICY \"raw_data\" ON \"water_monitoring\" DURATION 90d REPLICATION 1 DEFAULT;
-CREATE RETENTION POLICY \"downsampled\" ON \"water_monitoring\" DURATION 730d REPLICATION 1;
-"
-
-# Create continuous queries for downsampling
-docker exec -it water_influxdb influx -username admin -password your_secure_password -execute "
-CREATE CONTINUOUS QUERY \"downsample_hourly\" ON \"water_monitoring\" 
-BEGIN 
-  SELECT mean(water_level) AS water_level, mean(discharge) AS discharge, mean(discharge_percent) AS discharge_percent
-  INTO \"downsampled\".\"water_data_hourly\" 
-  FROM \"water_data\" 
-  GROUP BY time(1h), station_code, station_name_en, station_name_th
-END
-"
-```
-
-## 2. VictoriaMetrics Deployment (High Performance)
-
-### Why VictoriaMetrics?
-- **Extremely fast** and resource-efficient
-- **Better compression** than InfluxDB
-- **Prometheus-compatible** API
-- **Lower memory usage**
-- **Built-in clustering**
-
-### Docker Deployment
-```yaml
-# docker-compose.yml
-version: '3.8'
-services:
-  victoriametrics:
-    image: victoriametrics/victoria-metrics:latest
-    container_name: water_victoriametrics
-    ports:
-      - "8428:8428"
-    volumes:
-      - vm_data:/victoria-metrics-data
-    command:
-      - '--storageDataPath=/victoria-metrics-data'
-      - '--retentionPeriod=2y'
-      - '--httpListenAddr=:8428'
-      - '--maxConcurrentInserts=16'
-    restart: unless-stopped
-
-volumes:
-  vm_data:
-```
-
-### Environment Variables
-```bash
-# .env file
-DB_TYPE=victoriametrics
-VM_HOST=localhost
-VM_PORT=8428
-```
-
-## 3. PostgreSQL Deployment (Relational + Time-Series)
-
-### Why PostgreSQL?
-- **Mature and reliable**
-- **Excellent for complex queries**
-- **TimescaleDB extension** for time-series optimization
-- **Strong consistency guarantees**
-- **Rich ecosystem**
-
-### Docker Deployment with TimescaleDB
-```yaml
-# docker-compose.yml
-version: '3.8'
-services:
-  postgres:
-    image: timescale/timescaledb:latest-pg14
-    container_name: water_postgres
-    ports:
-      - "5432:5432"
-    volumes:
-      - postgres_data:/var/lib/postgresql/data
-      - ./init.sql:/docker-entrypoint-initdb.d/init.sql
-    environment:
-      - POSTGRES_DB=water_monitoring
-      - POSTGRES_USER=water_user
-      - POSTGRES_PASSWORD=secure_password
-    restart: unless-stopped
-
-volumes:
-  postgres_data:
-```
-
-### Database Initialization
-```sql
--- init.sql
-CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE;
-
--- Create hypertable for time-series optimization
-CREATE TABLE water_measurements (
-    id BIGSERIAL PRIMARY KEY,
-    timestamp TIMESTAMPTZ NOT NULL,
-    station_id INT NOT NULL,
-    water_level NUMERIC(10,3),
-    discharge NUMERIC(10,2),
-    discharge_percent NUMERIC(5,2),
-    status VARCHAR(20) DEFAULT 'active',
-    created_at TIMESTAMPTZ DEFAULT NOW()
-);
-
--- Convert to hypertable (TimescaleDB)
-SELECT create_hypertable('water_measurements', 'timestamp', chunk_time_interval => INTERVAL '1 day');
-
--- Create indexes
-CREATE INDEX idx_water_measurements_station_time ON water_measurements (station_id, timestamp DESC);
-CREATE INDEX idx_water_measurements_timestamp ON water_measurements (timestamp DESC);
-
--- Create retention policy (keep raw data for 2 years)
-SELECT add_retention_policy('water_measurements', INTERVAL '2 years');
-
--- Create continuous aggregates for performance
-CREATE MATERIALIZED VIEW water_measurements_hourly
-WITH (timescaledb.continuous) AS
-SELECT 
-    time_bucket('1 hour', timestamp) AS bucket,
-    station_id,
-    AVG(water_level) as avg_water_level,
-    MAX(water_level) as max_water_level,
-    MIN(water_level) as min_water_level,
-    AVG(discharge) as avg_discharge,
-    MAX(discharge) as max_discharge,
-    MIN(discharge) as min_discharge,
-    AVG(discharge_percent) as avg_discharge_percent
-FROM water_measurements
-GROUP BY bucket, station_id;
-
--- Refresh policy for continuous aggregates
-SELECT add_continuous_aggregate_policy('water_measurements_hourly',
-    start_offset => INTERVAL '1 day',
-    end_offset => INTERVAL '1 hour',
-    schedule_interval => INTERVAL '1 hour');
-```
-
-### Environment Variables
-```bash
-# .env file
-DB_TYPE=postgresql
-POSTGRES_CONNECTION_STRING=postgresql://water_user:secure_password@localhost:5432/water_monitoring
-```
-
-## 4. MySQL Deployment (Traditional Relational)
-
-### Docker Deployment
-```yaml
-# docker-compose.yml
-version: '3.8'
-services:
-  mysql:
-    image: mysql:8.0
-    container_name: water_mysql
-    ports:
-      - "3306:3306"
-    volumes:
-      - mysql_data:/var/lib/mysql
-      - ./mysql.cnf:/etc/mysql/conf.d/mysql.cnf
-      - ./init.sql:/docker-entrypoint-initdb.d/init.sql
-    environment:
-      - MYSQL_ROOT_PASSWORD=root_password
-      - MYSQL_DATABASE=water_monitoring
-      - MYSQL_USER=water_user
-      - MYSQL_PASSWORD=water_password
-    restart: unless-stopped
-
-volumes:
-  mysql_data:
-```
-
-### MySQL Configuration
-```ini
-# mysql.cnf
-[mysqld]
-# Optimize for time-series data
-innodb_buffer_pool_size = 1G
-innodb_log_file_size = 256M
-innodb_flush_log_at_trx_commit = 2
-innodb_flush_method = O_DIRECT
-
-# Partitioning support
-partition = ON
-
-# Query cache
-query_cache_type = 1
-query_cache_size = 128M
-
-# Connection settings
-max_connections = 200
-connect_timeout = 10
-wait_timeout = 600
-```
-
-### Environment Variables
-```bash
-# .env file
-DB_TYPE=mysql
-MYSQL_CONNECTION_STRING=mysql://water_user:water_password@localhost:3306/water_monitoring
-```
-
-## 5. Installation and Dependencies
-
-### Required Python Packages
-```bash
-# Base requirements
-pip install requests schedule
-
-# Database-specific packages
-pip install influxdb                    # For InfluxDB
-pip install sqlalchemy pymysql         # For MySQL
-pip install sqlalchemy psycopg2-binary # For PostgreSQL
-# VictoriaMetrics uses HTTP API (no extra packages needed)
-```
-
-### Updated requirements.txt
-```txt
-requests>=2.28.0
-schedule>=1.2.0
-pandas>=1.5.0
-
-# Database adapters (install as needed)
-influxdb>=5.3.1
-sqlalchemy>=1.4.0
-pymysql>=1.0.2
-psycopg2-binary>=2.9.0
-```
-
-## 6. Production Deployment Examples
-
-### Using InfluxDB (Recommended)
-```bash
-# Set environment variables
-export DB_TYPE=influxdb
-export INFLUX_HOST=your-influx-server.com
-export INFLUX_PORT=8086
-export INFLUX_DATABASE=water_monitoring
-export INFLUX_USERNAME=water_user
-export INFLUX_PASSWORD=your_secure_password
-
-# Run the scraper
-python water_scraper_v3.py
-```
-
-### Using PostgreSQL with TimescaleDB
-```bash
-# Set environment variables
-export DB_TYPE=postgresql
-export POSTGRES_CONNECTION_STRING=postgresql://water_user:password@your-postgres-server.com:5432/water_monitoring
-
-# Run the scraper
-python water_scraper_v3.py
-```
-
-### Using VictoriaMetrics
-```bash
-# Set environment variables
-export DB_TYPE=victoriametrics
-export VM_HOST=your-vm-server.com
-export VM_PORT=8428
-
-# Run the scraper
-python water_scraper_v3.py
-```
-
-## 7. Monitoring and Alerting
-
-### Grafana Dashboard Setup
-1. **Add Data Source**: Configure your database as a Grafana data source
-2. **Import Dashboard**: Use pre-built water monitoring dashboards
-3. **Set Alerts**: Configure alerts for abnormal water levels or discharge rates
-
-### Example Grafana Queries
-
-#### InfluxDB Queries
-```sql
--- Current water levels
-SELECT last("water_level") FROM "water_data" GROUP BY "station_code"
-
--- Discharge trends (last 24h)
-SELECT mean("discharge") FROM "water_data" WHERE time >= now() - 24h GROUP BY time(1h), "station_code"
-```
-
-#### PostgreSQL/TimescaleDB Queries
-```sql
--- Current water levels
-SELECT DISTINCT ON (station_id) 
-    station_id, water_level, discharge, timestamp
-FROM water_measurements 
-ORDER BY station_id, timestamp DESC;
-
--- Hourly averages (last 24h)
-SELECT 
-    time_bucket('1 hour', timestamp) as hour,
-    station_id,
-    AVG(water_level) as avg_level,
-    AVG(discharge) as avg_discharge
-FROM water_measurements 
-WHERE timestamp >= NOW() - INTERVAL '24 hours'
-GROUP BY hour, station_id
-ORDER BY hour DESC;
-```
-
-## 8. Performance Optimization Tips
-
-### For All Databases
-- **Batch inserts**: Insert multiple measurements at once
-- **Connection pooling**: Reuse database connections
-- **Indexing**: Ensure proper indexes on timestamp and station_id
-- **Retention policies**: Automatically delete old data
-
-### InfluxDB Specific
-- Use **tags** for metadata (station codes, names)
-- Use **fields** for numeric values (water levels, discharge)
-- Configure **retention policies** and **continuous queries**
-- Enable **compression** for long-term storage
-
-### PostgreSQL/TimescaleDB Specific
-- Use **hypertables** for automatic partitioning
-- Create **continuous aggregates** for common queries
-- Configure **compression** for older chunks
-- Use **parallel queries** for large datasets
-
-### VictoriaMetrics Specific
-- Use **labels** efficiently (similar to Prometheus)
-- Configure **retention periods** appropriately
-- Use **downsampling** for long-term storage
-- Enable **deduplication** if needed
-
-This deployment guide provides production-ready configurations for all supported database backends. Choose the one that best fits your infrastructure and requirements.

docs/DATA_SOURCES.md

@@ -0,0 +1,264 @@
+# Data Sources & External API Catalog
+
+Catalog of every data source available to the Ping River Monitor — what we ingest
+today, what the ThaiWater/HII ecosystem exposes, and vetted external feeds for
+future model inputs (rainfall, dam releases, forecasts).
+
+All "verified" claims below were empirically probed on **2026-08-11**. Endpoints
+marked *(catalog)* were recovered from the thaiwater.net frontend JS bundle but
+not exercised (auth required).
+
+---
+
+## 1. Currently ingested — RID hydrology telemetry
+
+The only source persisted to the database and used by the ML pipeline.
+
+| | |
+|---|---|
+| Endpoint | `POST https://hyd-app-db.rid.go.th/webservice/getGroupHourlyWaterLevelReportAllHL.ashx` |
+| Agency | Royal Irrigation Department (RID) |
+| Auth | None |
+| Cadence | Hourly (`hourlytime` 1.00–24.00; hour 24 = midnight next day) |
+| Params | `DW[UtokID]=1`, `DW[BasinID]=6` (Ping), `DW[TimeCurrent]=<Buddhist-calendar date>`, `rows=100` |
+| Variables | Water level (m, gauge datum), discharge (m³/s, `'***'` = malformed), discharge % of channel capacity |
+| Stations | 16 P-series gauges (P.1 anchor at Nawarat Bridge; see `src/data/stations.json`) |
+| Client | `rust/collector/src/rid.rs` (`ping-collector`; `fill-gaps` re-fetches old reports) |
+
+Human-facing page: <https://hyd-app-db.rid.go.th/hydro1h.html>
+
+---
+
+## 2. ThaiWater / HII ecosystem
+
+ThaiWater (<https://twa.thaiwater.net>) is the National Hydroinformatics
+Institute (HII) portal. It sits on **two distinct API layers** with very
+different access rules.
+
+### 2.1 `api-v3.thaiwater.net` — open, no authentication ✅
+
+Base: `https://api-v3.thaiwater.net/api/v1/thaiwater30/public/`
+
+Undocumented backend of the portal. No key, no session. No published rate
+limits or terms of use — be a good citizen (hourly polls, cache, filter to
+Ping basin `basin_code == 6`).
+
+#### `waterlevel_load` — national water-level snapshot (verified)
+
+```
+GET https://api-v3.thaiwater.net/api/v1/thaiwater30/public/waterlevel_load
+```
+
+- ~2.4 MB, 1,426 stations nationwide; **125 in Ping Basin, 61 in Chiang Mai
+  province, 34 with discharge**.
+- Per station: `waterlevel_m`, `waterlevel_msl`, `waterlevel_msl_previous`,
+  `flow_rate`, `discharge`, `storage_percent`, `situation_level` (1–4 flood
+  severity), `diff_wl_bank`, bank/critical levels (`min_bank`,
+  `critical_level_msl`, `warning_level_m`, `critical_level_m`, `qmax`),
+  `river_name`, basin, geocode, agency, lat/long, `is_key_station`.
+- Ping key stations present: P.1, P.67, P.75, P.81, P.92, P.17, P.87, P.4A,
+  P.7A, P.77, P.2A — plus RID mirrors (`ridhydro_P.1`, …), Tak-reach `TP.`/
+  `TUP.` codes, and HII sensor clusters (`PIN001-011`, `CHM001-005`).
+- **P.1 = internal station id `3226`** (lat 18.786961, long 99.005089).
+
+#### `waterlevel_graph` — hourly historical time series (verified) ⭐
+
+```
+GET .../waterlevel_graph?station_type=tele_waterlevel&station_id=3226&start_date=2024-09-25&end_date=2024-10-08
+```
+
+- Returns `{data: {graph_data: [{datetime, value, value_out, discharge}]}}`,
+  hourly. `value` is **water level in m MSL** (not gauge datum).
+- **Respects arbitrary date ranges. Archive verified back to at least
+  2019-08** (P.1 returned data for 2019-08-01). 14-day windows tested OK;
+  maximum window size not probed.
+- Oct 2024 record flood fully present: peak 305.8 m MSL @ 2024-10-05 12:00,
+  discharge 656 m³/s.
+- Datum conversion at P.1: 305.8 MSL peak = 5.30 m gauge ⇒
+  **gauge ≈ MSL − 300.5 m** (verify per station before use; each station has
+  its own datum offset).
+- Value: independent second historical source for cross-validating / gap-filling
+  the RID feed (RID grid is only ~56% filled).
+
+#### `rain_24h` — national rainfall snapshot (verified)
+
+```
+GET https://api-v3.thaiwater.net/api/v1/thaiwater30/public/rain_24h
+```
+
+- ~4.5 MB, 4,445 stations; **321 in Chiang Mai province**; agencies include
+  HII, DWR, RID.
+- Per station: `rain_1h`, `rain_24h` (mm), `rainfall_datetime`,
+  `station.id` (small int — the graph key), `station.tele_station_oldcode`
+  (e.g. `CHM005`, `STN0410`, `ridtele_TUP.14`), `sub_basin_id`, lat/long,
+  basin, geocode, agency.
+- **This is the missing rainfall input** for the flood model — near-real-time
+  hourly gauge rain across the upper Ping catchment.
+
+#### `rain_24h_graph` — trailing-window rainfall series (verified, limited) ⚠️
+
+```
+GET .../rain_24h_graph?station_type=tele_rainfall&station_id=418&start_date=...&end_date=...
+```
+
+- `station_id` is the **small `station.id`** from `rain_24h` (e.g. 418 =
+  CHM005 "Chiang Mai 5", Mae Taeng), *not* the top-level record id.
+- Returns hourly `{rainfall_datetime, rainfall_value}` — **but the date range
+  is IGNORED**: every request returns the same trailing ~36-hour window
+  (39 rows). Requests for 2020/2024 return identical data to today.
+- Consequence: **no rainfall history via this API**. To build training data,
+  persist `rain_24h` from now on and backfill history from satellite QPE or an
+  HII data request (§4).
+
+#### Probed and NOT available on api-v3 (all HTTP 404)
+
+`dam_daily`, `dam`, `dam_json`, `big_dam`, `mainstream_dam`, `weather`,
+`rain_graph`, `rainfall_graph`, `rain24hr_graph`. Dam data is v2-only (§2.2).
+
+### 2.2 `twa-api-public.thaiwater.net` — auth-gated (x-api-key / session) 🔒
+
+The layer the website's `GET /sensors/thaiwater` uses (`GET /v2/waterlevel`
+with an `x-api-key` header from `THAIWATER_API_KEY`;
+`rust/collector/src/web/hii.rs`, display-only, never persisted).
+Without a key: HTTP 401/500. No public key-registration page was found —
+obtain a sanctioned key from HII (<https://hii.or.th>).
+
+Full endpoint catalog *(catalog — recovered from frontend JS, not exercised)*:
+
+- **Water level / discharge**: `/v2/waterlevel`, `/v2/waterlevel/list`,
+  `/v2/waterlevel/{id}/detail`, `/v2/waterlevel/canal`,
+  `/v2/waterlevel/sea-waterlevel`, `/v2/waterlevel-discharge`,
+  `/v2/waterlevel-discharge/list`, `/v2/waterlevel-discharge/{id}/detail`,
+  `/v2/waterlevel-discharge/{id}/forecast-table`,
+  `/v2/waterlevel-discharge/forecast`, `/v2/waterlevel-discharge/forecast/list`,
+  `/v2/watergate`, `/v2/waterload-tide`
+- **Dams** (incl. Bhumibol): `/v2/large-dam/daily-geo-json`,
+  `/v2/large-dam/daily/list`, `/v2/large-dam/hourly/list`,
+  `/v2/large-dam/daily/{id}/detail`, `/v2/large-dam/hourly/{id}/detail`,
+  `/v2/medium-dam/daily-geo-json`, `/v2/medium-dam/daily/list`,
+  `/v2/medium-dam/{id}/detail`, `/v2/summary/summary4dam`,
+  `/v2/summary/dam-summary`, `/v2/summary/dam-crisis`
+- **Rainfall**: `/v2/rainfall/{type}`, `/v2/rainfall/{type}/list`,
+  `/v2/district-rain/actual-measure`, `/v2/district-rain/forecast`,
+  `/v2/district-rain/accumulate`, `/v2/summary/rainfall24h-ranking-province`,
+  `/v2/summary/rainfall-24hr-forecast`, `/v2/summary/rainfall-forecast`,
+  `/v2/summary/warning-rainfall-24h`, `/v2/summary/warning-rainfall-48h`
+- **Weather / hazards**: `/v2/weather`, `/v2/storm`, `/v2/wave`, `/v2/pm25`,
+  `/v2/pm10`, `/v2/flood/flash-flood`, `/v2/flood/flash-flood-alert`,
+  `/v2/drought/alert`, `/v2/drought/risk-area/list`,
+  `/v2/summary/weather-summary`, `/v2/summary/temperature-forecast`,
+  `/v2/summary-area/rainfall`
+- **Time-series / graph** (base `/data/platform/v1/public/`):
+  `tele_waterlevel/graph`, `flow/graph`, `latest_waterlevel/forecast/graph`,
+  `latest_watertide/forecast/graph`, `dam_pdaily_sum_by_date`,
+  `dam_pdaily_sum_by_region_graph`, `dam_rulecurve/graph`,
+  `medium_dam/graph_year`, `monthly_rainfall/stations`,
+  `monthly_rainfall/anomaly-stations`, `tele_watergate/graph`,
+  `salinity_forecast_cpy/graph`, `sea_waterlevel_forecast/graph`,
+  `latest_weather_area`, `latest_weather_area_daily`
+
+### 2.3 Other HII hosts
+
+| Host | What | Access |
+|---|---|---|
+| `https://standard.thaiwater.net` | **Official water-data standard** — canonical station/basin/province code registries, data-exchange formats, warning-level definitions (Thai) | Open, docs site |
+| `https://api.hii.or.th/tiservice/v1/ws/{token}/isohyet/daily/latest/province/{code}` | Daily isohyet rainfall by province | Token in path |
+| `https://live1.hii.or.th/product/latest/rain/one_map/data/*.tif` | Rainfall anomaly & 1–6-month forecast GeoTIFF rasters | Open |
+| `https://data.hii.or.th` | HII open-data catalog — 36 datasets (rainfall telemetry, water level, weather, climate) | Open browsing |
+| `https://tiwrm.hii.or.th` | Legacy reports | Open |
+
+Historical bulk telemetry: HII documents a request channel at
+**nhcsoc@hii.or.th**.
+
+---
+
+## 3. Dams & reservoirs
+
+> **Geography matters: Bhumibol Dam (Tak) is ~240 km DOWNSTREAM of P.1** and
+> cannot influence Chiang Mai water levels. Do not use it as a P.1 feature.
+
+The predictive upstream reservoir is **Mae Ngat Somboon Chon** (Mae Ngat
+tributary, joins the Ping above Chiang Mai; spilled 110 m³/s during the
+Oct 2024 flood). Mae Kuang Udom Thara is the second upstream reservoir.
+
+| Source | What | Access |
+|---|---|---|
+| `https://app.rid.go.th/reservoir/api/dams` | **INGESTED** — daily snapshot of all ~35 large dams (storage/inflow/outflow MCM, % of usable). `POST` with form field `date=YYYY-MM-DD` (empty = today); GET returns 404 "Unknown method." Archive ≥ 2009; `level_msl` (`DMD_Q`) populated in older years only. Mae Ngat = `DAM_ID 200103` — hit 113% usable capacity, ~19 MCM/day inflow, in Oct 2024. Collected daily by `src/rid_reservoir.py` into `rid_dams` + `rid_reservoir_daily`; backfill via `scripts/backfill_rid_reservoir.py` | Open, no auth |
+| `https://lsim.rid.go.th/ForeCast?reservoirid=22` | Mae Ngat daily status/forecast (RID) | **UNREACHABLE — do not plan around it.** Probed 2026-08-13 from a Thai consumer ISP (AIS Fibre, TH) *and* from abroad: DNS resolves (122.154.18.207) but ICMP is 100% loss and ports 80/443/8080 are filtered, while `app.rid.go.th` answers in 0.27 s over the same connection. Down or RID-internal-only — not a geo-block |
+| `https://app.rid.go.th/reservoir/api/dam` | **Per-dam daily series in ONE request** — `GET` with `dam_id=200103&date_start=YYYY-MM-DD&date_end=YYYY-MM-DD&percent=`. Archive to 2009 (scattered single-day gaps). Far cheaper than the per-day `api/dams` loop the backfill used (one request vs ~2,900); prefer it for gap repair and for adding other dams. Sibling `api/damgraph` takes the same params | Open, no auth |
+| `https://bigdata-api.rid.go.th` (SWOC) | **Intraday reservoir state** — RID SWOC telemetry, hourly with an explicit `hourly_time_utc` stamp; includes Mae Ngat (`TUP.16`) reservoir level m MSL and % capacity. **Snapshot-only — no archive**, so it can only be accumulated forward | Open, no auth |
+| ThaiWater `public/waterlevel_load` stations `ridhydro_TUP.16` (at the dam) / `ridhydro_TUP.11` (dam outlet) | Hourly Mae Ngat reservoir level and outlet stage/flow. **Snapshot-only — `waterlevel_graph` returns empty grids for these ids at every era** (verified 2026-08-13). Collected hourly by our HII collector since 2026-08-11; accumulating forward | Open, no auth |
+| ThaiWater `public/waterlevel_graph` station `P.75` (id 3253) | **The practical dam-release signal**: hourly stage+discharge 3.8 km below the Mae Ngat dam, history to 2019. Already ingested as a core RID station and a model feature since v1 | Open, no auth |
+| ThaiWater `public/waterlevel_graph` station `MOU301` "สะพานน้ำแม่งัด" (id 1475118) | 10-minute stage on the Mae Ngat *above* the reservoir (inflow arm). History only from ~mid-2025; level only, no discharge | Open, no auth |
+| ThaiWater `api-v3 .../analyst/dam` (dam.id 53) | EGAT-sourced copy of Mae Ngat carrying reservoir **level in m MSL historically** — the field RID's own API stopped populating (`DMD_Q`) after ~2013. Daily, no observation time | Open, no auth |
+| `https://tiwrm.hii.or.th/DATA/REPORT/php/rid_bigcm_raw.php?sdate=YYYY-MM-DD` | HII HTML mirror of the RID large-dam daily table. Daily and *intermittent* (2026 YTD publishes ~108 of 225 days) — a cross-check, not a primary source | Open, no auth |
+| `https://water.egat.co.th` | EGAT dams (Bhumibol/Sirikit) hourly+daily inflow/outflow/level | Endpoint catalog not public; contact EGAT (0-2436-8186). Only relevant downstream of Bhumibol |
+| ThaiWater `/v2/large-dam/*`, `dam_rulecurve/graph` | All large/medium dams incl. hourly | Requires HII API key (§2.2) |
+
+---
+
+## 4. Rainfall & weather (external)
+
+### Near-real-time (usable in the live inference path)
+
+| Source | Cadence / latency | Access | Notes |
+|---|---|---|---|
+| HII `rain_24h` (§2.1) | Hourly, near-real-time | Open JSON | Primary rain-gauge feed; persist from now on |
+| GSMaP NRT (JAXA) | Hourly, ~4 h latency, 0.1° | Free JAXA registration (FTP); or Google Earth Engine `JAXA/GPM_L3/GSMaP/v8/operational` (no registration) | Gauge-corrected `hourlyPrecipRateGC`; catchment-average rain where gauges are sparse |
+| NASA IMERG **Early Run** | Half-hourly, ~4 h latency, 0.1° | Free Earthdata login | NASA-stack alternative to GSMaP |
+
+### Forecasts (the only way past the ~17 h physical lead-time cap)
+
+| Source | What | Access |
+|---|---|---|
+| **Open-Meteo** (<https://open-meteo.com>) — ✅ **INGESTED** since 2026-08-12 (`src/ml/rain.py`): 5 upper-Ping catchment points feed the hgb-v3 model's rain features (trailing sums + forward-24h forecast, archive 2021+); the leader worker also persists hourly rows to the `openmeteo_rain` table | Hourly precip forecast ≤16 days, any lat/lon; **Historical Forecast API archive from 2021** (train on forecast-as-seen, leakage-free); Previous Runs API (fixed 1–7-day leads from Jan 2024); ERA5 back to 1940 | Free, no key, 10k calls/day, non-commercial w/ attribution |
+| TMD NWP API (`https://data.tmd.go.th/nwpapi/v1/forecast/location/...`) | WRF 4.2 daily/hourly forecasts by place, processed ~06:00 daily | Free Bearer-token registration (`/nwpapi/doc/main/`) |
+| GFS / ECMWF IFS open data | 0.25° global, 4×/day | Free (NOMADS / AWS / data.ecmwf.int); Open-Meteo already wraps both |
+
+### Training-only (too slow for live)
+
+| Source | Cadence | Latency |
+|---|---|---|
+| CHIRPS (`data.chc.ucsb.edu/products/CHIRPS-2.0/`) | Daily, 0.05° | ~2 days prelim / 3+ weeks final |
+| IMERG Late / Final | Half-hourly | ~14 h / ~3.5 months |
+| TMD observation API (`data.tmd.go.th/api/index1.php`) | 3-hourly / daily station obs, XML | Free uid+key registration |
+
+---
+
+## 5. Historical / open-data portals
+
+| Portal | Content |
+|---|---|
+| `https://data.hii.or.th` | 36 HII datasets (rainfall telemetry the most viewed) |
+| `https://data.go.th/dataset?organization=rid` | 4 RID datasets (API + ZIP) |
+| `https://gdcatalog.go.th` | Nationwide daily rainfall-station catalogs |
+| `https://hydro-1.net` | RID Upper-Northern Hydrology Center — hourly/daily tables, hydrology yearbooks (rating curves) for P-series stations; scrape/download |
+| `https://water.rid.go.th/flood/flood/daily.pdf` | RID daily flood bulletin (PDF only) |
+
+---
+
+## 6. Integration status & recommended order
+
+| Source | Status | Action |
+|---|---|---|
+| RID hourly gauges | ✅ Ingested (hourly → PostgreSQL) | — |
+| ThaiWater `/v2/waterlevel` | 🟡 Display-only (`/sensors/thaiwater`, needs `THAIWATER_API_KEY`, never persisted) | Optionally persist |
+| HII `rain_24h` | ✅ Ingested hourly by ping-collector (`rust/collector/src/hii.rs`) → `hii_rain_stations` + `hii_rainfall` (Ping-filtered; ~300 stations) | — |
+| HII `waterlevel_load` | ✅ Ingested hourly by ping-collector (`rust/collector/src/hii.rs`) → `hii_wl_stations` + `hii_waterlevel` (125 Ping stations, m MSL; `rid_code` column maps mirrors like `ridhydro_P.1` → `P.1`, `offset_msl` converts MSL → gauge datum) | — |
+| HII `waterlevel_graph` | ✅ Backfill via `scripts/backfill_hii_waterlevel.py` → `hii_waterlevel` (hourly MSL + discharge, archive ≥2019; full-year windows per request; upserts never overwrite live-snapshot columns) | Run once on the box: `python scripts/backfill_hii_waterlevel.py` (defaults: 2019-01-01 → today, RID-mirror + key stations; `--stations P.1,P.67`, `--all` for every Ping station) |
+| Mae Ngat reservoir (app.rid.go.th) | ✅ Ingested (2026-08-13) | Daily storage/inflow/outflow for all large dams → `rid_reservoir_daily`; candidate model features for next retrain |
+| Satellite QPE (GSMaP/IMERG) | ❌ | Backfill training rainfall (GEE) |
+| Open-Meteo rain | ✅ Hourly by ping-collector → `openmeteo_rain` (5 catchment points, past 5 days + 48 h forecast); the 2021+ archive trains the hgb-v3 rain features | — |
+| HII API key (dams, forecasts) | ❌ | Contact HII for sanctioned access |
+
+**Collector configuration** (ping-collector): HII, the dams and Open-Meteo are
+fetched once per clock hour, also while RID is being retried; one-shot via
+`ping-collector once`. Env vars: `ENABLE_HII_COLLECTION`,
+`ENABLE_RESERVOIR_COLLECTION`, `ENABLE_RAIN_COLLECTION` (default `true`),
+`HII_BASIN_CODE` (default `6` = Ping). Tables are created automatically
+(PostgreSQL). The archive backfills stay in Python (`scripts/backfill_*.py`,
+using `src/hii_collector.py` and `src/rid_reservoir.py`).
+
+**Caveats**: `api-v3` is an undocumented backend — no SLA, no ToS, may change
+without notice. Poll hourly at most, cache aggressively, and pursue official
+HII access for anything production-critical.

docs/DEBIAN_TROUBLESHOOTING.md

@@ -1,329 +0,0 @@
-# Debian/Linux Troubleshooting Guide
-
-This guide addresses common issues when running the Thailand Water Monitor on Debian and other Linux distributions.
-
-## Fixed Issues
-
-### SQLAlchemy Connection Error (RESOLVED)
-
-**Error Message:**
-```
-2025-07-24 19:48:31,920 - ERROR - Failed to connect to SQLITE: 'Connection' object has no attribute 'commit'
-2025-07-24 19:48:32,740 - ERROR - Error saving to SQLITE: 'Connection' object has no attribute 'commit'
-```
-
-**Root Cause:**
-This error occurred due to incompatibility between the database adapter code and newer versions of SQLAlchemy. The code was calling `conn.commit()` on a connection object that doesn't have a `commit()` method in newer SQLAlchemy versions.
-
-**Solution Applied:**
-Changed from `engine.connect()` to `engine.begin()` context manager, which automatically handles transactions:
-
-```python
-# OLD (problematic) code:
-with self.engine.connect() as conn:
-    conn.execute(text(sql))
-    conn.commit()  # This fails in newer SQLAlchemy
-
-# NEW (fixed) code:
-with self.engine.begin() as conn:
-    conn.execute(text(sql))
-    # Transaction automatically committed when context exits
-```
-
-**Status:** ✅ **FIXED** - The issue has been resolved in the current version.
-
-## Installation on Debian/Ubuntu
-
-### System Requirements
-
-```bash
-# Update package list
-sudo apt update
-
-# Install Python and pip
-sudo apt install python3 python3-pip python3-venv
-
-# Install system dependencies for database drivers
-sudo apt install build-essential python3-dev
-
-# For MySQL support (optional)
-sudo apt install default-libmysqlclient-dev
-
-# For PostgreSQL support (optional)
-sudo apt install libpq-dev
-```
-
-### Python Environment Setup
-
-```bash
-# Create virtual environment
-python3 -m venv water_monitor_env
-
-# Activate virtual environment
-source water_monitor_env/bin/activate
-
-# Install requirements
-pip install -r requirements.txt
-```
-
-### Running the Monitor
-
-```bash
-# Test run
-python water_scraper_v3.py --test
-
-# Run with specific database
-export DB_TYPE=sqlite
-python water_scraper_v3.py
-
-# Run demo
-python demo_databases.py
-```
-
-## Common Linux Issues
-
-### 1. Permission Errors
-
-**Error:**
-```
-PermissionError: [Errno 13] Permission denied: 'water_levels.db'
-```
-
-**Solution:**
-```bash
-# Check current directory permissions
-ls -la
-
-# Create data directory with proper permissions
-mkdir -p data
-chmod 755 data
-
-# Set database path to data directory
-export WATER_DB_PATH=data/water_levels.db
-```
-
-### 2. Missing System Dependencies
-
-**Error:**
-```
-ImportError: No module named '_sqlite3'
-```
-
-**Solution:**
-```bash
-# Install SQLite development headers
-sudo apt install libsqlite3-dev
-
-# Reinstall Python if needed
-sudo apt install python3-sqlite
-```
-
-### 3. Network/Firewall Issues
-
-**Error:**
-```
-requests.exceptions.ConnectionError: HTTPSConnectionPool
-```
-
-**Solution:**
-```bash
-# Test network connectivity
-curl -I https://hyd-app-db.rid.go.th/hydro1h.html
-
-# Check firewall rules
-sudo ufw status
-
-# Allow outbound HTTPS if needed
-sudo ufw allow out 443
-```
-
-### 4. Systemd Service Setup
-
-Create service file `/etc/systemd/system/water-monitor.service`:
-
-```ini
-[Unit]
-Description=Thailand Water Level Monitor
-After=network.target
-
-[Service]
-Type=simple
-User=water-monitor
-Group=water-monitor
-WorkingDirectory=/opt/water_level_monitor
-Environment=PATH=/opt/water_level_monitor/venv/bin
-Environment=DB_TYPE=sqlite
-Environment=WATER_DB_PATH=/opt/water_level_monitor/data/water_levels.db
-ExecStart=/opt/water_level_monitor/venv/bin/python water_scraper_v3.py
-Restart=always
-RestartSec=60
-
-# Security settings
-NoNewPrivileges=true
-PrivateTmp=true
-ProtectSystem=strict
-ProtectHome=true
-ReadWritePaths=/opt/water_level_monitor/data
-ReadWritePaths=/opt/water_level_monitor/logs
-
-[Install]
-WantedBy=multi-user.target
-```
-
-Enable and start:
-```bash
-sudo systemctl daemon-reload
-sudo systemctl enable water-monitor.service
-sudo systemctl start water-monitor.service
-sudo systemctl status water-monitor.service
-```
-
-### 5. Log Rotation
-
-Create `/etc/logrotate.d/water-monitor`:
-
-```
-/opt/water_level_monitor/water_monitor.log {
-    daily
-    missingok
-    rotate 30
-    compress
-    delaycompress
-    notifempty
-    create 644 water-monitor water-monitor
-    postrotate
-        systemctl reload water-monitor.service
-    endscript
-}
-```
-
-## Database-Specific Issues
-
-### SQLite
-
-**Issue:** Database locked
-```bash
-# Check for processes using the database
-sudo lsof /path/to/water_levels.db
-
-# Kill processes if needed
-sudo pkill -f water_scraper_v3.py
-```
-
-### VictoriaMetrics with HTTPS
-
-**Configuration:**
-```bash
-export DB_TYPE=victoriametrics
-export VM_HOST=https://your-vm-server.com
-export VM_PORT=443
-```
-
-**Test connection:**
-```bash
-curl -k https://your-vm-server.com/health
-```
-
-## Performance Optimization
-
-### 1. System Tuning
-
-```bash
-# Increase file descriptor limits
-echo "* soft nofile 65536" >> /etc/security/limits.conf
-echo "* hard nofile 65536" >> /etc/security/limits.conf
-
-# Optimize network settings
-echo "net.core.rmem_max = 16777216" >> /etc/sysctl.conf
-echo "net.core.wmem_max = 16777216" >> /etc/sysctl.conf
-sysctl -p
-```
-
-### 2. Database Optimization
-
-```bash
-# For SQLite
-export SQLITE_CACHE_SIZE=10000
-export SQLITE_SYNCHRONOUS=NORMAL
-
-# Monitor database size
-du -h data/water_levels.db
-```
-
-## Monitoring and Maintenance
-
-### Health Check Script
-
-Create `health_check.sh`:
-
-```bash
-#!/bin/bash
-LOG_FILE="/opt/water_level_monitor/water_monitor.log"
-SERVICE_NAME="water-monitor"
-
-# Check if service is running
-if ! systemctl is-active --quiet $SERVICE_NAME; then
-    echo "ERROR: $SERVICE_NAME is not running"
-    systemctl restart $SERVICE_NAME
-    exit 1
-fi
-
-# Check recent log entries
-RECENT_ERRORS=$(tail -n 100 $LOG_FILE | grep -c "ERROR")
-if [ $RECENT_ERRORS -gt 5 ]; then
-    echo "WARNING: $RECENT_ERRORS errors found in recent logs"
-    exit 1
-fi
-
-echo "OK: Service is healthy"
-exit 0
-```
-
-### Cron Job for Health Checks
-
-```bash
-# Add to crontab
-*/5 * * * * /opt/water_level_monitor/health_check.sh >> /var/log/water-monitor-health.log 2>&1
-```
-
-## Getting Help
-
-### Debug Information
-
-```bash
-# System information
-uname -a
-python3 --version
-pip list | grep -E "(sqlalchemy|requests|influxdb)"
-
-# Service logs
-journalctl -u water-monitor.service -f
-
-# Application logs
-tail -f water_monitor.log
-
-# Database information
-sqlite3 water_levels.db ".schema"
-sqlite3 water_levels.db "SELECT COUNT(*) FROM water_measurements;"
-```
-
-### Common Commands
-
-```bash
-# Restart service
-sudo systemctl restart water-monitor.service
-
-# View logs
-sudo journalctl -u water-monitor.service --since "1 hour ago"
-
-# Test configuration
-python config.py
-
-# Test database connection
-python demo_databases.py
-
-# Manual data fetch
-python water_scraper_v3.py --test
-```
-
-This troubleshooting guide should help resolve most common issues encountered when running the Thailand Water Monitor on Debian and other Linux distributions.

docs/ENHANCED_SCHEDULER_GUIDE.md

@@ -1,293 +0,0 @@
-# Enhanced Scheduler Guide
-
-This guide explains the new 15-minute scheduling system that runs continuously throughout each hour to ensure comprehensive data coverage.
-
-## ✅ **New Scheduling Behavior**
-
-### **15-Minute Schedule Pattern**
-- **Timing**: Runs every 15 minutes: 1:00, 1:15, 1:30, 1:45, 2:00, 2:15, 2:30, 2:45, etc.
-- **Hourly Full Checks**: At :00 minutes (includes gap filling and data updates)
-- **Quarter-Hour Quick Checks**: At :15, :30, :45 minutes (data fetch only)
-- **Continuous Coverage**: Ensures no data is missed throughout each hour
-
-### **Operation Types**
-- **Full Operations** (at :00): Data fetching + gap filling + data updates
-- **Quick Operations** (at :15, :30, :45): Data fetching only for performance
-
-## 🔧 **Technical Implementation**
-
-### **Scheduler States**
-```python
-# State tracking variables
-self.last_successful_update = None  # Timestamp of last successful data update
-self.retry_mode = False            # Whether in quick check mode (skip gap filling)
-self.next_hourly_check = None      # Next scheduled hourly check
-```
-
-### **Quarter-Hour Check Process**
-```python
-def quarter_hour_check(self):
-    """15-minute check for new data"""
-    current_time = datetime.datetime.now()
-    minute = current_time.minute
-    
-    # Determine if this is a full hourly check (at :00) or a quarter-hour check
-    if minute == 0:
-        logging.info("=== HOURLY CHECK (00:00) ===")
-        self.retry_mode = False  # Full check with gap filling and updates
-    else:
-        logging.info(f"=== 15-MINUTE CHECK ({minute:02d}:00) ===")
-        self.retry_mode = True   # Skip gap filling and updates on 15-min checks
-    
-    new_data_found = self.run_scraping_cycle()
-    
-    if new_data_found:
-        self.last_successful_update = datetime.datetime.now()
-        if minute == 0:
-            logging.info("New data found during hourly check")
-        else:
-            logging.info(f"New data found during 15-minute check at :{minute:02d}")
-    else:
-        if minute == 0:
-            logging.info("No new data found during hourly check")
-        else:
-            logging.info(f"No new data found during 15-minute check at :{minute:02d}")
-```
-
-### **Scheduler Setup**
-```python
-def start_scheduler(self):
-    """Start enhanced scheduler with 15-minute checks"""
-    # Schedule checks every 15 minutes (at :00, :15, :30, :45)
-    schedule.every().hour.at(":00").do(self.quarter_hour_check)
-    schedule.every().hour.at(":15").do(self.quarter_hour_check)
-    schedule.every().hour.at(":30").do(self.quarter_hour_check)
-    schedule.every().hour.at(":45").do(self.quarter_hour_check)
-    
-    while True:
-        schedule.run_pending()
-        time.sleep(30)  # Check every 30 seconds
-```
-
-## 📊 **New Data Detection Logic**
-
-### **Smart Detection Algorithm**
-```python
-def has_new_data(self) -> bool:
-    """Check if there is new data available since last successful update"""
-    # Get most recent timestamp from database
-    latest_data = self.get_latest_data(limit=1)
-    
-    # Check if we should have newer data by now
-    now = datetime.datetime.now()
-    expected_latest = now.replace(minute=0, second=0, microsecond=0)
-    
-    # If current time is past 5 minutes after the hour, we should have data
-    if now.minute >= 5:
-        if latest_timestamp < expected_latest:
-            return True  # New data expected
-    
-    # Check if we have data for the previous hour
-    previous_hour = expected_latest - datetime.timedelta(hours=1)
-    if latest_timestamp < previous_hour:
-        return True  # Missing recent data
-    
-    return False  # Data is up to date
-```
-
-### **Actual Data Verification**
-```python
-# Compare timestamps before and after scraping
-initial_timestamp = get_latest_timestamp_before_scraping()
-# ... perform scraping ...
-latest_timestamp = get_latest_timestamp_after_scraping()
-
-if initial_timestamp is None or latest_timestamp > initial_timestamp:
-    new_data_found = True
-    self.last_successful_update = datetime.datetime.now()
-```
-
-## 🚀 **Operational Modes**
-
-### **Mode 1: Full Hourly Operation (at :00)**
-- **Schedule**: Every hour at :00 minutes (1:00, 2:00, 3:00, etc.)
-- **Operations**: 
-  - ✅ Fetch current data
-  - ✅ Fill data gaps (last 7 days)
-  - ✅ Update existing data (last 2 days)
-- **Purpose**: Comprehensive data collection and maintenance
-
-### **Mode 2: Quick 15-Minute Checks (at :15, :30, :45)**
-- **Schedule**: Every 15 minutes at quarter-hour marks
-- **Operations**:
-  - ✅ Fetch current data only
-  - ❌ Skip gap filling (performance optimization)
-  - ❌ Skip data updates (performance optimization)
-- **Purpose**: Ensure no new data is missed between hourly checks
-
-## 📋 **Logging Output Examples**
-
-### **Successful Hourly Check (at :00)**
-```
-2025-07-26 01:00:00,123 - INFO - === HOURLY CHECK (00:00) ===
-2025-07-26 01:00:00,124 - INFO - Starting scraping cycle...
-2025-07-26 01:00:01,456 - INFO - Successfully fetched 384 data points from API
-2025-07-26 01:00:02,789 - INFO - New data found: 2025-07-26 01:00:00
-2025-07-26 01:00:03,012 - INFO - Filled 5 data gaps
-2025-07-26 01:00:04,234 - INFO - Updated 2 existing measurements
-2025-07-26 01:00:04,235 - INFO - New data found during hourly check
-```
-
-### **15-Minute Quick Check (at :15, :30, :45)**
-```
-2025-07-26 01:15:00,123 - INFO - === 15-MINUTE CHECK (15:00) ===
-2025-07-26 01:15:00,124 - INFO - Starting scraping cycle...
-2025-07-26 01:15:01,456 - INFO - Successfully fetched 299 data points from API
-2025-07-26 01:15:02,789 - INFO - New data found: 2025-07-26 01:00:00
-2025-07-26 01:15:02,790 - INFO - New data found during 15-minute check at :15
-```
-
-### **Continuous 15-Minute Pattern**
-```
-2025-07-26 01:00:00,123 - INFO - === HOURLY CHECK (00:00) ===
-2025-07-26 01:00:04,235 - INFO - New data found during hourly check
-
-2025-07-26 01:15:00,123 - INFO - === 15-MINUTE CHECK (15:00) ===
-2025-07-26 01:15:02,790 - INFO - No new data found during 15-minute check at :15
-
-2025-07-26 01:30:00,123 - INFO - === 15-MINUTE CHECK (30:00) ===
-2025-07-26 01:30:02,790 - INFO - No new data found during 15-minute check at :30
-
-2025-07-26 01:45:00,123 - INFO - === 15-MINUTE CHECK (45:00) ===
-2025-07-26 01:45:02,790 - INFO - No new data found during 15-minute check at :45
-
-2025-07-26 02:00:00,123 - INFO - === HOURLY CHECK (00:00) ===
-2025-07-26 02:00:04,235 - INFO - New data found during hourly check
-```
-
-## ⚙️ **Configuration Options**
-
-### **Environment Variables**
-```bash
-# Retry interval (default: 5 minutes)
-export RETRY_INTERVAL_MINUTES=5
-
-# Data availability buffer (default: 5 minutes after hour)
-export DATA_BUFFER_MINUTES=5
-
-# Gap filling days (default: 7 days)
-export GAP_FILL_DAYS=7
-
-# Update check days (default: 2 days)
-export UPDATE_DAYS=2
-```
-
-### **Scheduler Timing**
-```python
-# Hourly checks at top of hour
-schedule.every().hour.at(":00").do(self.hourly_check)
-
-# 5-minute retries (dynamically scheduled)
-schedule.every(5).minutes.do(self.retry_check).tag('retry')
-
-# Check every 30 seconds for responsive retry scheduling
-time.sleep(30)
-```
-
-## 🔍 **Performance Optimizations**
-
-### **Retry Mode Optimizations**
-- **Skip Gap Filling**: Avoids expensive historical data fetching during retries
-- **Skip Data Updates**: Avoids comparison operations during retries
-- **Focused API Calls**: Only fetches current day data during retries
-- **Reduced Database Queries**: Minimal database operations during retries
-
-### **Resource Management**
-- **API Rate Limiting**: 1-second delays between API calls
-- **Database Connection Pooling**: Efficient connection reuse
-- **Memory Efficiency**: Selective data processing
-- **Error Recovery**: Automatic retry with exponential backoff
-
-## 🛠️ **Troubleshooting**
-
-### **Common Scenarios**
-
-#### **Stuck in Retry Mode**
-```
-# Check if API is returning data
-curl -X POST https://hyd-app-db.rid.go.th/webservice/getGroupHourlyWaterLevelReportAllHL.ashx
-
-# Check database connectivity
-python water_scraper_v3.py --check-gaps 1
-
-# Manual data fetch test
-python water_scraper_v3.py --test
-```
-
-#### **Missing Hourly Triggers**
-```
-# Check system time synchronization
-timedatectl status
-
-# Verify scheduler is running
-ps aux | grep water_scraper
-
-# Check logs for scheduler activity
-tail -f water_monitor.log | grep "HOURLY CHECK"
-```
-
-#### **False New Data Detection**
-```
-# Check latest data in database
-sqlite3 water_monitoring.db "SELECT MAX(timestamp) FROM water_measurements;"
-
-# Verify timestamp parsing
-python -c "
-import datetime
-print('Current hour:', datetime.datetime.now().replace(minute=0, second=0, microsecond=0))
-"
-```
-
-## 📈 **Monitoring and Alerts**
-
-### **Key Metrics to Monitor**
-- **Hourly Success Rate**: Percentage of hourly checks that find new data
-- **Retry Duration**: How long system stays in retry mode
-- **Data Freshness**: Time since last successful data update
-- **API Response Time**: Performance of data fetching operations
-
-### **Alert Conditions**
-- **Extended Retry Mode**: System in retry mode for > 30 minutes
-- **No Data for 2+ Hours**: No new data found for extended period
-- **High Error Rate**: Multiple consecutive API failures
-- **Database Issues**: Connection or save failures
-
-### **Health Check Script**
-```bash
-#!/bin/bash
-# Check if system is stuck in retry mode
-RETRY_COUNT=$(tail -n 100 water_monitor.log | grep -c "RETRY CHECK")
-if [ $RETRY_COUNT -gt 6 ]; then
-    echo "WARNING: System may be stuck in retry mode ($RETRY_COUNT retries in last 100 log entries)"
-fi
-
-# Check data freshness
-LATEST_DATA=$(sqlite3 water_monitoring.db "SELECT MAX(timestamp) FROM water_measurements;")
-echo "Latest data timestamp: $LATEST_DATA"
-```
-
-## 🎯 **Best Practices**
-
-### **Production Deployment**
-1. **Monitor Logs**: Watch for retry mode patterns
-2. **Set Alerts**: Configure notifications for extended retry periods
-3. **Regular Maintenance**: Weekly gap filling and data validation
-4. **Backup Strategy**: Regular database backups before major operations
-
-### **Performance Tuning**
-1. **Adjust Buffer Time**: Modify data availability buffer based on API patterns
-2. **Optimize Retry Interval**: Balance between responsiveness and API load
-3. **Database Indexing**: Ensure proper indexes for timestamp queries
-4. **Connection Pooling**: Configure appropriate database connection limits
-
-This enhanced scheduler ensures reliable, efficient, and intelligent water level monitoring with automatic adaptation to data availability patterns.

docs/ENHANCEMENT_SUMMARY.md

@@ -1,227 +0,0 @@
-# 🚀 Northern Thailand Ping River Monitor - Enhancement Summary
-
-## 🎯 **What We've Accomplished**
-
-We've successfully transformed your water monitoring system from a simple scraper into a **production-ready, enterprise-grade monitoring platform** focused on the Ping River Basin in Northern Thailand, with modern web interfaces, station management capabilities, and comprehensive observability.
-
-## 🌟 **Major New Features Added**
-
-### 1. **FastAPI Web Interface** 🌐
-- **Interactive Dashboard** at `http://localhost:8000`
-- **REST API** with comprehensive endpoints
-- **Station Management** - Add, update, delete monitoring stations
-- **Real-time Health Monitoring** 
-- **Manual Data Collection Triggers**
-- **Interactive API Documentation** at `/docs`
-- **CORS Support** for web applications
-
-### 2. **Enhanced Architecture** 🏗️
-- **Type Safety** with Pydantic models and comprehensive type hints
-- **Data Validation Layer** with range checking and error handling
-- **Custom Exception Classes** for better error management
-- **Modular Design** with separated concerns
-
-### 3. **Observability & Monitoring** 📊
-- **Metrics Collection System** (counters, gauges, histograms)
-- **Health Checks** for database, API, and system resources
-- **Performance Tracking** with response times and success rates
-- **Enhanced Logging** with colors, rotation, and performance logs
-
-### 4. **Production Features** 🚀
-- **Rate Limiting** to prevent API abuse
-- **Request Tracking** with detailed statistics
-- **Configuration Validation** on startup
-- **Graceful Error Handling** and recovery
-- **Background Task Management**
-
-## 📁 **New Files Created**
-
-```
-src/
-├── models.py              # Data models and type definitions
-├── exceptions.py          # Custom exception classes
-├── validators.py          # Data validation layer
-├── metrics.py            # Metrics collection system
-├── health_check.py       # Health monitoring system
-├── rate_limiter.py       # Rate limiting and request tracking
-├── logging_config.py     # Enhanced logging configuration
-├── web_api.py           # FastAPI web interface
-├── main.py              # Enhanced CLI with multiple modes
-└── __init__.py          # Package initialization
-
-# Root files
-├── run.py                # Simple startup script
-├── test_integration.py   # Integration test suite
-├── test_api.py          # API endpoint tests
-└── ENHANCEMENT_SUMMARY.md # This file
-```
-
-## 🔧 **Enhanced Existing Files**
-
-- **`src/water_scraper_v3.py`** - Integrated new features, metrics, validation
-- **`src/config.py`** - Added configuration validation
-- **`requirements.txt`** - Added FastAPI, Pydantic, and monitoring dependencies
-- **`docker-compose.victoriametrics.yml`** - Added web API service
-- **`Dockerfile`** - Updated for new startup script
-- **`README.md`** - Updated with new features and usage instructions
-
-## 🌐 **Web API Endpoints**
-
-| Endpoint | Method | Description |
-|----------|--------|-------------|
-| `/` | GET | Interactive dashboard |
-| `/docs` | GET | API documentation |
-| `/health` | GET | System health status |
-| `/metrics` | GET | Application metrics |
-| `/stations` | GET | List all monitoring stations |
-| `/measurements/latest` | GET | Latest measurements |
-| `/measurements/station/{code}` | GET | Station-specific data |
-| `/scrape/trigger` | POST | Trigger manual data collection |
-| `/scraping/status` | GET | Scraping status and statistics |
-| `/config` | GET | Current configuration (masked) |
-
-## 🚀 **Usage Examples**
-
-### **Traditional Mode (Enhanced)**
-```bash
-# Test single cycle
-python run.py --test
-
-# Continuous monitoring
-python run.py
-
-# Fill data gaps
-python run.py --fill-gaps 7
-
-# Show system status
-python run.py --status
-```
-
-### **Web API Mode (NEW!)**
-```bash
-# Start web API server
-python run.py --web-api
-
-# Access dashboard
-open http://localhost:8000
-
-# View API documentation
-open http://localhost:8000/docs
-```
-
-### **Docker Deployment**
-```bash
-# Start complete stack
-docker-compose -f docker-compose.victoriametrics.yml up -d
-
-# Services available:
-# - Water API: http://localhost:8000
-# - Grafana: http://localhost:3000
-# - VictoriaMetrics: http://localhost:8428
-```
-
-## 📊 **Monitoring & Observability**
-
-### **Built-in Metrics**
-- API request counts and response times
-- Database connection status and save operations
-- Scraping cycle success/failure rates
-- System resource usage (memory, etc.)
-
-### **Health Checks**
-- Database connectivity and data freshness
-- External API availability
-- Memory usage monitoring
-- Overall system health status
-
-### **Enhanced Logging**
-- Colored console output for better readability
-- File rotation to prevent disk space issues
-- Performance logging for optimization
-- Structured logging with proper levels
-
-## 🔒 **Production Ready Features**
-
-### **Security & Reliability**
-- Rate limiting to prevent API abuse
-- Input validation and sanitization
-- Graceful error handling and recovery
-- Configuration validation on startup
-
-### **Performance**
-- Efficient metrics collection with minimal overhead
-- Background task management
-- Connection pooling and resource management
-- Optimized database operations
-
-### **Scalability**
-- Modular architecture for easy extension
-- Async support for high concurrency
-- Configurable resource limits
-- Health checks for load balancer integration
-
-## 🧪 **Testing**
-
-### **Integration Tests**
-```bash
-# Run all integration tests
-python test_integration.py
-```
-
-### **API Tests**
-```bash
-# Test API endpoints (server must be running)
-python test_api.py
-```
-
-## 📈 **Performance Improvements**
-
-1. **Request Tracking** - Monitor API performance and success rates
-2. **Rate Limiting** - Prevent API abuse and ensure stability
-3. **Data Validation** - Catch errors early and improve data quality
-4. **Metrics Collection** - Identify bottlenecks and optimization opportunities
-5. **Health Monitoring** - Proactive issue detection and alerting
-
-## 🎉 **Benefits Achieved**
-
-### **For Developers**
-- **Better Developer Experience** with type hints and validation
-- **Easier Debugging** with enhanced logging and error messages
-- **Comprehensive Testing** with integration and API tests
-- **Modern Architecture** following best practices
-
-### **For Operations**
-- **Web Dashboard** for easy monitoring and management
-- **Health Checks** for automated monitoring integration
-- **Metrics Collection** for performance analysis
-- **Production-Ready** deployment with Docker support
-
-### **For Users**
-- **REST API** for integration with other systems
-- **Real-time Data Access** via web interface
-- **Manual Controls** for triggering data collection
-- **Status Monitoring** for system visibility
-
-## 🔮 **Future Enhancement Opportunities**
-
-1. **Authentication & Authorization** - Add user management and API keys
-2. **Real-time WebSocket Updates** - Live data streaming to web clients
-3. **Advanced Analytics** - Trend analysis and forecasting
-4. **Alert System** - Email/SMS notifications for critical conditions
-5. **Multi-tenant Support** - Support for multiple organizations
-6. **Data Export** - CSV, Excel, and other format exports
-7. **Mobile App** - React Native or Flutter mobile interface
-
-## 🏆 **Summary**
-
-Your Thailand Water Monitor has been transformed from a simple data scraper into a **comprehensive, enterprise-grade monitoring platform** that includes:
-
-- ✅ **Modern Web Interface** with FastAPI
-- ✅ **Production-Ready Architecture** with proper error handling
-- ✅ **Comprehensive Monitoring** with metrics and health checks
-- ✅ **Type Safety** and data validation
-- ✅ **Enhanced Logging** and observability
-- ✅ **Docker Support** for easy deployment
-- ✅ **Extensive Testing** for reliability
-
-The system is now ready for production deployment and can serve as a foundation for further enhancements and integrations!

docs/FLOOD_FORECASTING.md

@@ -0,0 +1,930 @@
+# Flood forecasting
+
+Short-range flood-risk forecasts for the Ping River gauge network, trained on the
+monitor's own PostgreSQL history. This document covers what the system predicts,
+what it is built from, how well it actually performs, how to run it on the server,
+and when to retrain.
+
+Code lives in `src/ml/` (`data.py`, `features.py`, `train.py`, `predict.py`), the
+training entry point is `scripts/train_flood_model.py`, tests are in
+`tests/test_flood_forecast.py`, and trained artifacts land in `models/`.
+
+## 1. Overview
+
+For every station the system answers three questions at three lead times (6, 12
+and 24 hours):
+
+- **`p_warning`** — probability the water level reaches or exceeds the warning
+  threshold (3.0 m) at any point within the horizon.
+- **`p_danger`** — same for the danger threshold (4.5 m).
+- **`predicted_max_level`** — the expected peak level within the horizon, in
+  metres on the station's own datum.
+
+The window is open-ended forward: `exceed_warn_6` at 09:00 asks whether the level
+touches 3.0 m anywhere in (09:00, 15:00], not what it will be at 15:00 — the
+question an operator actually has.
+
+Fifteen of the sixteen stations have trained models. P.4A (Ban Mae Taeng) is
+excluded by `features.NOT_TRAINABLE` (17.2% hourly fill, effectively dead
+2019–2024: 289 rows in 2019, 769 in 2024) and is served by the persistence
+heuristic instead. It still feeds downstream stations as an *input*, where
+HistGradientBoosting's native NaN handling copes with the gaps.
+
+Every forecast row carries `source` (`model` or `heuristic`), `model_version`
+and `trained_at`, so a stale or degraded forecast is visible in the payload
+rather than silently indistinguishable from a good one.
+
+## 2. Data
+
+**Source of record.** PostgreSQL table `water_measurements` joined to `stations`.
+As verified on 2026-08-10 (`inventory.json`, `db_cross_check`): **592,240 rows,
+16 stations, 2018-08-01 through 2026-08-10**, zero mismatches against the HTTP
+API, and a `status` column that is uniformly `active` (there are no quality flags
+to filter on — bad readings must be caught by the feature pipeline, not by the
+database).
+
+**Coverage is the dominant data constraint.** Readings are nominally hourly
+(modal interval 1 h, ~95% of gaps), but only about **56% of hours on the complete
+hourly grid have a reading**: P.1, P.67, P.76 and P.84 at 56.1%, P.103 at 55.8%,
+P.21 at 55.4%, P.20 at 53.7%, P.87 at 53.3%, P.5 at 50.7%, P.4A at 17.2%.
+
+The missingness is **systematic, not random**. Measured over the 587 days from
+2025-01-01 in `models/cache/P.1.csv.gz`, the fraction of days with a reading at
+each hour is roughly 0.80 for 01:00–12:00, 0.68–0.69 for 13:00–16:00, 0.46–0.49
+for 17:00–21:00, 0.40–0.42 for 22:00–23:00, and 0.32 at midnight. That is a
+scrape-schedule fingerprint, not hydrology — and it is why hour-of-day is
+deliberately *not* a feature (see section 3).
+
+Long outages would poison training if used naively: P.87 lost 3,961 hours
+(165 days) in 2023, P.20 lost 2,681 hours in 2021, P.77 lost 2,522 hours in early
+2022, P.5 lost 2,429 hours over the 2020–21 turn. `features.TRAIN_START` excludes
+P.5 before 2022-01-01; the rest are handled by the per-row coverage gates.
+
+**How `src/ml/data.py` loads it.** `resolve_db_url()` picks a connection string in
+priority order: an explicit `--db-url` argument, then the `FLOOD_ML_DB_URL`
+environment variable, then the `POSTGRES_*` settings (`src/config.py`, the same
+`.env` the service reads), else `None`. `load_measurements()` then tries three tiers:
+
+1. **PostgreSQL** (`_fetch_from_db`) — the primary path. NULL discharge stays
+   NULL, which matters because the models must learn from the real missingness
+   pattern.
+2. **HTTP API** (`_fetch_from_api`, default `https://water.buildfor.life`) — a
+   fallback for running off-server. **Caveat:** the public history endpoint
+   backfills missing discharge with a synthetic rating-curve estimate, so this
+   path is not equivalent to the DB path. It is flagged as
+   `discharge_maybe_synthetic: true` in the cache metadata.
+3. **On-disk cache** (`models/cache/{station}.csv.gz` plus `meta.json`) — last
+   resort only. A successful DB or API fetch refreshes the cache; the cache is
+   never treated as a source of fresh data.
+
+`load_latest()` (used by the API) pulls the trailing 336 hours and never writes
+the cache.
+
+## 3. Physics and features
+
+### Upstream routing
+
+The Ping mainstem gives real forecast skill for free: a flood wave takes hours to
+travel downstream, so an upstream gauge reading *now* is information about a
+downstream gauge *later*. `data-scout` measured these travel times by
+cross-correlating water-level anomalies against the basin anchor P.1. The peak
+correlation lags, which are hard-coded in `features.UPSTREAM_LEADS`:
+
+| Station | Lead vs P.1 | Peak anomaly correlation | Distance to P.1 (km) |
+|---|---|---|---|
+| P.20 (Ban Chiang Dao) | 17 h | 0.59 | 84.5 |
+| P.92 (Ban Muang Aut) | 15 h | 0.66 | 63.8 |
+| P.75 (Ban Chai Lat) | 12 h | 0.61 | 45.0 |
+| P.4A (Ban Mae Taeng) | 12 h | 0.73 | 37.9 |
+| P.67 (Ban Tae) | 7 h | 0.74 | 25.3 |
+| P.21 (Ban Rim Tai) | 9 h | 0.56 | 15.0 |
+| P.103 (Ring Bridge 3) | 1 h | 0.88 | 9.3 |
+
+(Distances are cumulative straight-line gauge-to-gauge, from the inventory's
+`spatial_order_north_to_south`, not channel length — the real river is longer.)
+
+The lags are broadly consistent with distance, with one exception worth knowing
+about: P.21 is 10 km closer to P.1 than P.67 yet lags by 9 h rather than 7 h, and
+it has the weakest correlation of the mainstem set (0.56). Whatever the cause,
+the table encodes the measured lag rather than the one distance would predict —
+which is the point of measuring instead of assuming.
+
+Two stations are *downstream* of P.1 (P.5 at −12 h, P.81 at −4 h). For those,
+`UPSTREAM_LEADS` routes P.1 and P.103 forward as their inputs, which is the same
+physics running the other direction.
+
+Six western-tributary stations — P.82, P.84, P.87, P.77, P.85, P.76 — have empty
+`UPSTREAM_LEADS` and are **un-routed**. Their anomaly correlations with P.1 are
+0.22–0.32, low enough that routing them would inject noise rather than signal.
+They are forecast from their own history plus the P.1 basin-state features. This
+is a known gap: those catchments have no upstream gauge of their own in this
+network.
+
+### Feature set (`features.build_features`)
+
+Everything is computed on an hourly grid built by `make_hourly_grid()`, which
+keeps three aligned frames: `observed` (raw, NaN where nothing was recorded),
+`filled` (forward-filled with `FFILL_LIMIT_H = 3`), and `mask` (True where a real
+reading exists). Features read `filled`; labels read `observed` only.
+
+Per target station:
+
+- **Self level**: current level, lags at 1/2/3/6/12/24/48/72 h.
+- **Rate of rise**: level minus its own value 1/3/6/12/24 h ago — a river at
+  2.5 m and falling is a different situation from one at 2.5 m rising 30 cm/h.
+- **Rolling statistics**: 6/24 h means, 6/24/72 h maxima, 24 h minimum.
+- **Discharge**: current, lags at 6/24 h, 6 h rise (read from `observed`, so NULL
+  discharge stays NULL).
+- **Observation health**: `obs_age_h` (hours since the last real reading, capped
+  at `FFILL_LIMIT_H`) and `cov_24h` (fraction of the last 24 h actually observed),
+  so the model can learn to hedge when a gauge is going quiet.
+- **Routed upstream**, per `(upstream, lead)` pair: the upstream level at
+  `lead−3`, `lead`, and `lead+3` hours ago, its 6 h rise at `lead`, and its 24 h
+  rolling max at `lead−3`. The three-point bracket absorbs error in the measured
+  travel time rather than depending on it being exact.
+- **Basin state** (non-P.1 stations only): P.1 level, its 24 h rolling max, and
+  its 24 h rise.
+- **Seasonality**: `doy_sin`, `doy_cos` and an `is_monsoon` flag for June–October.
+
+**Hour-of-day is deliberately excluded.** Given the availability profile in
+section 2, an hour-of-day feature would let the model learn "readings at 03:00
+are more likely to exist" and route that through to the label — an artefact of
+when the scraper runs, with no hydrological content, that would evaporate the
+moment the scrape schedule changed.
+
+### No-leakage guarantees
+
+- Only `shift()`, backward `rolling()` and forward-fill are used — nothing
+  interpolates, and no row can read a value timestamped after itself.
+- `test_no_future_leakage` enforces this empirically: it adds +50 m to every
+  reading after time *t*, rebuilds the features, and asserts the rows at or
+  before *t* are bit-identical.
+- Labels come from `observed`, never `filled`, so a forward-filled value can
+  never become its own target.
+- The split is strictly temporal, and `early_stopping` is disabled in
+  `HGB_PARAMS` specifically because scikit-learn's internal validation split is
+  random and would leak across time.
+
+### Coverage gating
+
+A label is only trusted if enough of its forward window was actually observed.
+`build_labels` requires `MIN_WINDOW_COVERAGE = 0.5` — at least half the horizon's
+hours present — otherwise the label is NaN and the row is dropped from that head's
+training set. The one exception is deliberate: **an observed exceedance always
+produces a positive label regardless of coverage**, because a confirmed 3.5 m
+reading inside a sparse window is not ambiguous. Rows whose own features are
+stale (`obs_age_h` is NaN, i.e. the last real reading is more than 3 h old) are
+dropped entirely in `build_matrix`.
+
+## 4. Models
+
+### Architecture
+
+One `HistGradientBoosting` model per **station × horizon × head**:
+
+| Head | Type | Target |
+|---|---|---|
+| `max_{h}` | `HistGradientBoostingRegressor` (squared error) | *rise*: max observed level in (t, t+h] minus level at t (v2; serving adds the level back) |
+| `warn_{h}` | `HistGradientBoostingClassifier` | level ≥ 3.0 m anywhere in (t, t+h] |
+| `danger_{h}` | `HistGradientBoostingClassifier` | level ≥ 4.5 m anywhere in (t, t+h] |
+
+Nine heads per station, three horizons (6/12/24 h), fifteen trained stations.
+Hyperparameters are fixed (`HGB_PARAMS`: 300 iterations, learning rate 0.06, 31
+leaf nodes, minimum 50 samples per leaf, L2 1.0, `random_state=42`), chosen in an
+earlier sweep and not re-searched per run — training is deterministic and
+repeatable.
+
+HistGradientBoosting was chosen for three concrete reasons: it handles NaN
+natively (essential given ~44% missing hours), it needs no feature scaling, and
+it trains on CPU alone — no GPU anywhere in this pipeline (measured cost in
+section 6).
+
+### Head gating and fallbacks
+
+The system degrades in tiers rather than failing:
+
+1. **Belt-and-braces probability** *(since 2026-08-11 — see the re-examination
+   note in section 7)*: the sigmoid-of-regression probability
+   `p = 1/(1 + exp(−(predicted_max − threshold)/σ))` is always computed (σ =
+   the regressor's test-residual std, floor `MIN_SIGMA = 0.15` m), and when a
+   classifier head exists — trained only if the span had at least
+   `MIN_POSITIVES_FOR_CLASSIFIER = 30` positives — the served probability is
+   `max(classifier, sigmoid)`. The classifier can raise the alarm but never
+   silence it: on the gap-filled data a trained classifier stayed near zero
+   through the 2024 record crossing while the regression tracked it.
+2. **Sigmoid only**, when the classifier head is absent or skipped
+   (recorded in `skipped_heads` with its reason).
+3. **Persistence heuristic** (`predict._heuristic_forecast`), when there is no
+   model file at all, or the station's newest reading is more than
+   `STALE_AFTER_H = 6` hours old. It extrapolates the last 3 h rate of rise
+   forward with a 0.7 damping factor and a fixed σ of 0.3 m. It is not skilful; it
+   exists so the endpoint always returns something structurally valid.
+
+A station is skipped entirely if it has fewer than `MIN_ROWS_TO_TRAIN = 200`
+usable rows; an individual head is skipped below `MIN_ROWS_FOR_HEAD = 50` labeled
+rows. `_safe_fit` converts any fit failure (typically HistGradientBoosting's
+binning step rejecting an all-NaN or constant column) into a recorded skip rather
+than a station-killing exception.
+
+### Training procedure
+
+`train_station` runs two passes. First it evaluates on the strict temporal
+holdout (train ≤ 2024-12-31, test 2025-01-01 → 2026-08-10) to produce the metrics
+and the σ calibration. Then it **refits every head on the entire record** for the
+deployed artifact, so the shipped model has seen the most recent data. Because
+the full record has more labeled rows than the training half, the head-gating
+decisions can differ between the two passes — `skipped_heads` is therefore
+re-derived during the refit so it always describes what is actually in the saved
+bundle, not what the evaluation pass decided.
+
+### Bundle format
+
+`models/flood_{station}.joblib` contains: `station_code`, `model_version`
+(`hgb-v1+<git short SHA>`), `trained_at`, `sklearn_version`, `feature_names`,
+`horizons`, `thresholds`, `heads`, `sigma`, `skipped_heads`, `train_span`, and
+`n_train_rows`.
+
+`feature_names` is the important one. At prediction time `_model_forecast`
+rebuilds the feature row from live data and checks it against the bundle's stored
+list; if any expected column is missing it logs an error and falls back to the
+heuristic rather than feeding scikit-learn silently misaligned columns.
+`test_feature_name_stability` guards the same invariant at build time. Bundles are
+cached in memory keyed by `(path, mtime)`, so dropping in a retrained file
+invalidates the cache without a restart.
+
+## 5. Measured performance
+
+### Holdout metrics (`models/metrics.json`)
+
+Two evaluations exist and they differ sharply — the re-examination note in the
+next section explains why (the hourly grid was gap-filled from ~56% to ~93%
+between them, roughly doubling the test rows and collapsing the warning base
+rates).
+
+**Current model** `hgb-v3` (rise target + Open-Meteo rain features),
+generated 2026-08-12 on the gap-filled DB (~976k rows). Train ≤ 2024-12-31,
+test 2025-01-01 → 2026-08-12. P.1:
+
+| Horizon | Warning PR-AUC | MAE | MAE above 2 m | Test rows | Base rate |
+|---|---|---|---|---|---|
+| 6 h | 0.783 | 4.9 cm | 5.0 cm | 14,034 | 0.12% |
+| 12 h | 0.508 | 7.2 cm | 10.7 cm | 14,028 | 0.16% |
+| 24 h | 0.288 | 8.7 cm | 18.1 cm | 14,034 | 0.25% |
+
+(Progression across the same day's runs — v1 absolute target:
+5.5/8.1/10.5 cm MAE; v2 rise: 5.0/7.2/9.4; v3 rise+rain: 4.9/7.2/8.7 —
+with above-2 m MAE falling 24.0 → 20.0 → 18.1 cm at 24 h. PR-AUC belongs to
+the unchanged classifier heads; serving is belt-and-braces so alerting uses
+the improved regression path regardless.)
+
+Level accuracy improved; standalone classifier discrimination did not survive
+the data change (which is why serving is now `max(classifier, sigmoid)` — see
+"Head gating"). Recall-at-FAR is null at all horizons on this run. Across
+stations the 6 h warning PR-AUC now spans 0.987 (P.77) / 0.982 (P.5) / 0.956
+(P.85) / 0.950 (P.67) down to 0.436 (P.84), and danger heads are now evaluable
+at nine stations — strongest P.5 (0.958/0.883/0.811 at 6/12/24 h) and P.77
+(0.942/0.863/0.786); P.103's danger metrics, previously the highlight, are null
+on this span.
+
+**Historical evaluation** (`hgb-v1+49a3de0`, 2026-08-10, pre-gap-fill DB —
+kept for the record; these numbers described the sparser 56%-filled grid and do
+not reproduce on today's data):
+
+| Horizon | Warning PR-AUC | Recall @1% FAR | MAE | Test rows | Base rate |
+|---|---|---|---|---|---|
+| 6 h | 0.974 | 98.3% | 6.1 cm | 8,536 | 1.36% |
+| 12 h | 0.904 | 93.8% | 9.0 cm | 7,932 | 1.61% |
+| 24 h | 0.900 | 90.1% | 11.3 cm | 8,572 | 1.77% |
+
+The dramatic PR-AUC difference is mostly the base rate: the filled grid adds
+~5,500 quiet test hours per horizon while the number of positive hours barely
+changes, so the same ranking quality scores far lower — and the classifier's
+genuine out-of-distribution weakness (see the backtest sections) does the rest.
+
+### 2026-08-11 re-examination: fuller data changes the backtest story
+
+> **Read this before the two backtest sections below.** On 2026-08-11 the
+> backtests were codified into `scripts/backtest_render.py` (previously they
+> were one-off runs) and re-run after the database grew from 592k to ~976k
+> rows (a `--fill-gaps all` pass repaired most of the missing 44% of the
+> hourly grid). Three things changed:
+>
+> 1. **The 2024 crossing was 8 hours earlier than documented.** The recovered
+>    hours show P.1 crossing 3.70 m at **17:00 on 24 September 2024**, not
+>    01:00 on 25 September — confirmed independently by the HII sensor at
+>    Nawarat Bridge (hii_waterlevel, station 3226: 3.73 m at 17:00). The
+>    originally celebrated "24-hour warning" was therefore ~16 hours measured
+>    against the real river.
+> 2. **Retraining on the fuller data improves level accuracy but degrades the
+>    warning classifiers.** P.1 24 h MAE improved (11.3 → 10.5 cm), but the
+>    warning-head PR-AUC collapsed (0.900 → 0.288 at 24 h): with the filled
+>    grid the classifier trains on many more dry-season rows and now stays
+>    silent through the September 2024 record crossing while the regression
+>    head tracks it. Serving was changed to belt-and-braces —
+>    `max(classifier, sigmoid(regression))` — so alerting can never be worse
+>    than the regression path.
+> 3. **Honest current lead times, from the regenerated charts below:** the
+>    retrained configuration first alerts ~18 h *after* the true 24 Sep 2024
+>    crossing and roughly *at* the 27 Sep 2025 crossing. The earlier, better
+>    numbers came from models trained and evaluated on the sparser data. The
+>    conclusion is not that the old system was better — it is that gauge-only
+>    features fundamentally lack lead time for fast rises, which is exactly
+>    the rainfall-input and rise-target work now queued (see "Honest limits").
+>
+> `scripts/backtest_render.py` regenerates all three charts and fails its
+> acceptance gate while the 2024 lead stays under 12 h — keeping this page
+> honest is now automatic.
+>
+> **2026-08-12 follow-up — hgb-v2 (rise target).** A rolling-origin,
+> event-aware evaluation (`scripts/evaluate_variants.py`, one fold per monsoon
+> 2021-2025) compared the absolute-level target against rise-target variants.
+> The rise target — regression predicts *future max minus current level*, the
+> level is added back at serving — won decisively and is now deployed as
+> `hgb-v2`: the regenerated charts below show the 2024 first alert moving from
+> 18 h late to **6 h early** (11:00 vs the 17:00 crossing), the 2025 alert
+> from at-crossing to **45 h early**, the record-peak underprediction
+> eliminated (the model now slightly overshoots 5.30 m rather than capping
+> ~0.4 m below it), and P.1 MAE improving ~11% at every horizon. Weighted and
+> quantile variants were evaluated and rejected (more false alarms, no
+> calibration gain by Brier score). The ≥12 h acceptance gate still fails at
+> +6 h for 2024 — genuine further lead needs rainfall inputs, not modelling.
+>
+> **2026-08-12 follow-up 2 — hgb-v3 (rain features): the gate passes.**
+> Open-Meteo catchment rainfall (five upper-Ping points, forecast-model
+> archive 2021+, `src/ml/rain.py`) added four features: trailing 6/24/72 h
+> rain sums and `rain_fc24`, the forward-24 h forecast sum — the first input
+> that can act before water reaches any gauge. On the rolling-origin harness
+> (`models/eval_rain.json`) rain roughly halved flood-year Brier scores, cut
+> flood-regime MAE 20–40%, and moved the hard 2024 leads from +6 h to +11 h
+> (P.1) and +10 to +19 h (P.103); the marginal 2025 double-crest event trades
+> its artifact +46 h "lead" for a calibrated +2 h with zero false alarms. The
+> regenerated backtest below now shows a **13-hour warning for the 2024
+> record flood (alert 04:00, crossing 17:00) — the ≥12 h acceptance gate
+> passes for the first time**. P.1 MAE improves again to 4.9/7.2/8.7 cm at
+> 6/12/24 h. Serving fetches live rain hourly and degrades to NaN features
+> (never a crash) if Open-Meteo is unreachable.
+
+### The September 2025 flood, as the deployed configuration saw it
+
+![Observed vs predicted through the September 2025 flood — model trained only
+through 2024](img/backtest-2025-p1.png)
+
+This uses the deployed configuration (train ≤ 2024-12-31) on an event it never
+saw. *(Chart regenerated 2026-08-12 with the hgb-v2 rise target on the
+gap-filled data — see the re-examination note above for the history of these
+numbers.)* The v3 model first alerts at **16:00 on 27 September 2025 — 2 hours before
+the river crosses 3.70 m** at 18:00. This is a shorter lead than v2's 45 h,
+and deliberately so: v2's long "lead" was an alarm that latched through the
+near-miss 3.51 m crest of the 26th; v3's rain-informed probabilities are far
+better calibrated on this marginal event (Brier halved, zero false-alarm
+episodes on the season) and fire when exceedance actually becomes likely. A
+barely-over-threshold crest is intrinsically a short-notice event.
+
+### Headline validation: the October 2024 record flood
+
+The holdout above never sees a true extreme, because the 2024 flood is in the
+training half. So the model was retrained on data **ending 2024-08-31** and asked
+to forecast September–November 2024 cold, with no knowledge of the event that
+followed. This is the closest thing to a real operational test available.
+
+![Observed P.1 level vs the model's 24 h-ahead predicted peak through the
+October 2024 flood, with the warning probability below](img/backtest-2024-p1.png)
+
+The render above shows the whole event hour by hour *(regenerated 2026-08-12
+with the hgb-v3 rise + rain configuration)*. Top: the observed level (blue)
+against the 24 h-ahead predicted peak the model issued at each hour (amber,
+dashed) — the amber line leads the blue one into both flood waves. Bottom: the
+belt-and-braces probability of flooding within 24 h; the **first alert comes
+at 04:00 on 24 September, 13 hours before the true 17:00 crossing**, while
+the river in town still read 2.9 m — the rain features react to upstream
+precipitation before any gauge rises. The rise target removed the
+cannot-exceed-training-max ceiling, so the record 5.30 m peak is tracked
+rather than capped. The same historic model track drives the dashboard's
+"Replay Oct 2024 flood" feature.
+
+![Hour-by-hour detail of the detection window, 22–28 September
+2024](img/backtest-2024-p1-detail.png)
+
+The hour-by-hour detail of the detection window *(regenerated 2026-08-12,
+hgb-v3)* shows the sequence: the river crosses 3.70 m at **17:00 on
+24 September** (the hours recovered by gap-filling; independently confirmed by
+the HII sensor at the same bridge), and the model's probability crosses 0.5 at
+**04:00 — a 13-hour warning** delivered while the river stood at 2.9 m. The
+same alert under the absolute-level target came 18 hours *after* the crossing
+(v1), and 6 hours before it with the rise target alone (v2); catchment
+rainfall closed the rest. This clears the ≥12 h acceptance gate in
+`scripts/backtest_render.py`.
+
+The event bullets below quote the original (pre-gap-fill) evaluation of the
+deployed model and are kept for the historical record — see the re-examination
+note above for why the lead times no longer reproduce:
+
+- **25 September cold start.** P.1's first warning crossing of the episode was
+  alerted **24–26 hours ahead**. This is the genuinely impressive case: the river
+  was in normal state, and the alert came from upstream routing alone.
+- **5 October record peak** (P.1 5.30 m, P.103 9.93 m — the highest levels in the
+  eight-year record). Alerted **48 hours ahead**. Read this one carefully: the
+  river was already in sustained flood by then, so "48 hours" is the
+  `_first_alert_at` lookback window (`lookback_h = 48`) saturating, not a
+  measurement of true lead time. The model was correctly alarmed throughout;
+  the metric simply cannot express how much earlier than 48 h that started.
+- **P.103 danger head** over the same window: PR-AUC 0.98–0.99, recall at 1% FAR
+  87–95%. It called the danger-level crossings, not just the warning ones.
+- **8 November re-flood.** Caught **26–31 hours ahead** by the 12 and 24 h models
+  — a second, independent event in the same test window.
+- **P.103's 1 September "miss"** is a test-boundary artefact: the event begins in
+  the first hours of the test span, before the feature window has enough test-side
+  history to have produced a sustained alert. It is not a model failure, but it is
+  also not evidence of skill.
+
+### Honest limits
+
+**Genuine lead time is capped by gauge-only physics.** The longest upstream travel
+time into P.1 is 17 h (P.20), and the strongest predictors are much closer:
+P.103 at 1 h, P.67 at 7 h, P.21 at 9 h. Once a 24 h forecast reaches past roughly
+17 h, there is no observation that has "already happened" to inform it — the model
+is extrapolating basin state and season — unless it has rainfall. That is no
+longer hypothetical: hgb-v3's Open-Meteo features (see the re-examination
+notes in section 7) took the 2024 record-flood lead from 18 h late (v1
+gauge-only, absolute target) to 13 h early, precisely because catchment rain
+acts before any gauge rises, and `rain_fc24` — a weather *forecast* — acts
+before the rain itself falls. **Remaining honest limits:** marginal
+just-over-threshold crests (2025: +2 h) are intrinsically short-notice; the
+rain series only exists from 2021-03, so older training rows are rain-blind;
+forecast-rain quality bounds what the feature can add; and Mae Ngat reservoir
+state, though now ingested daily (see `docs/DATA_SOURCES.md`), measurably
+*hurts* alert lead as a model feature — see the 2026-08-13 experiment below.
+
+**Danger-level skill at P.1 is unproven.** P.1 never crossed 4.5 m in the
+2025-01-01 → 2026-08-10 test span (`base_rate_danger` is 0.0, so every danger
+metric is `null`). The danger head exists and is trained on the full record — the
+river has spent 57 hours above 4.5 m historically, 0.144% of all hours — but no
+out-of-sample number backs it. Treat `p_danger` at P.1 as indicative, not
+validated.
+
+**Thresholds are per-station as of 2026-08-10.** `THRESHOLDS` now carries
+calibrated (warning, danger) pairs for all 16 stations, derived from the DB's
+`discharge_percent` (RID % of channel capacity): warning = median level at
+75–85% capacity, danger = median level at 95–105%. P.1 instead uses the official
+Chiang Mai inundation map (`P1_FLOOD_STAGES`): warning 3.70 m (city flooding
+begins, stage 1) and danger 4.20 m (stage 5). The prior single default of
+(3.0, 4.5) m made P.103 badly over-alert (its bank-full level is ~6.75 m) and
+P.67 under-alert (overflow at ~2.9 m, 1.6 m below the old danger line).
+**A retrain is required after any threshold change** — classifier labels depend
+on them; until then, model rows report the thresholds baked into their bundle.
+P.1 additionally reports `stages`: exceedance probability for each of the seven
+official inundation stages (3.70–4.60 m), computed from the regression head and
+its calibration sigma, so they need no retrain and no per-stage classifiers.
+
+### 2026-08-13: Mae Ngat dam features — a documented negative result
+
+With `rid_reservoir_daily` backfilled to 2018 (daily Mae Ngat storage/inflow/
+outflow, `src/ml/dam.py`), the obvious v4 experiment was to feed reservoir
+state to the mainstem models: during the Oct 2024 flood the dam hit 113% of
+usable capacity with 19–22 MCM/day inflow spikes on the crossing days.
+
+**It fails the acceptance gate.** On the 2024 record-flood backtest (train
+< 1 Sep 2024, belt-and-braces alerting, identical to the deployed pipeline):
+
+| dam features               | first-alert lead | record-peak err (24 h ahead) |
+|----------------------------|------------------|------------------------------|
+| none (deployed v3 config)  | **+13 h** (PASS) | +0.24 m                      |
+| all four                   | +10 h (FAIL)     | +0.22 m                      |
+| storage % + 3-day delta    | +12 h            | +0.21…+0.27 m                |
+| inflow + outflow           | +10 h (FAIL)     | +0.35 m                      |
+| outflow only               | +12 h            | +0.20 m                      |
+
+Every subset costs 1–3 h of warning for at most a ~3 cm peak-error gain. The
+mechanism is the publication lag: RID posts the daily report on the morning of
+its own date (features apply it from 07:00, `dam.py`'s leakage rule), so at the
+04:00 first-alert hour of 24 Sep 2024 the freshest dam row still described
+23 Sep — a benign reservoir quietly absorbing inflow (outflow 0.13 MCM/day).
+The columns therefore argue *against* imminent flooding exactly when the rain
+features are (correctly) raising the alarm. The rolling-origin harness agrees:
+`rise_rain_dam` matches `rise_rain` on leads and false alarms, only nudging
+event-peak amplitude (−0.11 → −0.03 m on the Sep 2024 event), and `rise_dam`
+(dam without rain) is strictly worse with alarm-latch artifacts.
+
+**Disposition:** dam features are OFF by default (`train_all(use_dam=False)`;
+opt-in via `--dam` on the training CLI, `scripts/backtest_render.py --dam`,
+and the `rise_rain_dam` / `rise_dam` harness variants). The collector keeps
+accruing daily rows; revisit post-monsoon when the 2026 season adds dam-era
+flood events.
+
+**Why the lag is probably not the whole story — P.75 already *is* the dam
+signal.** A 2026-08-13 source sweep put the negative result on firmer
+ground: **P.75 "บ้านช่อแล" sits 3.8 km downstream of the Mae Ngat dam** on
+the Mae Ngat river (nearest other station: P.4A at 10.9 km), it reports
+hourly, and it has been a model input since v1 with a 12 h routed lead into
+P.1. Whatever the reservoir releases flows past P.75 within the hour and the
+model already reads it. The daily reservoir table therefore offers a stale,
+coarser proxy of a signal the features capture hourly and directly — which
+is the more likely reason it adds nothing and costs alarm responsiveness.
+
+That reframes what a future intraday source would have to beat: not "no dam
+information", but "hourly observed dam *outflow*". Genuine intraday
+reservoir-state feeds do exist and are open (`bigdata-api.rid.go.th` SWOC
+telemetry, and ThaiWater station `ridhydro_TUP.16` *at the dam*), but both
+are **snapshot-only — no archive** (verified: the history endpoint returns
+empty grids for them at every era, including the current one). They can only
+be accumulated forward, so they cannot retrain against 2024/2025 events.
+The HII collector already captures both hourly as of 2026-08-11; revisit
+after the 2026 monsoon, when a season of true intraday reservoir state
+exists alongside its flood events.
+
+**Shipped from the same work:** the HII gap-fill merge in the data loader
+(`fill_from_hii`, +9,341 h at P.81, +682 h at P.92, +810 h at P.20) is
+lead-neutral — the gate holds at 13 h with fill on — and ships enabled.
+
+### 2026-09-12: three candidates on top of hgb-v3 — two rejected, one deferred
+
+Same rolling-origin harness (`src/ml/evaluate.py`, five monsoon folds
+2021–2025, P.1 and P.103), all variants run from the identical
+`models/cache/` snapshot (`--from-cache`), results in
+`models/eval_2026-09-12*.json`, tables via `scripts/summarize_eval.py`.
+Baseline is `rise_rain`, the deployed configuration.
+
+**Quantile regression heads (`rise_rain_quantile`, `_uw`) — rejected.** The
+August result that quantile loss beat L2 on MAE held with rain in the model
+(P.1 0.083/0.081 vs 0.087; P.103 0.143/0.124 vs 0.152), and Brier improved a
+hair, but the operational numbers went the wrong way: at P.103 the 2022-08-14
+crossing dropped from +6 h to +1 h lead, 2022-10-02 from +9 h to +5/+3 h, and
+the 2024-09-30 event from +9 h to +4 h; at P.1 2022 dropped +5 → +3/+2 h and
+2025 +2 → +1 h, with one false-alarm episode where the baseline had none. A
+median predicts the *typical* rise, and on the run-up to a crossing the typical
+rise is not the one that matters. MAE is not the objective; lead is.
+
+**Quantile heads for sigma only (`rise_rain_qsigma`) — no effect.** The
+hybrid keeps the L2 point prediction (so every lead is identical to the
+baseline by construction — p≥0.5 alerts are sigma-independent) and derives a
+per-row sigma from q90−q50. Brier moved 0.0031 → 0.0029 at P.1 and
+0.0061 → 0.0060 at P.103, i.e. within noise, at the cost of three fitted
+heads per horizon instead of one. Per-row uncertainty from this family of
+models is not informative enough here to be worth the training time; the
+0.15 m floor stays.
+
+**Forward-48 h forecast rain (`rise_rain_fc48`) — deferred.** Adding the
+`(t, t+48]` Open-Meteo sum alongside `rain_fc24` left MAE, Brier and false
+alarms unchanged and every event lead within ±1 h of baseline, *except* the
+2024-10-03 P.1 record crossing, which went from +21 h to +72 h (and +55 → +69 h
+at P.103). That is one event with the highest stakes in the record, on the
+same feature family that already produced the 2024 gain, but n=1 is not
+evidence: the P.103 2025-09-26 event lost 2 h in the same run. Rerun after the
+2026 season adds events; if the 48 h window still moves only the biggest
+onsets, promote it. Serving would need no new data source (`fetch_forecast`
+already pulls `forecast_days=2`).
+
+**HII gauge rain — not evaluable yet.** `hii_rainfall` (~130 gauges in the
+upper-Ping box, DWR/FOP/HII/RID/TMD) is the obvious independent rain source,
+but the table only exists since 2026-08-11 and the api-v3 archive endpoint
+ignores its date range (see `docs/DATA_SOURCES.md` §2.1), so every training
+row before that is NaN and no fold in the harness has gauge data in its test
+span. `src/ml/hii_rain.py` builds the catchment mean and
+`GET /api/hii/rainfall/catchment` exposes it next to the Open-Meteo series with
+a 24 h-sum bias/MAE/correlation, so the two sources' relationship is on record
+by the time the 2027 fold (train ≤ 2027-04-30, test Jun–Nov 2027) can test it.
+
+### 2026-10-18: forecast rain was missing at serving time — fixed
+
+`build_features` cut the rain series to the gauge grid before taking the
+forward sum, and the grid ends at the newest reading. So the row serving
+predicts from always had `rain_fc24 = NaN`, although training and every
+evaluation above had the real value (the Open-Meteo forecast, already fetched
+with `forecast_days=2`). The model was answering as if no weather forecast
+existed. The forward sum now runs over the grid extended by 24 h; earlier rows
+are unchanged, so training is unaffected.
+
+Cost of the bug, measured with the harness: the `rise_rain_fc24_missing`
+variant is `rise_rain` with `rain_fc24` blanked on the test rows (results in
+`models/eval_2026-10-18_fc24.json`, five monsoon folds, `--from-cache`):
+
+| Station | Variant | Events | Warned early | Median lead | False alarms | MAE |
+|---|---|---|---|---|---|---|
+| P.1 | fixed (`rise_rain`) | 4 | 4 | 7.5 h | 0 | 0.087 |
+| P.1 | as served | 4 | 4 | 7.0 h | 0 | 0.087 |
+| P.103 | fixed | 7 | 7 | 9.0 h | 2 | 0.152 |
+| P.103 | as served | 7 | 7 | 9.0 h | 2 | 0.153 |
+| P.21 | fixed | 10 | 8 | 2.5 h | 5 | 0.215 |
+| P.21 | as served | 10 | 7 | 2.0 h | 10 | 0.232 |
+
+P.1 and P.103 lost little; P.21 doubled its false alarms. The two very long
+"as served" leads (P.21 2025-08-26 +69 h, P.103 2025-10-03 +35 h) are alarm
+episodes that happened to precede an event, not real early warnings. Since the
+fix, the served features match what this section evaluates.
+
+## 6. Deployment
+
+### Serving
+
+The forecast is computed by `ping-collector` (`rust/forecast`, a parity-tested
+port of `src/ml/predict.py`, which stays as the reference implementation) from
+`models/flood_<station>.json.gz`, which `train_all()` exports next to each
+joblib bundle. It runs after every collection cycle that brought newer
+readings and at least hourly, stores the rows in `forecast_history` (section 9's
+skill tracking) and `forecast_current`, and `GET /forecast` serves the stored
+row: one JSON row per station × horizon with the fields listed in section 1.
+A stored forecast older than `FORECAST_MAX_AGE_SECONDS` (2 h, i.e. the service
+has stopped forecasting) is still served, marked `X-Data-Stale: true`. See
+`rust/README.md`.
+
+A station without a model bundle, or whose bundle needs a feature the Rust side
+cannot build, gets the heuristic, as in Python. So **before the first training
+run `/forecast` returns 200 with an all-heuristic payload**, provided there is
+recent gauge data; you get an empty `200 []` only when there is no recent data
+at all. Judge deployment state by the `source` field, not the status code.
+
+The Python serving path (`predict.get_latest_forecasts`) is not called by the
+service any more; `scripts/forecast_parity.py` runs both on the live database
+and `scripts/install.sh` does that on every install.
+
+### Dashboard
+
+The "Flood risk outlook" panel (`src/static/dashboard.html`, `loadForecasts()`)
+loads non-blocking after the map renders and **stays hidden unless `/forecast`
+returns a non-empty array** — a non-OK response, an empty array, or a thrown
+fetch all just leave the panel hidden, and the rest of the dashboard is
+unaffected. Per the note above, this means the panel appears with heuristic-only
+content once data is flowing but before any model is trained; the per-chip
+tooltip is what tells you so. Stations are sorted worst-risk first, each showing
+three chips (6/12/24 h) coloured by risk band, with the tooltip carrying the exact
+warning and danger percentages, the predicted peak level, and a "heuristic
+fallback" note when the row did not come from a model. The panel is labelled
+*experimental*.
+
+### Training on the server
+
+The server already has the PostgreSQL connection configured, so no host override
+is needed:
+
+```bash
+cd /opt/thailand-water-monitor
+uv run python scripts/train_flood_model.py --stations all
+```
+
+`resolve_db_url()` picks up the `POSTGRES_*` settings from `.env`. The run writes
+fifteen `models/flood_{station}.joblib` bundles, their `.json.gz` exports for the
+service, and `models/metrics.json`. The monthly timer does the same through
+`scripts/retrain.sh` with a staged promote (section 7).
+
+### Artifacts and dependencies
+
+The fifteen bundles total **101.4 MB** — mean 6.76 MB, from 4.04 MB (P.20) to
+9.35 MB (P.103 and P.87, with P.5 next at 8.99 MB) — plus `metrics.json` at
+0.34 MB and a 2.1 MB `models/cache/`. **These are not in
+git**, and they should stay that way — artifacts are produced on the server, not
+shipped. `.gitignore` excludes `models/*.joblib`, `models/cache/` and
+`models/metrics.json` for exactly this reason.
+
+Two pins matter and are already in `requirements.txt` / `pyproject.toml`:
+`scikit-learn==1.9.0` and `numpy>=1.24,<2` (pandas 2.0.3 wheels are ABI
+incompatible with numpy 2.x). Bundles record `sklearn_version`; unpickling a
+bundle under a different scikit-learn version is not guaranteed to work, so
+retrain after any scikit-learn upgrade rather than assuming the artifacts carry
+over.
+
+### Measured resource use
+
+All figures below were measured on 2026-08-10 on a development workstation —
+**24 physical / 32 logical cores at 2.20 GHz, 32 GiB RAM** (Python 3.11.9,
+scikit-learn 1.9.0, joblib 1.5.3, numpy 1.26.4, pandas 2.0.3) — **not** on the
+production server. They come from two independent benchmark runs on that same
+machine, which is why a couple of figures below are quoted as narrow ranges.
+Treat the CPU times as a floor and the memory figures as representative, since
+RSS barely depends on core count. Training read the `models/cache/` csv.gz files
+(592,240 rows load in 0.5 s); loading the same history from PostgreSQL was not
+measured and will be slower.
+
+**Training** (`train_all`, all 15 stations, evaluation pass plus full refit):
+
+| Measurement | Value |
+|---|---|
+| Full 15-station run, unrestricted threads | **199 s (3.3 min)** |
+| Peak RSS during the full run | **209 MB** |
+| Single station, unrestricted (P.1 / P.103) | 16.5 s / 18.7 s |
+
+HistGradientBoosting threads through OpenMP, and it scales only modestly. Timing
+P.1 alone under `OMP_NUM_THREADS`:
+
+| Threads | 1 | 2 | 4 | unrestricted (32) |
+|---|---|---|---|---|
+| P.1 train time | 36.5 s | 23.5 s | 14.7 s | 16.5 s |
+
+Two things follow. **Four threads is the sweet spot** — 32 threads was marginally
+*slower* than 4, so oversubscription costs you a little. And **even one core is
+enough**: at 36.5 s per station, a single-core box retrains all fifteen in roughly
+9 minutes (extrapolated, not measured end-to-end).
+
+Per station the fit costs **8–17 s**, and P.1 is the worst case at 16.9 s — it is
+the basin anchor, so it carries 64 features against 32 for stations with fewer
+upstream inputs (P.85 9.6 s, P.20 8.2 s). Two things are *not* the cost driver.
+Evaluation isn't: P.1 with `skip_eval=True` took 17.3 s, no faster than the full
+path. Nor is feature engineering — `build_matrix` over P.1's whole 8-year history
+is 256 ms against 8–17 s of fitting. **The fit is the cost.**
+
+One honest caveat about the run that produced the current artifacts. By file
+mtime it wrote all fifteen models between 11:55:29 and 12:01:15 — **5 min 46 s**,
+averaging 25 s/station including joblib serialization, which lines up with the
+measured fits. But `models/cache/meta.json` records the data fetch finishing at
+11:45:49, so end to end that run spanned about 15.5 minutes, and the 9 min 40 s
+gap between fetch and first model could not be reconstructed from the surviving
+artifacts. Do not attribute it to per-station training cost. Either way the
+conclusion holds: **retraining is minutes, not tens of minutes.**
+
+**Inference in Python** (15 bundles, 16 stations × 3 horizons = 48 rows; the
+retired in-process serving path, measured before the Rust forecaster took
+over: there the whole run takes milliseconds, 0.25 ms per station against
+158 ms, and the 15 exported models need about 70 MB resident):
+
+| Measurement | Value |
+|---|---|
+| Cold call — every bundle unpickled from disk | **6.7 s** |
+| Warm call — bundles in `_MODEL_CACHE` | **0.72 s** median (0.63–0.84 s) |
+| RSS after imports, before any model | 71 MB |
+| RSS with all 15 bundles resident | **288 MB** |
+
+The 101.4 MB of on-disk pickles expand to roughly **203–211 MB resident** — about
+2× — and they stay there: `_MODEL_CACHE` replaces an entry when the file's mtime
+changes but never drops one to reclaim memory. That is the single largest memory
+cost of the whole feature.
+
+Where the time goes: cold start is 5.30 s, of which 0.46 s is the import and
+4.84 s is unpickling, and 2.6 s of *that* is the first bundle alone paying a
+one-time lazy `sklearn.ensemble` import — the remaining fourteen average 159 ms.
+Of the ~640 ms warm compute, model prediction is ~525 ms, the hourly grid 47 ms,
+and feature building 71 ms across all sixteen stations.
+
+**Live-endpoint measurements of the retired Python API** (a second, independent
+benchmark run against a real uvicorn instance of the app, same day, same
+workstation, RSS summed over the process tree):
+
+| Measurement | Value |
+|---|---|
+| `/forecast` cache hit (15-min TTL) | **2.4 ms** median |
+| `/forecast` cache miss, default threads | 15.2 s (≈7 s of that was the HTTP data fallback; a local DB replaces it) |
+| `/forecast` cache miss, `OMP_NUM_THREADS=1` | 10.9 s |
+| API process RSS, idle → models resident | 76 MB → **335 MB** |
+
+The endpoint-level RSS (335 MB) is higher than the models-only figure above
+because the live process also retains the pandas frames from the data pull and
+the HTTP/JSON machinery — use 335 MB as the sizing number.
+
+One threading subtlety cuts the other way in serving: inference is ~135
+single-row predicts, and at one row OpenMP thread dispatch costs more than the
+math — `OMP_NUM_THREADS=1` makes the warm compute 2.6× faster (642 ms → 252 ms).
+Training shows the opposite (2.3× slower single-threaded), so set the variable
+per process, never globally.
+
+**Server sizing, in plain terms:** this is a small workload and almost any server
+runs it. The service (`ping-collector`, Rust) stays under 100 MB resident with
+every model loaded (its unit caps it at 512 MB); the monthly retrain peaks at
+~210–315 MB and shares the same box. No GPU anywhere. The retrain unit sets
+`OMP_NUM_THREADS=4` so it cannot monopolise every core while the site is
+serving.
+
+## 7. Retraining policy
+
+**Why it matters here specifically.** This is not a generic "models go stale"
+argument:
+
+- **Channel geometry changes after every major flood.** Scour, deposition and
+  bank failure shift the level-to-discharge relationship at a gauge, and RID
+  revises rating curves after big events. A model trained on the pre-2024 channel
+  is predicting levels for a cross-section that no longer exists.
+- **Extreme events extend the label range.** The highest P.103 reading before the
+  2024 season was 7.54 m (October 2022); the 2024 event pushed it to 8.27 m on
+  26 September and 9.93 m on 5 October. Gradient boosting cannot extrapolate past
+  its training range — predictions saturate at the largest value it has seen — so
+  every new record is what makes the next one predictable.
+- **Station outages change feature availability.** P.87's 165-day gap in 2023 and
+  P.4A's five dead years mean the set of populated features drifts over time.
+  Retraining lets head gating and NaN handling re-adapt to the current sensors.
+
+**Recommended schedule:**
+
+| When | Why |
+|---|---|
+| **Every year, May–June (pre-monsoon)** | The minimum. Ensures the model entering the flood season has seen last season in full. |
+| **Monthly, July–November** | Cheap insurance during the season — a full retrain costs minutes, not hours (section 6), so `nice` it and forget it. |
+| **After any major flood event** | Non-negotiable. Channel geometry and rating curves have changed, and the new extreme extends the trainable label range. |
+
+Staleness is auditable without guesswork: `model_version` embeds the git short SHA
+of the code that trained the bundle (`hgb-v1+49a3de0`), and `trained_at` is a
+timestamp in every bundle. Both are echoed in every `/forecast` row, so you can
+tell from the API response alone which code produced a forecast and how old the
+model is.
+
+**Scheduled retrain (since 2026-09-12).** `scripts/water-monitor-retrain.timer`
+fires `water-monitor-retrain.service` on the 1st of every month at 03:30 server
+time (`Persistent=true`, so a missed run catches up at boot). The unit runs
+`scripts/retrain.sh` as the service user with `OMP_NUM_THREADS=4`, `Nice=15`:
+
+1. trains all stations into `models/.staging/` (the service keeps serving the
+   old bundles throughout);
+2. refuses to promote unless `metrics.json` reports a `hgb-v3+` version and at
+   least 14 trained stations (exit 3, staging discarded, old models untouched);
+3. renames the new bundles into `models/`, moving the previous generation to
+   `models/.previous/` for rollback.
+
+No restart: ping-collector reloads changed `.json.gz` exports by mtime on its
+next forecast run. `systemctl list-timers water-monitor-retrain.timer` shows the next
+run; `sudo systemctl start water-monitor-retrain.service` runs it now (after a
+flood, say); `journalctl -u water-monitor-retrain` has the log. The installer
+(`scripts/install.sh`) enables the timer.
+
+**Why the trainer refuses to run without rain (since 2026-09-12).** On
+2026-09-01 the server retrain could not reach the Open-Meteo archive on a
+checkout with no `models/cache/`, logged a warning, and quietly overwrote the
+v3 bundles with gauge-only v2 ones — the 13-hour early warning on the 2024 flood
+became an 18-hour late one and nothing on the dashboard said so. `train_all()`
+now raises `RainUnavailableError` (CLI exit 2) in that situation. Gauge-only
+bundles are still available, but only by asking for them: `--no-rain`.
+
+## 8. Operations runbook
+
+All commands assume the project virtualenv is active (`.venv` locally).
+
+**Train (all stations, with evaluation):**
+
+```bash
+python scripts/train_flood_model.py --stations all
+```
+
+**Train a subset, refit-only (skips the holdout evaluation — much faster, but
+produces no metrics and leaves σ at the `MIN_SIGMA` floor):**
+
+```bash
+python scripts/train_flood_model.py --stations P.1,P.103 --skip-eval
+```
+
+**Train from a workstation against the server's database:**
+
+```bash
+export FLOOD_ML_DB_URL='postgresql://user:pass@host:5432/dbname'
+python scripts/train_flood_model.py --stations all
+```
+
+Do not commit that URL anywhere. If the DB is unreachable the loader silently
+falls back to the HTTP API, whose discharge values are partly synthetic — check
+the log line `PostgreSQL fetch failed, falling back to HTTP API` before trusting a
+run.
+
+**Verify before promoting.** Training writes `models/metrics.json` alongside the
+bundles. Check it before treating a run as good:
+
+```bash
+python -c "import json; m=json.load(open('models/metrics.json')); \
+print(m['model_version'], m['split']); \
+print({s: v['status'] for s, v in m['stations'].items()}); \
+print({h: (d.get('pr_auc_warn'), d.get('mae')) for h, d in m['stations']['P.1']['per_horizon'].items()})"
+```
+
+Expect fifteen `trained` and one `heuristic` (P.4A). A station that reports
+`failed` names its reason in the same payload. Compare against the *previous
+run's* `metrics.json`, not an absolute bar: after the 2026-08-11 gap-fill the
+expected baseline is P.1 6 h warning PR-AUC ≈ 0.78 and MAE ≈ 5.5 cm (the
+historical ~0.97 figure belonged to the sparse pre-fill grid — see section 5).
+A *material drop from the previous run* usually means a data problem (a gauge
+that went quiet, or a bad backfill) rather than a modelling one.
+
+**Run the tests** (synthetic data only, no database or network required):
+
+```bash
+python -m pytest tests/test_flood_forecast.py -v
+```
+
+Tests cover leakage, label alignment, the coverage gate, forward-fill and
+staleness, a train/predict round trip, the heuristic fallback, feature-name
+stability, and the rain-downgrade guard (no rain series → `RainUnavailableError`,
+nothing written; `--no-rain` → v2; rain present → v3 with the rain columns in
+`feature_names`). The file runs in well under a minute, so there is no excuse
+for skipping it before a deploy.
+
+**Understanding graceful degradation.** Three things can make a forecast row
+non-model-backed, and all of them are visible in the payload:
+
+- `source: "heuristic"`, `model_version: "heuristic-v1"` — either no bundle exists
+  for that station (P.4A always, every station before the first training run), or
+  the station's newest reading is more than 6 hours old.
+- A single horizon coming back heuristic while others are model-backed — that
+  horizon's head is in the bundle's `skipped_heads`, almost always because the
+  station had fewer than 30 positive examples for that threshold.
+- A whole station flipping to heuristic after a code change — the feature-name
+  check caught a mismatch between the feature builder and the stored
+  `feature_names` (in Python `_model_forecast`; in Rust a bundle needing a
+  feature `rust/forecast` cannot build). The fix is to retrain or to port the
+  feature; the log line names the missing columns.
+
+A live example from the 2026-08-10 cache: of 48 forecast rows, 42 came from
+models and 6 were heuristic — three for P.4A, which has no bundle by design, and
+three for P.92, whose newest reading was 02:00 while the basin's newest was 09:00.
+That 7 hours of staleness crossed `STALE_AFTER_H = 6`, so P.92 correctly dropped
+to persistence. Both fallback triggers, working as intended, in one ordinary call.
+
+Inspect a bundle's skipped heads directly (`joblib.load` unpickles, so only ever
+point it at a bundle this pipeline's own `train.py` wrote — never a file from
+elsewhere):
+
+```bash
+python -c "import joblib; b=joblib.load('models/flood_P.1.joblib'); \
+print(b['model_version'], b['trained_at'], b['n_train_rows']); print(b['skipped_heads'])"
+```

docs/GAP_FILLING_GUIDE.md

@@ -1,275 +0,0 @@
-# Gap Filling and Data Integrity Guide
-
-This guide explains the enhanced gap-filling functionality that addresses data gaps and missing timestamps in the Thailand Water Monitor.
-
-## ✅ **Issues Resolved**
-
-### **1. Data Gaps Problem**
-- **Before**: Tool only fetched current day data, leaving gaps in historical records
-- **After**: Automatically detects and fills missing timestamps for the last 7 days
-
-### **2. Missing Midnight Timestamps**
-- **Before**: Jump from 23:00 to 01:00 (missing 00:00 midnight data)
-- **After**: Specifically checks for and fills midnight hour gaps
-
-### **3. Changed Values**
-- **Before**: No mechanism to update existing data if values changed on the server
-- **After**: Compares existing data with fresh API data and updates changed values
-
-## 🔧 **New Features**
-
-### **Command Line Interface**
-```bash
-# Check for missing data gaps
-python water_scraper_v3.py --check-gaps [days]
-
-# Fill missing data gaps
-python water_scraper_v3.py --fill-gaps [days]
-
-# Update existing data with latest values
-python water_scraper_v3.py --update-data [days]
-
-# Run single test cycle
-python water_scraper_v3.py --test
-
-# Show help
-python water_scraper_v3.py --help
-```
-
-### **Automatic Gap Detection**
-The system now automatically:
-- Generates expected hourly timestamps for the specified time range
-- Compares with existing database records
-- Identifies missing timestamps
-- Groups missing data by date for efficient API calls
-
-### **Intelligent Gap Filling**
-- **Historical Data Fetching**: Retrieves data for specific dates to fill gaps
-- **Selective Insertion**: Only inserts data for actually missing timestamps
-- **API Rate Limiting**: Includes delays between API calls to be respectful
-- **Error Handling**: Continues processing even if some dates fail
-
-### **Data Update Mechanism**
-- **Change Detection**: Compares water levels, discharge rates, and percentages
-- **Precision Checking**: Uses appropriate thresholds (0.001m for water level, 0.1 cms for discharge)
-- **Selective Updates**: Only updates records where values have actually changed
-
-## 📊 **Test Results**
-
-### **Before Enhancement**
-```
-Found 22 missing timestamps in the last 2 days:
-  2025-07-23: Missing hours [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]
-  2025-07-24: Missing hours [0, 20, 21, 22, 23]
-  2025-07-25: Missing hours [0, 9]
-```
-
-### **After Gap Filling**
-```
-Gap filling completed. Filled 96 missing data points
-
-Remaining gaps:
-  2025-07-24: Missing hours [10]
-  2025-07-25: Missing hours [0, 10]
-```
-
-**Improvement**: Reduced from 22 missing timestamps to 3 (86% improvement)
-
-## 🚀 **Enhanced Scraping Cycle**
-
-The regular scraping cycle now includes three phases:
-
-### **Phase 1: Current Data Collection**
-```python
-# Fetch and save current data
-water_data = self.fetch_water_data()
-success = self.save_to_database(water_data)
-```
-
-### **Phase 2: Gap Filling (Last 7 Days)**
-```python
-# Check for and fill missing data
-filled_count = self.fill_data_gaps(days_back=7)
-```
-
-### **Phase 3: Data Updates (Last 2 Days)**
-```python
-# Update existing data with latest values
-updated_count = self.update_existing_data(days_back=2)
-```
-
-## 🔧 **Technical Improvements**
-
-### **Database Connection Handling**
-- **SQLite Optimization**: Added timeout and thread safety parameters
-- **Retry Logic**: Exponential backoff for database lock errors
-- **Transaction Management**: Proper use of `engine.begin()` for automatic commits
-
-### **Error Recovery**
-```python
-# Retry logic with exponential backoff
-for attempt in range(max_retries):
-    try:
-        success = self.db_adapter.save_measurements(water_data)
-        if success:
-            return True
-    except Exception as e:
-        if "database is locked" in str(e).lower():
-            time.sleep(2 ** attempt)  # 1s, 2s, 4s delays
-            continue
-```
-
-### **Memory Efficiency**
-- **Selective Data Processing**: Only processes data for missing timestamps
-- **Batch Processing**: Groups operations by date to minimize API calls
-- **Resource Management**: Proper cleanup and connection handling
-
-## 📋 **Usage Examples**
-
-### **Daily Maintenance**
-```bash
-# Check for gaps in the last week
-python water_scraper_v3.py --check-gaps 7
-
-# Fill any found gaps
-python water_scraper_v3.py --fill-gaps 7
-
-# Update recent data for accuracy
-python water_scraper_v3.py --update-data 2
-```
-
-### **Historical Data Recovery**
-```bash
-# Check for gaps in the last month
-python water_scraper_v3.py --check-gaps 30
-
-# Fill gaps for the last month (be patient, this takes time)
-python water_scraper_v3.py --fill-gaps 30
-```
-
-### **Production Monitoring**
-```bash
-# Quick test to ensure system is working
-python water_scraper_v3.py --test
-
-# Check for recent gaps
-python water_scraper_v3.py --check-gaps 1
-```
-
-## 🔍 **Monitoring and Alerts**
-
-### **Gap Detection Output**
-```
-Found 22 missing timestamps:
-  2025-07-23: Missing hours [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]
-  2025-07-24: Missing hours [0, 20, 21, 22, 23]
-  2025-07-25: Missing hours [0, 9]
-```
-
-### **Gap Filling Progress**
-```
-Fetching data for 2025-07-24 to fill 5 missing timestamps
-Successfully fetched 368 data points from API for 2025-07-24
-Filled 80 data points for 2025-07-24
-Gap filling completed. Filled 96 missing data points
-```
-
-### **Update Detection**
-```
-Checking for updates on 2025-07-24
-Update needed for P.1 at 2025-07-24 15:00:00
-Updated 5 measurements for 2025-07-24
-Data update completed. Updated 5 measurements
-```
-
-## ⚙️ **Configuration Options**
-
-### **Environment Variables**
-```bash
-# Database configuration
-export DB_TYPE=sqlite
-export WATER_DB_PATH=water_monitoring.db
-
-# Gap filling settings (can be added to config.py)
-export GAP_FILL_DAYS=7        # Days to check for gaps
-export UPDATE_DAYS=2          # Days to check for updates
-export API_DELAY=1            # Seconds between API calls
-export MAX_RETRIES=3          # Database retry attempts
-```
-
-### **Customizable Parameters**
-- **Gap Check Period**: Default 7 days, configurable via command line
-- **Update Period**: Default 2 days, configurable via command line
-- **API Rate Limiting**: 1-second delay between calls (configurable)
-- **Retry Logic**: 3 attempts with exponential backoff (configurable)
-
-## 🛠️ **Troubleshooting**
-
-### **Common Issues**
-
-#### **Database Locked Errors**
-```
-ERROR - Error saving to SQLITE: database is locked
-```
-**Solution**: The retry logic now handles this automatically with exponential backoff.
-
-#### **API Rate Limiting**
-```
-WARNING - Too many requests to API
-```
-**Solution**: Increase delay between API calls or reduce the number of days processed at once.
-
-#### **Missing Data Still Present**
-```
-Found X missing timestamps after gap filling
-```
-**Possible Causes**:
-- Data not available on the Thai government server for those timestamps
-- Network issues during API calls
-- API returned empty data for those specific times
-
-### **Debug Commands**
-```bash
-# Enable debug logging
-export LOG_LEVEL=DEBUG
-python water_scraper_v3.py --check-gaps 1
-
-# Test specific date range
-python water_scraper_v3.py --fill-gaps 1
-
-# Check database directly
-sqlite3 water_monitoring.db "SELECT COUNT(*) FROM water_measurements;"
-sqlite3 water_monitoring.db "SELECT timestamp, COUNT(*) FROM water_measurements GROUP BY timestamp ORDER BY timestamp DESC LIMIT 10;"
-```
-
-## 📈 **Performance Metrics**
-
-### **Gap Filling Efficiency**
-- **API Calls**: Grouped by date to minimize requests
-- **Processing Speed**: ~100-400 data points per API call
-- **Success Rate**: 86% gap reduction in test case
-- **Resource Usage**: Minimal memory footprint with selective processing
-
-### **Database Performance**
-- **SQLite Optimization**: Connection pooling and timeout handling
-- **Transaction Efficiency**: Batch inserts with proper transaction management
-- **Retry Success**: Automatic recovery from temporary lock conditions
-
-## 🎯 **Best Practices**
-
-### **Regular Maintenance**
-1. **Daily**: Run `--check-gaps 1` to monitor recent data quality
-2. **Weekly**: Run `--fill-gaps 7` to catch any missed data
-3. **Monthly**: Run `--update-data 7` to ensure data accuracy
-
-### **Production Deployment**
-1. **Automated Scheduling**: Use cron or systemd timers for regular gap checks
-2. **Monitoring**: Set up alerts for excessive missing data
-3. **Backup**: Regular database backups before major gap-filling operations
-
-### **Data Quality Assurance**
-1. **Validation**: Check for reasonable value ranges after gap filling
-2. **Comparison**: Compare filled data with nearby timestamps for consistency
-3. **Documentation**: Log all gap-filling activities for audit trails
-
-This enhanced gap-filling system ensures comprehensive and accurate water level monitoring with minimal data loss and automatic recovery capabilities.

docs/GEOLOCATION_GUIDE.md

@@ -1,475 +0,0 @@
-# Geolocation Support for Grafana Geomap
-
-This guide explains the geolocation functionality added to the Thailand Water Monitor for use with Grafana's geomap visualization.
-
-## ✅ **Implemented Features**
-
-### **Database Schema Updates**
-All database adapters now support geolocation fields:
-- **latitude**: Decimal latitude coordinates (DECIMAL(10,8) for SQL, REAL for SQLite)
-- **longitude**: Decimal longitude coordinates (DECIMAL(11,8) for SQL, REAL for SQLite)  
-- **geohash**: Geohash string for efficient spatial indexing (VARCHAR(20)/TEXT)
-
-### **Station Data Enhancement**
-Station mapping now includes geolocation fields:
-```python
-'8': {
-    'code': 'P.1', 
-    'thai_name': 'สะพานนวรัฐ', 
-    'english_name': 'Nawarat Bridge',
-    'latitude': 15.6944,    # Decimal degrees
-    'longitude': 100.2028,  # Decimal degrees
-    'geohash': 'w5q6uuhvfcfp25'  # Geohash for P.1
-}
-```
-
-## 🗄️ **Database Schema**
-
-### **Updated Stations Table**
-```sql
-CREATE TABLE stations (
-    id INTEGER PRIMARY KEY,
-    station_code TEXT UNIQUE NOT NULL,
-    thai_name TEXT NOT NULL,
-    english_name TEXT NOT NULL,
-    latitude REAL,           -- NEW: Latitude coordinate
-    longitude REAL,          -- NEW: Longitude coordinate
-    geohash TEXT,           -- NEW: Geohash for spatial indexing
-    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
-    updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
-);
-```
-
-### **Database Support**
-- ✅ **SQLite**: REAL columns for coordinates, TEXT for geohash
-- ✅ **PostgreSQL**: DECIMAL(10,8) and DECIMAL(11,8) for coordinates, VARCHAR(20) for geohash
-- ✅ **MySQL**: DECIMAL(10,8) and DECIMAL(11,8) for coordinates, VARCHAR(20) for geohash
-- ✅ **VictoriaMetrics**: Geolocation data included in metric labels
-
-## 📊 **Current Station Data**
-
-### **P.1 - Nawarat Bridge (Sample)**
-- **Station Code**: P.1
-- **Thai Name**: สะพานนวรัฐ
-- **English Name**: Nawarat Bridge
-- **Latitude**: 15.6944
-- **Longitude**: 100.2028
-- **Geohash**: w5q6uuhvfcfp25
-
-### **Remaining Stations**
-The following stations are ready for geolocation data when coordinates become available:
-- P.20 - บ้านเชียงดาว (Ban Chiang Dao)
-- P.75 - บ้านช่อแล (Ban Chai Lat)
-- P.92 - บ้านเมืองกึ๊ด (Ban Muang Aut)
-- P.4A - บ้านแม่แตง (Ban Mae Taeng)
-- P.67 - บ้านแม่แต (Ban Tae)
-- P.21 - บ้านริมใต้ (Ban Rim Tai)
-- P.103 - สะพานวงแหวนรอบ 3 (Ring Bridge 3)
-- P.82 - บ้านสบวิน (Ban Sob win)
-- P.84 - บ้านพันตน (Ban Panton)
-- P.81 - บ้านโป่ง (Ban Pong)
-- P.5 - สะพานท่านาง (Tha Nang Bridge)
-- P.77 - บ้านสบแม่สะป๊วด (Baan Sop Mae Sapuord)
-- P.87 - บ้านป่าซาง (Ban Pa Sang)
-- P.76 - บ้านแม่อีไฮ (Banb Mae I Hai)
-- P.85 - บ้านหล่ายแก้ว (Baan Lai Kaew)
-
-## 🗺️ **Grafana Geomap Integration**
-
-### **Data Source Configuration**
-The geolocation data is automatically included in all database queries and can be used directly in Grafana:
-
-#### **SQLite/PostgreSQL/MySQL Query Example**
-```sql
-SELECT 
-    m.timestamp,
-    s.station_code,
-    s.english_name,
-    s.thai_name,
-    s.latitude,
-    s.longitude,
-    s.geohash,
-    m.water_level,
-    m.discharge,
-    m.discharge_percent
-FROM water_measurements m
-JOIN stations s ON m.station_id = s.id
-WHERE s.latitude IS NOT NULL 
-  AND s.longitude IS NOT NULL
-ORDER BY m.timestamp DESC
-```
-
-#### **VictoriaMetrics Query Example**
-```promql
-water_level{latitude!="",longitude!=""}
-```
-
-### **Geomap Panel Configuration**
-
-#### **1. Create Geomap Panel**
-1. Add new panel in Grafana
-2. Select "Geomap" visualization
-3. Configure data source (SQLite/PostgreSQL/MySQL/VictoriaMetrics)
-
-#### **2. Configure Location Fields**
-- **Latitude Field**: `latitude`
-- **Longitude Field**: `longitude`
-- **Alternative**: Use `geohash` field for geohash-based positioning
-
-#### **3. Configure Display Options**
-- **Station Labels**: Use `station_code` or `english_name`
-- **Tooltip Information**: Include `thai_name`, `water_level`, `discharge`
-- **Color Mapping**: Map to `water_level` or `discharge_percent`
-
-#### **4. Sample Geomap Configuration**
-```json
-{
-  "type": "geomap",
-  "title": "Thailand Water Stations",
-  "targets": [
-    {
-      "rawSql": "SELECT latitude, longitude, station_code, english_name, water_level, discharge_percent FROM stations s JOIN water_measurements m ON s.id = m.station_id WHERE s.latitude IS NOT NULL AND m.timestamp = (SELECT MAX(timestamp) FROM water_measurements WHERE station_id = s.id)",
-      "format": "table"
-    }
-  ],
-  "fieldConfig": {
-    "defaults": {
-      "custom": {
-        "hideFrom": {
-          "legend": false,
-          "tooltip": false,
-          "vis": false
-        }
-      },
-      "mappings": [],
-      "color": {
-        "mode": "continuous-GrYlRd",
-        "field": "water_level"
-      }
-    }
-  },
-  "options": {
-    "view": {
-      "id": "coords",
-      "lat": 15.6944,
-      "lon": 100.2028,
-      "zoom": 8
-    },
-    "controls": {
-      "mouseWheelZoom": true,
-      "showZoom": true,
-      "showAttribution": true
-    },
-    "layers": [
-      {
-        "type": "markers",
-        "config": {
-          "size": {
-            "field": "discharge_percent",
-            "min": 5,
-            "max": 20
-          },
-          "color": {
-            "field": "water_level"
-          },
-          "showLegend": true
-        }
-      }
-    ]
-  }
-}
-```
-
-## 🔧 **Adding New Station Coordinates**
-
-### **Method 1: Update Station Mapping**
-Edit `water_scraper_v3.py` and add coordinates to the station mapping:
-```python
-'1': {
-    'code': 'P.20', 
-    'thai_name': 'บ้านเชียงดาว', 
-    'english_name': 'Ban Chiang Dao',
-    'latitude': 19.3056,    # Add actual coordinates
-    'longitude': 98.9264,   # Add actual coordinates
-    'geohash': 'w4r6...'    # Add actual geohash
-}
-```
-
-### **Method 2: Direct Database Update**
-```sql
-UPDATE stations 
-SET latitude = 19.3056, longitude = 98.9264, geohash = 'w4r6uuhvfcfp25'
-WHERE station_code = 'P.20';
-```
-
-### **Method 3: Bulk Update Script**
-```python
-import sqlite3
-
-coordinates = {
-    'P.20': {'lat': 19.3056, 'lon': 98.9264, 'geohash': 'w4r6uuhvfcfp25'},
-    'P.75': {'lat': 18.7756, 'lon': 99.1234, 'geohash': 'w4r5uuhvfcfp25'},
-    # Add more stations...
-}
-
-conn = sqlite3.connect('water_monitoring.db')
-cursor = conn.cursor()
-
-for station_code, coords in coordinates.items():
-    cursor.execute("""
-        UPDATE stations 
-        SET latitude = ?, longitude = ?, geohash = ?
-        WHERE station_code = ?
-    """, (coords['lat'], coords['lon'], coords['geohash'], station_code))
-
-conn.commit()
-conn.close()
-```
-
-## 🌐 **Geohash Information**
-
-### **What is Geohash?**
-Geohash is a geocoding system that represents geographic coordinates as a short alphanumeric string. It provides:
-- **Spatial Indexing**: Efficient spatial queries
-- **Proximity**: Similar geohashes indicate nearby locations
-- **Hierarchical**: Longer geohashes provide more precision
-
-### **Geohash Precision Levels**
-- **5 characters**: ~2.4km precision
-- **6 characters**: ~610m precision  
-- **7 characters**: ~76m precision
-- **8 characters**: ~19m precision
-- **9+ characters**: <5m precision
-
-### **Example: P.1 Geohash**
-- **Geohash**: `w5q6uuhvfcfp25`
-- **Length**: 14 characters
-- **Precision**: Sub-meter accuracy
-- **Location**: Nawarat Bridge, Thailand
-
-## 📈 **Grafana Visualization Examples**
-
-### **1. Station Location Map**
-- **Type**: Geomap with markers
-- **Data**: Current station locations
-- **Color**: Water level or discharge percentage
-- **Size**: Discharge volume
-
-### **2. Regional Water Levels**
-- **Type**: Geomap with heatmap
-- **Data**: Water level data across regions
-- **Visualization**: Color-coded intensity map
-- **Filters**: Time range, station groups
-
-### **3. Alert Zones**
-- **Type**: Geomap with threshold markers
-- **Data**: Stations exceeding alert thresholds
-- **Visualization**: Red markers for high water levels
-- **Alerts**: Automated notifications for critical levels
-
-## 🔄 **Updating a Running System**
-
-### **Automated Migration Script**
-Use the provided migration script to safely add geolocation columns to your existing database:
-
-```bash
-# Stop the water monitoring service first
-sudo systemctl stop water-monitor
-
-# Run the migration script
-python migrate_geolocation.py
-
-# Restart the service
-sudo systemctl start water-monitor
-```
-
-### **Migration Script Features**
-- ✅ **Auto-detects database type** from environment variables
-- ✅ **Checks existing columns** to avoid conflicts
-- ✅ **Supports all database types** (SQLite, PostgreSQL, MySQL)
-- ✅ **Adds sample data** for P.1 station
-- ✅ **Safe operation** - won't break existing data
-
-### **Step-by-Step Migration Process**
-
-#### **1. Stop the Application**
-```bash
-# If running as systemd service
-sudo systemctl stop water-monitor
-
-# If running in screen/tmux
-# Use Ctrl+C to stop the process
-
-# If running as Docker container
-docker stop water-monitor
-```
-
-#### **2. Backup Your Database**
-```bash
-# SQLite backup
-cp water_monitoring.db water_monitoring.db.backup
-
-# PostgreSQL backup
-pg_dump water_monitoring > water_monitoring_backup.sql
-
-# MySQL backup
-mysqldump water_monitoring > water_monitoring_backup.sql
-```
-
-#### **3. Run Migration Script**
-```bash
-# Default (uses environment variables)
-python migrate_geolocation.py
-
-# Or specify database path for SQLite
-SQLITE_DB_PATH=/path/to/water_monitoring.db python migrate_geolocation.py
-```
-
-#### **4. Verify Migration**
-```bash
-# Check SQLite schema
-sqlite3 water_monitoring.db ".schema stations"
-
-# Check PostgreSQL schema
-psql -d water_monitoring -c "\d stations"
-
-# Check MySQL schema
-mysql -e "DESCRIBE water_monitoring.stations"
-```
-
-#### **5. Update Application Code**
-Ensure you have the latest version of the application with geolocation support:
-```bash
-# Pull latest code
-git pull origin main
-
-# Install any new dependencies
-pip install -r requirements.txt
-```
-
-#### **6. Restart Application**
-```bash
-# Systemd service
-sudo systemctl start water-monitor
-
-# Docker container
-docker start water-monitor
-
-# Manual execution
-python water_scraper_v3.py
-```
-
-### **Migration Output Example**
-```
-2025-07-28 17:30:00,123 - INFO - Starting geolocation column migration...
-2025-07-28 17:30:00,124 - INFO - Detected database type: SQLITE
-2025-07-28 17:30:00,125 - INFO - Migrating SQLite database: water_monitoring.db
-2025-07-28 17:30:00,126 - INFO - Current columns in stations table: ['id', 'station_code', 'thai_name', 'english_name', 'created_at', 'updated_at']
-2025-07-28 17:30:00,127 - INFO - Added latitude column
-2025-07-28 17:30:00,128 - INFO - Added longitude column
-2025-07-28 17:30:00,129 - INFO - Added geohash column
-2025-07-28 17:30:00,130 - INFO - Successfully added columns: latitude, longitude, geohash
-2025-07-28 17:30:00,131 - INFO - Updated P.1 station with sample geolocation data
-2025-07-28 17:30:00,132 - INFO - P.1 station geolocation: ('P.1', 15.6944, 100.2028, 'w5q6uuhvfcfp25')
-2025-07-28 17:30:00,133 - INFO - ✅ Migration completed successfully!
-2025-07-28 17:30:00,134 - INFO - You can now restart your water monitoring application
-2025-07-28 17:30:00,135 - INFO - The system will automatically use the new geolocation columns
-```
-
-## 🔍 **Troubleshooting**
-
-### **Migration Issues**
-
-#### **Database Locked Error**
-```bash
-# Stop all processes using the database
-sudo systemctl stop water-monitor
-pkill -f water_scraper
-
-# Wait a few seconds, then run migration
-sleep 5
-python migrate_geolocation.py
-```
-
-#### **Permission Denied**
-```bash
-# Check database file permissions
-ls -la water_monitoring.db
-
-# Fix permissions if needed
-sudo chown $USER:$USER water_monitoring.db
-chmod 664 water_monitoring.db
-```
-
-#### **Missing Dependencies**
-```bash
-# For PostgreSQL
-pip install psycopg2-binary
-
-# For MySQL
-pip install pymysql
-
-# For all databases
-pip install -r requirements.txt
-```
-
-### **Verification Issues**
-
-#### **Missing Coordinates**
-If stations don't appear on the geomap:
-1. Check if latitude/longitude are NULL in database
-2. Verify geolocation data in station mapping
-3. Ensure database schema includes geolocation columns
-4. Run migration script if columns are missing
-
-#### **Incorrect Positioning**
-If stations appear in wrong locations:
-1. Verify coordinate format (decimal degrees)
-2. Check latitude/longitude order (lat first, lon second)
-3. Validate geohash accuracy
-
-### **Rollback Procedure**
-If migration causes issues:
-
-#### **SQLite Rollback**
-```bash
-# Stop application
-sudo systemctl stop water-monitor
-
-# Restore backup
-cp water_monitoring.db.backup water_monitoring.db
-
-# Restart with old version
-sudo systemctl start water-monitor
-```
-
-#### **PostgreSQL Rollback**
-```sql
--- Remove added columns
-ALTER TABLE stations DROP COLUMN IF EXISTS latitude;
-ALTER TABLE stations DROP COLUMN IF EXISTS longitude;
-ALTER TABLE stations DROP COLUMN IF EXISTS geohash;
-```
-
-#### **MySQL Rollback**
-```sql
--- Remove added columns
-ALTER TABLE stations DROP COLUMN latitude;
-ALTER TABLE stations DROP COLUMN longitude;
-ALTER TABLE stations DROP COLUMN geohash;
-```
-
-## 🎯 **Next Steps**
-
-### **Immediate Actions**
-1. **Gather Coordinates**: Collect GPS coordinates for all 16 stations
-2. **Update Database**: Add coordinates to remaining stations
-3. **Create Dashboards**: Build Grafana geomap visualizations
-
-### **Future Enhancements**
-1. **Automatic Geocoding**: API integration for address-to-coordinate conversion
-2. **Mobile GPS**: Mobile app for field coordinate collection
-3. **Satellite Integration**: Satellite imagery overlay in Grafana
-4. **Geofencing**: Alert zones based on geographic boundaries
-
-The geolocation functionality is now fully implemented and ready for use with Grafana's geomap visualization. Station P.1 (Nawarat Bridge) serves as a working example with complete coordinate data.

docs/GITEA_WORKFLOWS.md

@@ -1,302 +0,0 @@
-# 🔄 Gitea Actions Workflows - Northern Thailand Ping River Monitor
-
-## 📋 Overview
-
-This document describes the Gitea Actions workflows configured for the Northern Thailand Ping River Monitor project. These workflows provide comprehensive CI/CD, security scanning, and documentation generation.
-
-## 🚀 Available Workflows
-
-### 1. **CI/CD Pipeline** (`.gitea/workflows/ci.yml`)
-
-**Triggers:**
-- Push to `main` or `develop` branches
-- Pull requests to `main`
-- Daily scheduled runs at 2 AM UTC
-
-**Jobs:**
-- **Test Suite**: Multi-version Python testing (3.9-3.12)
-- **Code Quality**: Linting, formatting, and type checking
-- **Build**: Docker image creation and testing
-- **Integration Test**: Testing with VictoriaMetrics service
-- **Deploy Staging**: Automatic deployment to staging (develop branch)
-- **Deploy Production**: Manual deployment to production (main branch)
-- **Performance Test**: Load testing after production deployment
-
-**Key Features:**
-- ✅ Multi-Python version testing
-- ✅ Docker multi-architecture builds (amd64, arm64)
-- ✅ Service integration testing
-- ✅ Automatic staging deployment
-- ✅ Manual production approval
-- ✅ Performance validation
-
-### 2. **Security & Dependency Updates** (`.gitea/workflows/security.yml`)
-
-**Triggers:**
-- Daily scheduled runs at 3 AM UTC
-- Manual dispatch
-- Changes to requirements files or Dockerfile
-
-**Jobs:**
-- **Dependency Scan**: Safety, Bandit, Semgrep security scans
-- **Docker Security**: Trivy vulnerability scanning
-- **License Check**: License compliance verification
-- **Dependency Updates**: Automated update detection
-- **Code Quality**: Complexity and maintainability analysis
-
-**Key Features:**
-- 🔒 Daily security scans
-- 📦 Dependency vulnerability detection
-- 📄 License compliance checking
-- 🔄 Automated update notifications
-- 📊 Code quality metrics
-
-### 3. **Release Workflow** (`.gitea/workflows/release.yml`)
-
-**Triggers:**
-- Git tags matching `v*.*.*` pattern
-- Manual dispatch with version input
-
-**Jobs:**
-- **Create Release**: Automated release creation with changelog
-- **Test Release**: Comprehensive testing across Python versions
-- **Build Release**: Multi-architecture Docker images with proper tags
-- **Security Scan**: Trivy security scanning of release images
-- **Deploy Release**: Production deployment with health checks
-- **Validate Release**: Post-deployment validation and testing
-
-**Key Features:**
-- 🏷️ Automated release creation
-- 📝 Changelog generation
-- 🐳 Multi-architecture Docker builds
-- 🔒 Security scanning
-- ✅ Comprehensive validation
-
-### 4. **Documentation** (`.gitea/workflows/docs.yml`)
-
-**Triggers:**
-- Changes to documentation files
-- Changes to Python source files
-- Manual dispatch
-
-**Jobs:**
-- **Validate Docs**: Link checking and structure validation
-- **Generate API Docs**: OpenAPI specification generation
-- **Build Sphinx Docs**: Comprehensive API documentation
-- **Documentation Summary**: Build status and artifact summary
-
-**Key Features:**
-- 📚 Automated API documentation
-- 🔗 Link validation
-- 📖 Sphinx documentation generation
-- ✅ Documentation completeness checking
-
-## 🔧 Workflow Configuration
-
-### **Required Secrets**
-
-Configure these secrets in your Gitea repository settings:
-
-```bash
-GITEA_TOKEN          # Gitea access token for container registry
-GITHUB_TOKEN         # GitHub personal access token for better rate limits
-SLACK_WEBHOOK_URL    # Optional: Slack notifications
-STAGING_WEBHOOK_URL  # Optional: Staging deployment webhook
-PRODUCTION_WEBHOOK_URL # Optional: Production deployment webhook
-```
-
-**Setting up GitHub Token:**
-1. Go to GitHub → Settings → Developer settings → Personal access tokens
-2. Generate a new token with `public_repo` scope (for public repos) or `repo` scope (for private repos)
-3. Copy the token
-4. In your Gitea repository → Settings → Secrets → Add `GITHUB_TOKEN` with the copied value
-
-### **Environment Variables**
-
-Key environment variables used across workflows:
-
-```yaml
-PYTHON_VERSION: '3.11'           # Default Python version
-REGISTRY: git.b4l.co.th          # Container registry
-IMAGE_NAME: grabowski/northern-thailand-ping-river-monitor
-```
-
-## 📊 Workflow Status
-
-### **CI/CD Pipeline Status**
-- **Test Coverage**: Multi-version Python testing
-- **Code Quality**: Automated linting and formatting
-- **Security**: Integrated security scanning
-- **Deployment**: Automated staging, manual production
-
-### **Security Monitoring**
-- **Daily Scans**: Automated vulnerability detection
-- **Dependency Updates**: Proactive update notifications
-- **License Compliance**: Automated license checking
-- **Code Quality**: Continuous quality monitoring
-
-### **Release Management**
-- **Automated Releases**: Tag-based release creation
-- **Multi-Architecture**: Support for amd64 and arm64
-- **Security Validation**: Pre-deployment security checks
-- **Health Monitoring**: Post-deployment validation
-
-## 🚀 Usage Examples
-
-### **Triggering Workflows**
-
-**Manual CI/CD Run:**
-```bash
-# Push to trigger CI/CD
-git push origin main
-
-# Create pull request to trigger testing
-git checkout -b feature/new-feature
-git push origin feature/new-feature
-# Create PR in Gitea UI
-```
-
-**Manual Security Scan:**
-```bash
-# Trigger via Gitea Actions UI
-# Go to Actions → Security & Dependency Updates → Run workflow
-```
-
-**Creating a Release:**
-```bash
-# Create and push a tag
-git tag v3.1.1
-git push origin v3.1.1
-
-# Or use manual dispatch in Gitea Actions UI
-```
-
-### **Monitoring Workflow Results**
-
-**Check Workflow Status:**
-1. Navigate to your repository in Gitea
-2. Click on "Actions" tab
-3. View workflow runs and their status
-
-**Download Artifacts:**
-1. Click on a completed workflow run
-2. Scroll to "Artifacts" section
-3. Download reports and logs
-
-**View Security Reports:**
-1. Go to Security workflow runs
-2. Download security-reports artifacts
-3. Review JSON reports for vulnerabilities
-
-## 🔍 Troubleshooting
-
-### **Common Issues**
-
-**Workflow Fails on Dependencies:**
-```bash
-# Check requirements.txt for version conflicts
-pip-compile requirements.in
-```
-
-**Docker Build Fails:**
-```bash
-# Test Docker build locally
-make docker-build
-docker run --rm ping-river-monitor python run.py --test
-```
-
-**Security Scan Failures:**
-```bash
-# Run security scans locally
-safety check -r requirements.txt
-bandit -r src/
-```
-
-**Test Failures:**
-```bash
-# Run tests locally
-make test
-python tests/test_integration.py
-```
-
-### **Debugging Workflows**
-
-**Enable Debug Logging:**
-Add to workflow file:
-```yaml
-env:
-  ACTIONS_STEP_DEBUG: true
-  ACTIONS_RUNNER_DEBUG: true
-```
-
-**Check Workflow Logs:**
-1. Go to failed workflow run
-2. Click on failed job
-3. Expand failed step to see detailed logs
-
-**Validate Workflow Syntax:**
-```bash
-# Validate YAML syntax
-make validate-workflows
-```
-
-## 📈 Performance Optimization
-
-### **Caching Strategy**
-- **Pip Cache**: Cached across workflow runs
-- **Docker Layer Cache**: GitHub Actions cache for faster builds
-- **Dependency Cache**: Cached based on requirements.txt hash
-
-### **Parallel Execution**
-- **Matrix Builds**: Multiple Python versions tested in parallel
-- **Independent Jobs**: Security scans run independently of tests
-- **Conditional Execution**: Jobs skip when not needed
-
-### **Resource Management**
-- **Timeout Settings**: Prevent hanging workflows
-- **Resource Limits**: Appropriate runner sizing
-- **Artifact Cleanup**: Automatic cleanup of old artifacts
-
-## 🔒 Security Best Practices
-
-### **Secret Management**
-- Use Gitea repository secrets for sensitive data
-- Never commit secrets to repository
-- Rotate secrets regularly
-- Use least-privilege access tokens
-
-### **Container Security**
-- Multi-stage Docker builds for smaller images
-- Non-root user in containers
-- Regular base image updates
-- Vulnerability scanning before deployment
-
-### **Code Security**
-- Automated security scanning in CI/CD
-- Dependency vulnerability monitoring
-- License compliance checking
-- Code quality enforcement
-
-## 📚 Additional Resources
-
-### **Gitea Actions Documentation**
-- [Gitea Actions Overview](https://docs.gitea.io/en-us/usage/actions/)
-- [Workflow Syntax](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions)
-- [Available Actions](https://github.com/marketplace?type=actions)
-
-### **Project-Specific Resources**
-- [Contributing Guide](../CONTRIBUTING.md)
-- [Deployment Checklist](../DEPLOYMENT_CHECKLIST.md)
-- [Project Structure](PROJECT_STRUCTURE.md)
-
-### **Monitoring and Alerts**
-- Workflow status badges in README
-- Email notifications for failures
-- Slack/Discord integration for team updates
-- Grafana dashboards for deployment metrics
-
----
-
-**Workflow Version**: v3.1.3  
-**Last Updated**: 2025-08-12  
-**Maintained By**: Ping River Monitor Team

docs/HTTPS_CONFIGURATION.md

@@ -1,389 +0,0 @@
-# HTTPS VictoriaMetrics Configuration Guide
-
-This guide explains how to configure the Thailand Water Monitor to connect to VictoriaMetrics through HTTPS and reverse proxies.
-
-## Configuration Options
-
-### 1. Environment Variables for HTTPS
-
-```bash
-# Option 1: Full HTTPS URL (Recommended)
-export DB_TYPE=victoriametrics
-export VM_HOST=https://vm.example.com
-export VM_PORT=443
-
-# Option 2: Host and port separately
-export DB_TYPE=victoriametrics
-export VM_HOST=vm.example.com
-export VM_PORT=443
-
-# Option 3: Custom port with HTTPS
-export DB_TYPE=victoriametrics
-export VM_HOST=https://vm.example.com
-export VM_PORT=8443
-```
-
-### 2. Windows PowerShell Configuration
-
-```powershell
-# Set environment variables for HTTPS
-$env:DB_TYPE="victoriametrics"
-$env:VM_HOST="https://vm.example.com"
-$env:VM_PORT="443"
-
-# Run the water monitor
-python water_scraper_v3.py
-```
-
-### 3. Linux/Mac Configuration
-
-```bash
-# Set environment variables for HTTPS
-export DB_TYPE=victoriametrics
-export VM_HOST=https://vm.example.com
-export VM_PORT=443
-
-# Run the water monitor
-python water_scraper_v3.py
-```
-
-## Reverse Proxy Examples
-
-### 1. Nginx Reverse Proxy
-
-```nginx
-server {
-    listen 443 ssl http2;
-    server_name vm.example.com;
-    
-    # SSL Configuration
-    ssl_certificate /path/to/certificate.crt;
-    ssl_certificate_key /path/to/private.key;
-    ssl_protocols TLSv1.2 TLSv1.3;
-    ssl_ciphers ECDHE-RSA-AES256-GCM-SHA512:DHE-RSA-AES256-GCM-SHA512;
-    
-    # Security headers
-    add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
-    add_header X-Frame-Options DENY always;
-    add_header X-Content-Type-Options nosniff always;
-    
-    # Optional: Basic authentication
-    # auth_basic "VictoriaMetrics";
-    # auth_basic_user_file /etc/nginx/.htpasswd;
-    
-    location / {
-        proxy_pass http://localhost:8428;
-        proxy_set_header Host $host;
-        proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header X-Forwarded-Proto $scheme;
-        
-        # WebSocket support (if needed)
-        proxy_http_version 1.1;
-        proxy_set_header Upgrade $http_upgrade;
-        proxy_set_header Connection "upgrade";
-        
-        # Timeouts
-        proxy_connect_timeout 60s;
-        proxy_send_timeout 60s;
-        proxy_read_timeout 60s;
-    }
-}
-
-# Redirect HTTP to HTTPS
-server {
-    listen 80;
-    server_name vm.example.com;
-    return 301 https://$server_name$request_uri;
-}
-```
-
-### 2. Apache Reverse Proxy
-
-```apache
-<VirtualHost *:443>
-    ServerName vm.example.com
-    
-    # SSL Configuration
-    SSLEngine on
-    SSLCertificateFile /path/to/certificate.crt
-    SSLCertificateKeyFile /path/to/private.key
-    SSLProtocol all -SSLv3 -TLSv1 -TLSv1.1
-    SSLCipherSuite ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384
-    
-    # Security headers
-    Header always set Strict-Transport-Security "max-age=31536000; includeSubDomains"
-    Header always set X-Frame-Options DENY
-    Header always set X-Content-Type-Options nosniff
-    
-    # Reverse proxy configuration
-    ProxyPreserveHost On
-    ProxyPass / http://localhost:8428/
-    ProxyPassReverse / http://localhost:8428/
-    
-    # Optional: Basic authentication
-    # AuthType Basic
-    # AuthName "VictoriaMetrics"
-    # AuthUserFile /etc/apache2/.htpasswd
-    # Require valid-user
-</VirtualHost>
-
-<VirtualHost *:80>
-    ServerName vm.example.com
-    Redirect permanent / https://vm.example.com/
-</VirtualHost>
-```
-
-### 3. Traefik Reverse Proxy
-
-```yaml
-# docker-compose.yml with Traefik
-version: '3.8'
-
-services:
-  traefik:
-    image: traefik:v2.10
-    command:
-      - --api.dashboard=true
-      - --entrypoints.web.address=:80
-      - --entrypoints.websecure.address=:443
-      - --providers.docker=true
-      - --certificatesresolvers.letsencrypt.acme.tlschallenge=true
-      - --certificatesresolvers.letsencrypt.acme.email=admin@example.com
-      - --certificatesresolvers.letsencrypt.acme.storage=/letsencrypt/acme.json
-    ports:
-      - "80:80"
-      - "443:443"
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
-      - letsencrypt:/letsencrypt
-    labels:
-      - traefik.http.routers.api.rule=Host(`traefik.example.com`)
-      - traefik.http.routers.api.tls.certresolver=letsencrypt
-
-  victoriametrics:
-    image: victoriametrics/victoria-metrics:latest
-    command:
-      - '--storageDataPath=/victoria-metrics-data'
-      - '--retentionPeriod=2y'
-      - '--httpListenAddr=:8428'
-    volumes:
-      - vm_data:/victoria-metrics-data
-    labels:
-      - traefik.enable=true
-      - traefik.http.routers.vm.rule=Host(`vm.example.com`)
-      - traefik.http.routers.vm.tls.certresolver=letsencrypt
-      - traefik.http.services.vm.loadbalancer.server.port=8428
-
-volumes:
-  vm_data:
-  letsencrypt:
-```
-
-## Testing HTTPS Configuration
-
-### 1. Test Connection
-
-```bash
-# Test HTTPS connection
-curl -k https://vm.example.com/health
-
-# Test with specific port
-curl -k https://vm.example.com:8443/health
-
-# Test API endpoint
-curl -k "https://vm.example.com/api/v1/query?query=up"
-```
-
-### 2. Test with Water Monitor
-
-```bash
-# Set environment variables
-export DB_TYPE=victoriametrics
-export VM_HOST=https://vm.example.com
-export VM_PORT=443
-
-# Test with demo script
-python demo_databases.py victoriametrics
-
-# Run full water monitor
-python water_scraper_v3.py
-```
-
-### 3. Verify SSL Certificate
-
-```bash
-# Check SSL certificate
-openssl s_client -connect vm.example.com:443 -servername vm.example.com
-
-# Check certificate expiration
-echo | openssl s_client -connect vm.example.com:443 2>/dev/null | openssl x509 -noout -dates
-```
-
-## Configuration Examples
-
-### 1. Production HTTPS Setup
-
-```bash
-# Environment variables for production
-export DB_TYPE=victoriametrics
-export VM_HOST=https://metrics.company.com
-export VM_PORT=443
-export LOG_LEVEL=INFO
-export SCRAPING_INTERVAL_HOURS=1
-
-# Run water monitor
-python water_scraper_v3.py
-```
-
-### 2. Development with Self-Signed Certificate
-
-```bash
-# For development with self-signed certificates
-export DB_TYPE=victoriametrics
-export VM_HOST=https://dev-vm.local
-export VM_PORT=443
-export PYTHONHTTPSVERIFY=0  # Disable SSL verification (dev only)
-
-python water_scraper_v3.py
-```
-
-### 3. Custom Port Configuration
-
-```bash
-# Custom HTTPS port
-export DB_TYPE=victoriametrics
-export VM_HOST=https://vm.example.com
-export VM_PORT=8443
-
-python water_scraper_v3.py
-```
-
-## Troubleshooting HTTPS Issues
-
-### 1. SSL Certificate Errors
-
-```bash
-# Error: SSL certificate verify failed
-# Solution: Check certificate validity
-openssl x509 -in certificate.crt -text -noout
-
-# Temporary workaround (not recommended for production)
-export PYTHONHTTPSVERIFY=0
-```
-
-### 2. Connection Timeout
-
-```bash
-# Error: Connection timeout
-# Check firewall and network connectivity
-telnet vm.example.com 443
-nc -zv vm.example.com 443
-```
-
-### 3. DNS Resolution Issues
-
-```bash
-# Error: Name resolution failed
-# Check DNS resolution
-nslookup vm.example.com
-dig vm.example.com
-```
-
-### 4. Proxy Configuration Issues
-
-```bash
-# Check proxy logs
-# Nginx
-tail -f /var/log/nginx/error.log
-
-# Apache
-tail -f /var/log/apache2/error.log
-
-# Test direct connection to backend
-curl http://localhost:8428/health
-```
-
-## Security Best Practices
-
-### 1. SSL/TLS Configuration
-
-- Use TLS 1.2 or higher
-- Disable weak ciphers
-- Enable HSTS headers
-- Use strong SSL certificates
-
-### 2. Authentication
-
-```nginx
-# Basic authentication in Nginx
-auth_basic "VictoriaMetrics Access";
-auth_basic_user_file /etc/nginx/.htpasswd;
-
-# Create password file
-htpasswd -c /etc/nginx/.htpasswd username
-```
-
-### 3. Network Security
-
-- Use firewall rules to restrict access
-- Consider VPN for internal access
-- Implement rate limiting
-- Monitor access logs
-
-### 4. Certificate Management
-
-```bash
-# Auto-renewal with Let's Encrypt
-certbot renew --dry-run
-
-# Certificate monitoring
-echo | openssl s_client -connect vm.example.com:443 2>/dev/null | \
-openssl x509 -noout -dates | grep notAfter
-```
-
-## Docker Configuration for HTTPS
-
-### 1. Docker Compose with HTTPS
-
-```yaml
-version: '3.8'
-
-services:
-  water-monitor:
-    build: .
-    environment:
-      - DB_TYPE=victoriametrics
-      - VM_HOST=https://vm.example.com
-      - VM_PORT=443
-    restart: unless-stopped
-    depends_on:
-      - victoriametrics
-
-  victoriametrics:
-    image: victoriametrics/victoria-metrics:latest
-    ports:
-      - "8428:8428"
-    volumes:
-      - vm_data:/victoria-metrics-data
-    command:
-      - '--storageDataPath=/victoria-metrics-data'
-      - '--retentionPeriod=2y'
-      - '--httpListenAddr=:8428'
-
-volumes:
-  vm_data:
-```
-
-### 2. Environment File (.env)
-
-```bash
-# .env file
-DB_TYPE=victoriametrics
-VM_HOST=https://vm.example.com
-VM_PORT=443
-LOG_LEVEL=INFO
-SCRAPING_INTERVAL_HOURS=1
-```
-
-This configuration guide provides comprehensive instructions for setting up HTTPS connectivity to VictoriaMetrics through reverse proxies, ensuring secure and reliable data transmission for the Thailand Water Monitor.

docs/MATRIX_QUICK_START.md

@@ -0,0 +1,79 @@
+# Quick Matrix Alerting Setup
+
+## Step 1: Get Matrix Account
+1. Go to https://app.element.io or install Element app
+2. Create account or login with existing Matrix account
+
+## Step 2: Get Access Token
+
+### Method 1: Element Web (Recommended)
+1. Open Element in browser: https://app.element.io
+2. Login to your account
+3. Click Settings (gear icon) → Help & About → Advanced
+4. Copy your "Access Token" (starts with `syt_...` or similar)
+
+### Method 2: Command Line
+```bash
+curl -X POST https://matrix.org/_matrix/client/v3/login \
+  -H "Content-Type: application/json" \
+  -d '{
+    "type": "m.login.password",
+    "user": "your_username",
+    "password": "your_password"
+  }'
+```
+
+## Step 3: Create Alert Room
+1. In Element, click "+" to create new room
+2. Name: "Water Level Alerts"
+3. Set to Private
+4. Copy the room ID from room settings (format: `!roomid:matrix.org`)
+
+## Step 4: Configure .env File
+Add these to your `.env` file:
+```bash
+MATRIX_HOMESERVER=https://matrix.org
+MATRIX_ACCESS_TOKEN=syt_your_access_token_here
+MATRIX_ROOM_ID=!your_room_id:matrix.org
+```
+
+## Step 5: Restart and check
+The Matrix messages are sent by `ping-collector` (the Rust service, with
+`FORECAST_IN_COLLECTOR=true`), together with the ntfy notifications:
+
+```bash
+sudo systemctl restart ping-collector
+journalctl -u ping-collector | grep -i matrix      # "Matrix notifications: room ... on ..."
+# what would be sent right now (ntfy and Matrix), changing nothing:
+sudo -u water-monitor /opt/thailand-water-monitor/bin/ping-collector notify --dry-run
+```
+
+## What the room receives
+The same transitions as the ntfy topics (docs/NOTIFICATIONS.md), each once:
+a station crossing its warning or danger level and back (0.10 m hysteresis),
+the P.1 early-warning outlook turning on and off, and the gauge feed going
+stale or recovering. ntfy's per-topic summaries ("Ping River: warning level
+reached") are not repeated in the room. For example:
+
+```
+⚠️ **Warning level at P.1**
+Nawarat Bridge, Chiang Mai city: 3.75 m at 18 Oct 14:00, above the warning level of 3.70 m.
+
+📈 Dashboard: https://water.buildfor.life/
+```
+
+The room has its own delivery state (`matrix:` keys in `notification_state`),
+so a Matrix outage is retried on Matrix only and does not re-send ntfy.
+
+The old `run.py --alert-check` cron job (every 15 minutes, repeating an alert
+for as long as a level stayed high) is replaced by this;
+`scripts/install.sh` comments it out of the crontab if it is still there,
+keeping a backup next to the app.
+
+## Troubleshooting
+- **HTTP 401/403** in `journalctl -u ping-collector`: the access token is invalid
+  or the account is not in the room.
+- **Room not found**: the room ID needs the `!` prefix and the `:homeserver`
+  suffix.
+- **Nothing sent**: there was no transition; `ping-collector notify --dry-run`
+  shows the readings judged and what would be sent.

docs/MIGRATION_QUICKSTART.md

@@ -1,136 +0,0 @@
-# Geolocation Migration Quick Start
-
-This is a quick reference guide for updating a running Thailand Water Monitor system to add geolocation support for Grafana geomap.
-
-## 🚀 **Quick Migration (5 minutes)**
-
-### **Step 1: Stop Application**
-```bash
-# Stop the service (choose your method)
-sudo systemctl stop water-monitor
-# OR
-docker stop water-monitor
-# OR use Ctrl+C if running manually
-```
-
-### **Step 2: Backup Database**
-```bash
-# SQLite backup
-cp water_monitoring.db water_monitoring.db.backup
-
-# PostgreSQL backup
-pg_dump water_monitoring > backup.sql
-
-# MySQL backup
-mysqldump water_monitoring > backup.sql
-```
-
-### **Step 3: Run Migration**
-```bash
-# Run the automated migration script
-python migrate_geolocation.py
-```
-
-### **Step 4: Restart Application**
-```bash
-# Restart the service
-sudo systemctl start water-monitor
-# OR
-docker start water-monitor
-# OR
-python water_scraper_v3.py
-```
-
-## ✅ **Expected Output**
-```
-2025-07-28 17:30:00,123 - INFO - Starting geolocation column migration...
-2025-07-28 17:30:00,124 - INFO - Detected database type: SQLITE
-2025-07-28 17:30:00,127 - INFO - Added latitude column
-2025-07-28 17:30:00,128 - INFO - Added longitude column
-2025-07-28 17:30:00,129 - INFO - Added geohash column
-2025-07-28 17:30:00,133 - INFO - ✅ Migration completed successfully!
-```
-
-## 🗺️ **Verify Geolocation Works**
-
-### **Check Database**
-```bash
-# SQLite
-sqlite3 water_monitoring.db "SELECT station_code, latitude, longitude, geohash FROM stations WHERE station_code = 'P.1';"
-
-# Expected output: P.1|15.6944|100.2028|w5q6uuhvfcfp25
-```
-
-### **Test Application**
-```bash
-# Run a test cycle
-python water_scraper_v3.py --test
-
-# Should complete without errors
-```
-
-## 🔧 **Grafana Setup**
-
-### **Query for Geomap**
-```sql
-SELECT 
-    s.latitude, s.longitude, s.station_code, s.english_name,
-    m.water_level, m.discharge_percent
-FROM stations s 
-JOIN water_measurements m ON s.id = m.station_id
-WHERE s.latitude IS NOT NULL 
-  AND m.timestamp = (SELECT MAX(timestamp) FROM water_measurements WHERE station_id = s.id)
-```
-
-### **Geomap Configuration**
-1. Create new panel → Select "Geomap"
-2. Set **Latitude field**: `latitude`
-3. Set **Longitude field**: `longitude`
-4. Set **Color field**: `water_level`
-5. Set **Size field**: `discharge_percent`
-
-## 🚨 **Troubleshooting**
-
-### **Database Locked**
-```bash
-sudo systemctl stop water-monitor
-pkill -f water_scraper
-sleep 5
-python migrate_geolocation.py
-```
-
-### **Permission Error**
-```bash
-sudo chown $USER:$USER water_monitoring.db
-chmod 664 water_monitoring.db
-```
-
-### **Missing Dependencies**
-```bash
-pip install psycopg2-binary pymysql
-```
-
-## 🔄 **Rollback (if needed)**
-```bash
-# Stop application
-sudo systemctl stop water-monitor
-
-# Restore backup
-cp water_monitoring.db.backup water_monitoring.db
-
-# Restart
-sudo systemctl start water-monitor
-```
-
-## 📚 **More Information**
-- **Full Guide**: See `GEOLOCATION_GUIDE.md`
-- **Migration Script**: `migrate_geolocation.py`
-- **Database Schema**: Updated with latitude, longitude, geohash columns
-
-## 🎯 **What You Get**
-- ✅ **P.1 Station** ready for geomap (Nawarat Bridge)
-- ✅ **Database Schema** updated for all 16 stations
-- ✅ **Grafana Compatible** data structure
-- ✅ **Backward Compatible** - existing data preserved
-
-**Total Time**: ~5 minutes for complete migration

docs/NOTIFICATIONS.md

@@ -0,0 +1,157 @@
+# Flood notifications (ntfy)
+
+Public push notifications for threshold crossings, without accounts, mailing
+lists or app-store review: the monitor publishes to a self-hosted
+[ntfy](https://ntfy.sh) server, and anyone subscribes to the topics they care
+about from the free ntfy app (iOS, Android, F-Droid) or a browser tab.
+
+ntfy is one Go binary with a sqlite cache: ~30 MB RSS idle, negligible CPU. It
+runs on the same VPS as the monitor.
+
+## What subscribers get
+
+Every message is a **transition**, never a state. Crossing up into a level sends
+one message; dropping back below it (with 0.10 m hysteresis) sends one
+all-clear. A river that sits at 3.9 m for three days produces two messages, not
+seventy-two. In a quiet season a subscriber hears nothing.
+
+| Topic | Trigger | Priority |
+|---|---|---|
+| `ping-warning` | any gauge crosses its warning threshold; levels falling back | 4 (high) / 2 |
+| `ping-danger` | any gauge crosses its danger threshold | 5 (max, breaks Do-Not-Disturb) |
+| `ping-<station>-warning` | that gauge crosses warning; back to normal | 4 / 2 |
+| `ping-<station>-danger` | that gauge crosses danger; back below danger | 5 / 3 |
+| `ping-p1-outlook` | model P(warning within 24 h) at P.1 rises through 50 % (clears below 25 %) | 4 / 2 |
+| `ping-status` | no reading for ≥ 3 h; feed recovered; with the Rust notifier also: RID reports ≥ 18 h behind while ThaiWater is current, and back | 3 / 2 |
+
+Station slugs are the code lowercased without the dot: `p1`, `p103`, `p67`.
+Thresholds are the ones in `src/ml/features.py` (`THRESHOLDS`): P.1 3.70 /
+4.20 m, P.103 5.95 / 6.75 m, and so on.
+
+The outlook topic is opt-in for a reason: it is model output, and the message
+says so. Observed-crossing topics only ever report a gauge reading.
+
+Each message carries a click-through and an "Open dashboard" action button to
+the public dashboard.
+
+## How it runs
+
+The notifier runs in `ping-collector` (`rust/collector/src/notify.rs`) when
+`FORECAST_IN_COLLECTOR=true`, after each forecast run. It is a port of the
+Python notifier it replaced in 2026-10, with the same topics, texts and state
+keys (checked word for word on 751 generated messages), so the switch sent
+nothing twice. One difference: it also uses ThaiWater (HII) telemetry. For the stations whose HII twin
+is the same sensor (P.1, P.103, P.20, P.4A, P.67, P.75, P.82, P.84, P.92, and
+P.81 bias-corrected), a reading newer than RID's drives the level alerts; the
+message then says "ThaiWater telemetry; the RID report is not in yet". RID's
+report regularly lags 2-10 h at night, which used to send "gauge feed stale"
+around midnight and "recovered" in the morning. Now the feed is stale only
+when neither source has a reading for 3 h, and RID alone lagging is reported
+once after `RID_STALE_ALERT_HOURS` (18, 0 = off). With `MATRIX_*` set, the
+same transitions also go to a Matrix room, each once (per station, outlook,
+feed; not the per-topic summaries), with their own `matrix:` state keys.
+
+It runs in the same process that serves the dashboard, right after the
+forecast, so it sees exactly the readings and forecasts the dashboard shows.
+Per-key last-sent state is stored in the `notification_state` table of the
+monitor's own database, so a restart or redeploy never re-sends and never
+misses a crossing that happened while the service was down (the next cycle
+compares against the persisted state).
+
+If ntfy is unreachable the transition is **not** recorded, so it is retried
+on the next cycle rather than silently lost. Any other failure in the notify
+step is logged and never reaches the collection loop.
+
+The dashboard's "🔔 Get alerts" button appears only when `NTFY_SERVER` is
+set; it reads `GET /api/notifications` and renders subscribe links
+(`ntfy://` deep links for the app, https links for the web UI).
+
+## Deployment
+
+On the monitor VPS, as root:
+
+```bash
+cd /opt/thailand-water-monitor
+NTFY_DOMAIN=ntfy.buildfor.life bash scripts/install_ntfy.sh
+```
+
+This installs the ntfy .deb, writes `/etc/ntfy/server.yml` (listen on the
+host's Tailscale address, port 2586; anonymous read, token-only write, 72 h
+message cache, signup/login/metrics off, tight visitor limits), enables the
+systemd unit,
+creates the `monitor` user with **write-only access to `ping-*`**, mints a
+token, and appends `NTFY_SERVER` (public URL for subscribers),
+`NTFY_PUBLISH_URL` (loopback, what the monitor POSTs to), `NTFY_TOPIC_PREFIX`
+and `NTFY_TOKEN` to `.env` if they are not there yet. Then:
+
+```bash
+systemctl restart ping-collector
+journalctl -u ping-collector -n 20 | grep ntfy    # "ntfy notifications: publish to ..., topics ping-*"
+curl -s 'https://ntfy.buildfor.life/ping-status/json?poll=1'   # anonymous read works
+```
+
+The reverse proxy is a separate VPS on the same tailnet, so ntfy listens on
+the monitor host's Tailscale address and nothing is exposed on a public
+interface. On the Caddy machine:
+
+```caddyfile
+ntfy.buildfor.life {
+    reverse_proxy <monitor tailscale ip>:2586
+}
+```
+
+Caddy proxies websockets and keeps long-poll connections open by default;
+subscribers hold one open. `behind-proxy: true` makes ntfy rate-limit on
+`X-Forwarded-For` rather than treating every subscriber as the proxy.
+
+Publishing does not depend on the domain: `NTFY_PUBLISH_URL` points the
+monitor at the Tailscale address directly, so a DNS or proxy problem never
+holds back an alert. Test the pipeline before the domain is live with
+`curl -s 'http://<tailscale ip>:2586/ping-status/json?poll=1'`.
+
+## Configuration
+
+| Variable | Default | Meaning |
+|---|---|---|
+| `NTFY_SERVER` | *(empty = off)* | public base URL subscribers use; shown on the dashboard |
+| `NTFY_PUBLISH_URL` | = `NTFY_SERVER` | where the monitor POSTs; the local ntfy address (`http://<tailscale ip>:2586`), so publishing never waits on DNS/proxy |
+| `NTFY_TOPIC_PREFIX` | `ping` | first segment of every topic |
+| `NTFY_TOKEN` | *(empty)* | bearer token if the server requires auth to publish (it does, see above) |
+| `PUBLIC_URL` | `https://water.buildfor.life/` | click-through target in messages |
+
+| `RID_STALE_ALERT_HOURS` | `18` | report RID lagging this long while ThaiWater is current (0 = never) |
+| `MATRIX_HOMESERVER` / `MATRIX_ACCESS_TOKEN` / `MATRIX_ROOM_ID` | *(empty = off)* | the team room (docs/MATRIX_QUICK_START.md) |
+
+Tunables in `src/ml/export.py` (compiled into ping-collector through
+`rust/forecast/flood_config.json`: regenerate with
+`uv run python -m src.ml.export --config` and rebuild): `CLEAR_MARGIN_M`
+(0.10), `OUTLOOK_ON` / `OUTLOOK_OFF` (0.50 / 0.25), `FEED_STALE_AFTER_H` (3),
+the capacity guard and the station names in the messages.
+
+## Testing
+
+The tests in `rust/collector/src/notify.rs` cover the state machine: quiet
+river sends nothing; crossing once, then silence while above, then all-clear;
+hysteresis on the way down; escalation to danger and back; outlook on/off with
+heuristic forecasts ignored; HII readings replacing a lagging RID reading;
+stale feed only when both sources are; RID lagging reported once; a failed
+publish retried next cycle; the Matrix rendering; and the 652-step replay of
+the Python notifier's messages. `notification_state` round-trips against
+PostgreSQL in `db_tests.rs`.
+
+`ping-collector notify --dry-run` shows the readings judged and what would be
+sent right now, changing nothing. To exercise the real path against a real
+ntfy locally: run `ntfy serve` (any platform, same binary), point
+`NTFY_SERVER`/`NTFY_TOKEN` at it and poll the topic JSON.
+
+## Why ntfy and not …
+
+- **Matrix**: needs a homeserver account per subscriber and a room invite;
+  fine for a team, wrong for the public. The team room gets the same
+  transitions from ping-collector (docs/MATRIX_QUICK_START.md).
+- **Gotify**: also self-hosted and light, but Android-only client and one
+  account per subscriber.
+- **Email / SMS**: deliverability work, cost per message, no priority
+  semantics; ntfy can forward to email per subscription if someone wants it.
+- **Telegram / LINE bots**: platform lock-in and a bot token in the loop; can be
+  added later as ntfy→webhook fan-out without touching the monitor.

docs/PROJECT_STATUS.md

@@ -1,206 +0,0 @@
-# Thailand Water Monitor - Current Project Status
-
-## 📁 **Clean Project Structure**
-
-The project has been cleaned up and organized with the following structure:
-
-```
-water_level_monitor/
-├── 📄 .gitignore                             # Git ignore rules
-├── 📄 README.md                              # Main project documentation
-├── 📄 requirements.txt                       # Python dependencies
-├── 📄 config.py                              # Configuration management
-├── 📄 water_scraper_v3.py                    # Main application (15-min scheduler)
-├── 📄 database_adapters.py                   # Multi-database support
-├── 📄 demo_databases.py                      # Database demonstration
-├── 📄 Dockerfile                             # Container configuration
-├── 📄 docker-compose.victoriametrics.yml     # VictoriaMetrics stack
-├── 📚 Documentation/
-│   ├── 📄 DATABASE_DEPLOYMENT_GUIDE.md       # Multi-database setup guide
-│   ├── 📄 DEBIAN_TROUBLESHOOTING.md          # Linux deployment guide
-│   ├── 📄 ENHANCED_SCHEDULER_GUIDE.md        # 15-minute scheduler guide
-│   ├── 📄 GAP_FILLING_GUIDE.md              # Data gap filling guide
-│   ├── 📄 HTTPS_CONFIGURATION.md            # HTTPS setup guide
-│   └── 📄 VICTORIAMETRICS_SETUP.md          # VictoriaMetrics guide
-└── 📁 grafana/                              # Grafana configuration
-    ├── 📁 provisioning/
-    │   ├── 📁 datasources/
-    │   │   └── 📄 victoriametrics.yml        # VictoriaMetrics data source
-    │   └── 📁 dashboards/
-    │       └── 📄 dashboard.yml              # Dashboard provider config
-    └── 📁 dashboards/
-        └── 📄 water-monitoring-dashboard.json # Pre-built dashboard
-```
-
-## 🧹 **Files Removed During Cleanup**
-
-### **Old Data Files**
-- ❌ `thailand_water_data_v2.csv` - Old CSV export
-- ❌ `water_monitor.log` - Log file (regenerated automatically)
-- ❌ `water_monitoring.db` - SQLite database (recreated automatically)
-
-### **Outdated Documentation**
-- ❌ `FINAL_SUMMARY.md` - Contained references to non-existent v2 files
-- ❌ `PROJECT_SUMMARY.md` - Outdated project information
-
-### **System Files**
-- ❌ `__pycache__/` - Python compiled files directory
-
-## ✅ **Current Features**
-
-### **Enhanced 15-Minute Scheduler**
-- **Timing**: Runs every 15 minutes (1:00, 1:15, 1:30, 1:45, 2:00, etc.)
-- **Full Checks**: At :00 minutes (gap filling + data updates)
-- **Quick Checks**: At :15, :30, :45 minutes (data fetch only)
-- **Gap Filling**: Automatically fills missing historical data
-- **Data Updates**: Updates existing records when values change
-
-### **Multi-Database Support**
-- **VictoriaMetrics** (Recommended) - High-performance time-series
-- **InfluxDB** - Purpose-built time-series database
-- **PostgreSQL + TimescaleDB** - Relational with time-series optimization
-- **MySQL** - Traditional relational database
-- **SQLite** - Local development and testing
-
-### **Production Features**
-- **Docker Support**: Complete containerization
-- **Grafana Integration**: Pre-built dashboards
-- **HTTPS Configuration**: Secure deployment options
-- **Health Monitoring**: Comprehensive logging and error handling
-- **Gap Detection**: Automatic identification of missing data
-- **Retry Logic**: Database lock handling and network error recovery
-
-## 🚀 **Quick Start**
-
-### **1. Basic Setup (SQLite)**
-```bash
-cd water_level_monitor
-pip install -r requirements.txt
-python water_scraper_v3.py
-```
-
-### **2. VictoriaMetrics Setup**
-```bash
-# Start VictoriaMetrics + Grafana
-docker-compose -f docker-compose.victoriametrics.yml up -d
-
-# Configure environment
-export DB_TYPE=victoriametrics
-export VM_HOST=localhost
-export VM_PORT=8428
-
-# Run monitor
-python water_scraper_v3.py
-```
-
-### **3. Test Different Databases**
-```bash
-# Test all supported databases
-python demo_databases.py all
-
-# Test specific database
-python demo_databases.py victoriametrics
-```
-
-## 📊 **Data Collection**
-
-### **Station Coverage**
-- **16 Water Monitoring Stations** across Thailand
-- **Accurate Station Codes**: P.1, P.20, P.21, P.4A, P.5, P.67, P.75, P.76, P.77, P.81, P.82, P.84, P.85, P.87, P.92, P.103
-- **Bilingual Names**: Thai and English station identification
-
-### **Metrics Collected**
-- 🌊 **Water Level**: Measured in meters (m)
-- 💧 **Discharge**: Measured in cubic meters per second (cms)
-- 📊 **Discharge Percentage**: Relative to station capacity
-- ⏰ **Timestamp**: Hour 24 handling (midnight = 00:00 next day)
-
-### **Data Frequency**
-- **Every 15 Minutes**: Continuous monitoring
-- **~300+ Data Points**: Per collection cycle
-- **Automatic Gap Filling**: Historical data recovery
-- **Data Updates**: Changed values detection and correction
-
-## 🔧 **Command Line Tools**
-
-### **Main Application**
-```bash
-python water_scraper_v3.py                    # Run continuous monitoring
-python water_scraper_v3.py --test             # Single test cycle
-python water_scraper_v3.py --help             # Show help
-```
-
-### **Gap Management**
-```bash
-python water_scraper_v3.py --check-gaps [days]  # Check for missing data
-python water_scraper_v3.py --fill-gaps [days]   # Fill missing data gaps
-python water_scraper_v3.py --update-data [days] # Update existing data
-```
-
-### **Database Testing**
-```bash
-python demo_databases.py                      # SQLite demo
-python demo_databases.py victoriametrics      # VictoriaMetrics demo
-python demo_databases.py all                  # Test all databases
-```
-
-## 📈 **Monitoring & Visualization**
-
-### **Grafana Dashboard**
-- **URL**: http://localhost:3000 (when using docker-compose)
-- **Username**: admin
-- **Password**: admin_password
-- **Features**: Time series charts, status tables, gauges, alerts
-
-### **VictoriaMetrics API**
-- **URL**: http://localhost:8428
-- **Health**: http://localhost:8428/health
-- **Metrics**: http://localhost:8428/metrics
-- **Query API**: http://localhost:8428/api/v1/query
-
-## 🛡️ **Security & Production**
-
-### **HTTPS Configuration**
-- Complete guide in `HTTPS_CONFIGURATION.md`
-- SSL certificate setup
-- Reverse proxy configuration
-- Security best practices
-
-### **Deployment Options**
-- **Docker**: Containerized deployment
-- **Systemd**: Linux service configuration
-- **Cloud**: AWS, GCP, Azure deployment guides
-- **Monitoring**: Health checks and alerting
-
-## 📚 **Documentation**
-
-### **Available Guides**
-1. **README.md** - Main project documentation
-2. **DATABASE_DEPLOYMENT_GUIDE.md** - Multi-database setup
-3. **ENHANCED_SCHEDULER_GUIDE.md** - 15-minute scheduler details
-4. **GAP_FILLING_GUIDE.md** - Data integrity and gap filling
-5. **DEBIAN_TROUBLESHOOTING.md** - Linux deployment troubleshooting
-6. **VICTORIAMETRICS_SETUP.md** - VictoriaMetrics configuration
-7. **HTTPS_CONFIGURATION.md** - Secure deployment setup
-
-### **Key Features Documented**
-- ✅ Installation and configuration
-- ✅ Multi-database support
-- ✅ 15-minute scheduling system
-- ✅ Gap filling and data integrity
-- ✅ Production deployment
-- ✅ Monitoring and troubleshooting
-- ✅ Security configuration
-
-## 🎯 **Project Status: PRODUCTION READY**
-
-The Thailand Water Monitor is now:
-- ✅ **Clean**: All old and redundant files removed
-- ✅ **Organized**: Clear project structure with proper documentation
-- ✅ **Enhanced**: 15-minute scheduling with gap filling
-- ✅ **Scalable**: Multi-database support with VictoriaMetrics
-- ✅ **Secure**: HTTPS configuration and security best practices
-- ✅ **Monitored**: Comprehensive logging and Grafana dashboards
-- ✅ **Documented**: Complete guides for all features and deployment options
-
-The project is ready for production deployment with professional-grade monitoring capabilities.

docs/PROJECT_STRUCTURE.md

@@ -1,272 +0,0 @@
-# 🏗️ Project Structure - Northern Thailand Ping River Monitor
-
-## 📁 Directory Layout
-
-```
-Northern-Thailand-Ping-River-Monitor/
-├── 📁 src/                          # Main application source code
-│   ├── __init__.py                  # Package initialization
-│   ├── main.py                      # CLI entry point and main application
-│   ├── water_scraper_v3.py          # Core data collection engine
-│   ├── web_api.py                   # FastAPI web interface
-│   ├── config.py                    # Configuration management
-│   ├── database_adapters.py         # Database abstraction layer
-│   ├── models.py                    # Data models and type definitions
-│   ├── exceptions.py                # Custom exception classes
-│   ├── validators.py                # Data validation layer
-│   ├── metrics.py                   # Metrics collection system
-│   ├── health_check.py              # Health monitoring system
-│   ├── rate_limiter.py              # Rate limiting and request tracking
-│   └── logging_config.py            # Enhanced logging configuration
-├── 📁 docs/                         # Documentation files
-│   ├── STATION_MANAGEMENT_GUIDE.md  # Station management documentation
-│   ├── ENHANCEMENT_SUMMARY.md       # Feature enhancement summary
-│   └── PROJECT_STRUCTURE.md         # This file
-├── 📁 scripts/                      # Utility scripts
-│   └── migrate_geolocation.py       # Database migration script
-├── 📁 grafana/                      # Grafana configuration
-│   ├── dashboards/                  # Dashboard definitions
-│   └── provisioning/                # Grafana provisioning config
-├── 📁 tests/                        # Test files
-│   ├── test_integration.py          # Integration test suite
-│   ├── test_station_management.py   # Station management tests
-│   └── test_api.py                  # API endpoint tests
-├── 📄 run.py                        # Simple startup script
-├── 📄 requirements.txt              # Production dependencies
-├── 📄 requirements-dev.txt          # Development dependencies
-├── 📄 setup.py                      # Package installation script
-├── 📄 Dockerfile                    # Docker container definition
-├── 📄 docker-compose.victoriametrics.yml  # Complete stack deployment
-├── 📄 Makefile                      # Common development tasks
-├── 📄 .env.example                  # Environment configuration template
-├── 📄 .gitignore                    # Git ignore patterns
-├── 📄 .gitlab-ci.yml                # CI/CD pipeline configuration
-├── 📄 LICENSE                       # MIT license
-├── 📄 README.md                     # Main project documentation
-└── 📄 CONTRIBUTING.md               # Contribution guidelines
-```
-
-## 🔧 Core Components
-
-### **Application Layer**
-- **`src/main.py`** - Command-line interface and application orchestration
-- **`src/web_api.py`** - FastAPI web interface with REST endpoints
-- **`src/water_scraper_v3.py`** - Core data collection and processing engine
-
-### **Data Layer**
-- **`src/database_adapters.py`** - Multi-database support (SQLite, MySQL, PostgreSQL, InfluxDB, VictoriaMetrics)
-- **`src/models.py`** - Pydantic data models and type definitions
-- **`src/validators.py`** - Data validation and sanitization
-
-### **Infrastructure Layer**
-- **`src/config.py`** - Configuration management with environment variable support
-- **`src/logging_config.py`** - Structured logging with rotation and colors
-- **`src/metrics.py`** - Application metrics collection (counters, gauges, histograms)
-- **`src/health_check.py`** - System health monitoring and status checks
-
-### **Utility Layer**
-- **`src/exceptions.py`** - Custom exception hierarchy
-- **`src/rate_limiter.py`** - API rate limiting and request tracking
-
-## 🌐 Web API Structure
-
-### **Endpoints Organization**
-```
-/                           # Dashboard homepage
-├── /health                 # System health status
-├── /metrics               # Application metrics
-├── /config                # Configuration (masked)
-├── /stations              # Station management
-│   ├── GET /              # List all stations
-│   ├── POST /             # Create new station
-│   ├── GET /{id}          # Get specific station
-│   ├── PUT /{id}          # Update station
-│   └── DELETE /{id}       # Delete station
-├── /measurements          # Data access
-│   ├── /latest            # Latest measurements
-│   └── /station/{code}    # Station-specific data
-└── /scraping              # Data collection control
-    ├── /trigger           # Manual data collection
-    └── /status            # Scraping status
-```
-
-### **API Models**
-- **Request Models**: Station creation/update, query parameters
-- **Response Models**: Station info, measurements, health status
-- **Error Models**: Standardized error responses
-
-## 🗄️ Database Architecture
-
-### **Supported Databases**
-1. **SQLite** - Local development and testing
-2. **MySQL** - Traditional relational database
-3. **PostgreSQL** - Advanced relational with TimescaleDB support
-4. **InfluxDB** - Purpose-built time-series database
-5. **VictoriaMetrics** - High-performance metrics storage
-
-### **Schema Design**
-```sql
--- Stations table
-stations (
-    id INTEGER PRIMARY KEY,
-    station_code VARCHAR(10) UNIQUE,
-    thai_name VARCHAR(255),
-    english_name VARCHAR(255),
-    latitude DECIMAL(10,8),
-    longitude DECIMAL(11,8),
-    geohash VARCHAR(20),
-    status VARCHAR(20),
-    created_at TIMESTAMP,
-    updated_at TIMESTAMP
-)
-
--- Measurements table
-water_measurements (
-    id BIGINT PRIMARY KEY,
-    timestamp DATETIME,
-    station_id INTEGER,
-    water_level DECIMAL(10,3),
-    discharge DECIMAL(10,2),
-    discharge_percent DECIMAL(5,2),
-    status VARCHAR(20),
-    created_at TIMESTAMP,
-    FOREIGN KEY (station_id) REFERENCES stations(id),
-    UNIQUE(timestamp, station_id)
-)
-```
-
-## 🐳 Docker Architecture
-
-### **Multi-Stage Build**
-1. **Builder Stage** - Compile dependencies and build artifacts
-2. **Production Stage** - Minimal runtime environment
-
-### **Service Composition**
-- **ping-river-monitor** - Data collection service
-- **ping-river-api** - Web API service
-- **victoriametrics** - Time-series database
-- **grafana** - Visualization dashboard
-
-## 📊 Monitoring Architecture
-
-### **Metrics Collection**
-- **Counters** - API requests, database operations, scraping cycles
-- **Gauges** - Current values, connection status, resource usage
-- **Histograms** - Response times, processing durations
-
-### **Health Checks**
-- **Database Health** - Connection status, data freshness
-- **API Health** - External API availability, response times
-- **System Health** - Memory usage, disk space, CPU load
-
-### **Logging Levels**
-- **DEBUG** - Detailed execution information
-- **INFO** - General operational messages
-- **WARNING** - Potential issues and recoverable errors
-- **ERROR** - Serious problems requiring attention
-- **CRITICAL** - System-threatening issues
-
-## 🔧 Configuration Management
-
-### **Environment Variables**
-```bash
-# Database
-DB_TYPE=victoriametrics
-VM_HOST=localhost
-VM_PORT=8428
-
-# Application
-SCRAPING_INTERVAL_HOURS=1
-LOG_LEVEL=INFO
-DATA_RETENTION_DAYS=365
-
-# Security
-SECRET_KEY=your-secret-key
-API_KEY=your-api-key
-```
-
-### **Configuration Hierarchy**
-1. Environment variables (highest priority)
-2. .env file
-3. Default values in config.py (lowest priority)
-
-## 🧪 Testing Architecture
-
-### **Test Categories**
-- **Unit Tests** - Individual component testing
-- **Integration Tests** - System component interaction
-- **API Tests** - Endpoint functionality and responses
-- **Performance Tests** - Load and stress testing
-
-### **Test Data**
-- **Mock Data** - Simulated API responses
-- **Test Database** - Isolated test environment
-- **Fixtures** - Reusable test data sets
-
-## 📦 Deployment Architecture
-
-### **Development**
-```bash
-python run.py --web-api    # Local development server
-```
-
-### **Production**
-```bash
-docker-compose up -d       # Full stack deployment
-```
-
-### **CI/CD Pipeline**
-1. **Test Stage** - Run all tests and quality checks
-2. **Build Stage** - Create Docker images
-3. **Deploy Stage** - Deploy to staging/production
-4. **Health Check** - Verify deployment success
-
-## 🔒 Security Architecture
-
-### **Input Validation**
-- Pydantic models for API requests
-- Data range validation for measurements
-- SQL injection prevention through ORM
-
-### **Authentication** (Future)
-- API key authentication
-- JWT token support
-- Role-based access control
-
-### **Data Protection**
-- Environment variable configuration
-- Sensitive data masking in logs
-- HTTPS support for production
-
-## 📈 Performance Architecture
-
-### **Optimization Strategies**
-- Database connection pooling
-- Query optimization and indexing
-- Response caching for static data
-- Async processing for I/O operations
-
-### **Scalability Considerations**
-- Horizontal scaling with load balancers
-- Database read replicas
-- Microservice architecture readiness
-- Container orchestration support
-
-## 🔄 Data Flow Architecture
-
-### **Collection Flow**
-```
-External API → Rate Limiter → Data Validator → Database Adapter → Database
-```
-
-### **API Flow**
-```
-HTTP Request → FastAPI → Business Logic → Database Adapter → HTTP Response
-```
-
-### **Monitoring Flow**
-```
-Application Events → Metrics Collector → Health Checks → Monitoring Dashboard
-```
-
-This architecture provides a solid foundation for a production-ready water monitoring system with excellent maintainability, scalability, and observability.

docs/RUST_REWRITE.md

@@ -0,0 +1,162 @@
+# The Rust rewrite
+
+Status: **done** (2026-10-18). The service is `ping-collector`
+([rust/README.md](../rust/README.md)); Python keeps model training,
+evaluation and the archive backfills. Phases 1-3 replaced the Python
+collector, the forecast + notifier and the website/API one at a time, each
+checked against the Python side on production data before the switch;
+phase 4 removed the Python service code. The migration tooling
+(`scripts/collector_parity.py`, `scripts/api_parity.py`,
+`scripts/install_collector.sh` with its cutover/rollback modes, the
+`spikes/rust-forecast` spike) is in git history before the cleanup commit.
+
+The assessment below was written before phase 1 and is kept as the record of
+why and how.
+
+## Short version
+
+- **Feasible.** The risky part, serving the scikit-learn forecast from Rust with
+  no train/serve skew, works: the spike reproduces the pandas feature matrix
+  and the HistGradientBoosting regression output bit for bit (classifier heads
+  to the last ulp).
+- **Training stays Python.** There is no Rust HGB that trains like
+  scikit-learn, and the evaluation harness (`scripts/evaluate_variants.py`) is
+  where model work happens. A "Rust rewrite" means a Rust service (collectors,
+  API, forecast serving, ntfy) plus a Python offline job (monthly retrain and
+  evaluation) that exports models for Rust.
+- **Performance is not a reason.** The forecast takes 158 ms per station in Python
+  and 0.25 ms in Rust, but it runs once an hour. Load tests showed the site
+  is bandwidth-bound once caches are hit.
+- **Real reasons, if any:** one process instead of leader-elected uvicorn workers
+  (the root of per-worker state bugs like `/scraping/status`), a fraction of the
+  memory (324 MB today), typed parsing and SQL, one deployable binary, and the
+  same stack as proplan (axum + sqlx).
+- **Recommendation:** no big-bang rewrite. If Rust is wanted, start with the
+  collector + notifier daemon (phase 1 below), which stands alone and can be
+  abandoned without loss. The cheap Python alternative is at the end.
+
+## What exists (Python, `src/`)
+
+| Area | LOC | Fate in Rust |
+| --- | ---: | --- |
+| Collectors: RID hourly, HII/ThaiWater, RID dams, Open-Meteo | 2,500 | port; typed serde payloads |
+| Web API (20 routes), schemas, health, metrics | 2,300 | port with identical JSON (golden-file tests) |
+| DB adapters: Postgres, SQLite, MySQL, InfluxDB, VictoriaMetrics | 1,300 | Postgres only via sqlx, existing 13 tables, no data migration |
+| ntfy notifications; Matrix alerting (CLI path) | 1,050 | port ntfy; Matrix only if still used |
+| ML serving: predict, features, rain/dam/HII fill | ~1,300 | port; parity-tested (spike) |
+| ML training + evaluation | ~1,700 | **stays Python** |
+| CLI (`main.py`) | 650 | small clap CLI (`fill-gaps`, test cycle) |
+| `static/dashboard.html` (164 KB) | | unchanged, served as a static file |
+| tests | 3,900 | Rust unit tests + parity fixtures + API golden files |
+
+## Target shape
+
+One binary, one process (tokio):
+
+- tasks: RID / HII / dams / rain collectors on their schedules; forecast
+  precompute when new readings land; ntfy notifier. Shared state is an
+  `Arc<RwLock<..>>`, so there's no leader election, no status hand-off file and
+  no executor sizing.
+- axum + tower-http: same routes and JSON, compression (gzip/brotli also covers
+  the 257 KB river GeoJSON), static dashboard, CORS; `utoipa` if `/docs` stays.
+- sqlx against the existing schema. Python and Rust can run against the same
+  database during migration.
+- models: `train_all()` also exports each bundle (`models/flood_<station>.json.gz`,
+  1.2-2.3 MB), `scripts/retrain.sh` promotes it. Rust reloads changed files on
+  its next run. A bundle whose `feature_names` Rust cannot build falls back to
+  the heuristic, as Python does.
+
+## The permanent cost: the Python/Rust model contract
+
+Every feature added in `features.py` must be ported to Rust in the same change,
+or the new bundle is refused. That has to be enforced by CI: Python writes a parity
+fixture per station (as `export_fixture.py` does), and a Rust test asserts
+bit-identical features and regression output. The spike found two traps that
+silently flip tree branches (HGB thresholds often equal real data values):
+serde_json's default float parsing (1 ulp off; enable `float_roundtrip`) and
+pandas' Kahan-compensated rolling sums/means (replicated in `rolling_kahan()`).
+This slows model iteration, which is the most active part of the repo.
+
+## Benefits and costs
+
+| For | Against |
+| --- | --- |
+| Single process: no leader election, no per-worker state | Two languages forever; parity CI is mandatory |
+| Memory ~324 MB -> est. 60-80 MB (mostly tree arrays; less with u32/u16 nodes) | Weeks of work with little user-visible change |
+| Typed RID/HII parsing, compile-checked SQL | Re-introduces bugs into recently hardened code (tz, alerts, security) |
+| One static binary; no venv on the server for the API | Server still needs Python + sklearn for the monthly retrain |
+| Same stack as proplan | Feature work = Python + Rust change every time |
+
+## Plan (each phase ships on its own)
+
+| Phase | Scope | Status |
+| --- | --- | --- |
+| 0 | Spike: P.1 features + HGB inference | done, bit-identical |
+| 1 | `ping-collector`: RID, HII, dams, Open-Meteo | in production since 2026-10-17 23:09 (Python: `COLLECTION_ENABLED=false`) |
+| 2 | Forecast crate + ntfy | in production since 2026-10-18 15:08 (`FORECAST_IN_COLLECTOR=true`); live check on production data: 48/48 rows identical, 96/96 probabilities bit-identical |
+| 3 | Website + read API in `ping-collector`, Matrix like ntfy | in production since 2026-10-18 16:40 (`WEB_LISTEN`, `water-monitor.service` disabled); on production data 53 of 53 compared routes identical to FastAPI (`scripts/api_parity.py`) |
+| 4 | Cleanup: Python service code removed | done 2026-10-18 |
+
+0. **Spike: done.** P.1 features + HGB inference, bit-identical.
+1. **Collector daemon** (`rust/collector`, ~1.5k LOC incl. tests): RID with the
+   fixed semantics (store late readings, retry while behind, re-fetch
+   yesterday's holes), HII, dams, Open-Meteo, into the same tables. The Python
+   API runs with `COLLECTION_ENABLED=false` and re-forecasts when the newest
+   RID reading in the database advances. ntfy moved to phase 2: the outlook
+   alert needs the forecasts, which are still Python. Cutover and rollback
+   were one command each (`scripts/install_collector.sh`, since removed).
+2. **Forecast crate + ntfy** (`rust/forecast`, run by `ping-collector` with
+   `FORECAST_IN_COLLECTOR=true`): all stations, HII/rain/dam inputs, heuristic
+   fallback, probabilities and stages, `forecast_history`, `forecast_current`
+   (served by the API) and `notification_state`. Parity: fixtures from the
+   Python production path (CI: P.1 model; locally all 15), a live comparison
+   script, and 751 notifications replayed word for word. Porting found a
+   Python serving bug, fixed first: forecast rain (`rain_fc24`) was always
+   NaN at serving time (docs/FLOOD_FORECASTING.md section 5). The ntfy change:
+   ThaiWater mirror readings keep level alerts and the feed check current
+   while RID lags at night. The Python API then only serves.
+3. **Website + read API** (`rust/collector/src/web`, axum): every FastAPI
+   route with the same parameters, JSON fields and order, status codes,
+   FastAPI-style errors (`{"detail": ...}`, pydantic's 422 list), cache
+   headers, gzip, CORS, Umami API tracking and the stale-while-revalidate
+   response caches (now with byte budgets). `/docs` is the same Swagger UI;
+   its OpenAPI document is generated from the route table whose parameter
+   declarations also do the validation. One process serves, collects,
+   forecasts and notifies (`WEB_LISTEN`), so the status file, the leader
+   election and the `forecast_current` hand-off between processes are gone
+   in effect. Checked by `scripts/api_parity.py`, which runs `ping-collector
+   serve` next to the FastAPI app on the same database and compares every
+   route (query values vary per run so FastAPI's caches answer fresh);
+   `--cutover-api` refuses to switch on a difference. Intended differences:
+   `/health` counts ThaiWater telemetry of the same gauges while RID lags (as
+   the notifier); `/scraping/status` adds the collector version and the last
+   forecast run; `/forecast` serves an old stored forecast marked
+   `X-Data-Stale` instead of recomputing (same process); `/sensors/thaiwater`
+   is cached for a minute; database errors serve the last good response
+   where FastAPI answered `[]`. Matrix: the notifier's transitions go to the
+   room too (per station, outlook, feed; not the per-topic summaries), with
+   their own `matrix:` state keys seeded from ntfy's, replacing the
+   `run.py --alert-check` cron job that repeated alerts every 15 minutes.
+   Python keeps training, evaluation and the monthly retrain timer.
+4. **Cleanup** (done): FastAPI/uvicorn, the Python collector, notifier, Matrix
+   alerting, CLI (`run.py`), the multi-backend adapters (SQLite, MySQL,
+   InfluxDB, VictoriaMetrics), demo databases, Docker/Grafana files and the
+   migration scripts are gone, about 17k lines. The Python environment is the
+   training stack (pandas, scikit-learn, SQLAlchemy/psycopg2): `src/ml`, the
+   backfill clients/stores (`src/hii_collector.py`, `src/rid_reservoir.py`)
+   and `src/ml/predict.py` as the reference implementation the Rust forecaster
+   is tested against. PostgreSQL only; the Python database tests run against
+   a throwaway schema (`TEST_DATABASE_URL`, as the Rust ones). One installer,
+   `scripts/install.sh`.
+
+Decided before phase 3 (2026-10-18): Matrix stays and behaves like ntfy;
+`/docs` stays; SQLite dev mode goes; the monthly retrain stays on the same
+machine (Python, `water-monitor-retrain.timer`).
+
+## Cheaper alternative (stay Python)
+
+Most of the operational gain comes from taking collection out of the web
+workers: run the collector as its own systemd unit (one process) and the
+uvicorn workers as read-only API. That's about a day of work. It removes leader
+election and per-worker state, but doesn't bring the memory, typing or
+single-binary benefits.

docs/STATION_MANAGEMENT_GUIDE.md

@@ -1,241 +0,0 @@
-# 🏔️ Station Management Guide - Northern Thailand Ping River Monitor
-
-## 🎯 **Overview**
-
-The Northern Thailand Ping River Monitor now includes comprehensive station management capabilities, allowing you to dynamically add, update, and remove monitoring stations through the web API.
-
-## 🌊 **Current Coverage**
-
-The system currently monitors **16 water stations** along the Ping River Basin:
-
-### **Upper Ping River (Chiang Mai Province)**
-- **P.20** - Ban Chiang Dao (บ้านเชียงดาว)
-- **P.75** - Ban Chai Lat (บ้านช่อแล)
-- **P.92** - Ban Muang Aut (บ้านเมืองกึ๊ด)
-- **P.4A** - Ban Mae Taeng (บ้านแม่แตง)
-- **P.67** - Ban Tae (บ้านแม่แต)
-- **P.21** - Ban Rim Tai (บ้านริมใต้)
-- **P.103** - Ring Bridge 3 (สะพานวงแหวนรอบ 3)
-
-### **Middle Ping River**
-- **P.1** - Nawarat Bridge (สะพานนวรัฐ) - *Main reference station*
-- **P.82** - Ban Sob win (บ้านสบวิน)
-- **P.84** - Ban Panton (บ้านพันตน)
-- **P.81** - Ban Pong (บ้านโป่ง)
-- **P.5** - Tha Nang Bridge (สะพานท่านาง)
-
-### **Lower Ping River**
-- **P.77** - Baan Sop Mae Sapuord (บ้านสบแม่สะป๊วด)
-- **P.87** - Ban Pa Sang (บ้านป่าซาง)
-- **P.76** - Banb Mae I Hai (บ้านแม่อีไฮ)
-- **P.85** - Baan Lai Kaew (บ้านหล่ายแก้ว)
-
-## 🔧 **Station Management API**
-
-### **List All Stations**
-```bash
-GET /stations
-```
-
-**Response:**
-```json
-[
-  {
-    "station_id": 1,
-    "station_code": "P.20",
-    "thai_name": "บ้านเชียงดาว",
-    "english_name": "Ban Chiang Dao",
-    "latitude": 19.36731448032191,
-    "longitude": 98.9688487015384,
-    "geohash": null,
-    "status": "active"
-  }
-]
-```
-
-### **Get Specific Station**
-```bash
-GET /stations/{station_id}
-```
-
-### **Add New Station**
-```bash
-POST /stations
-Content-Type: application/json
-
-{
-  "station_code": "P.NEW",
-  "thai_name": "สถานีใหม่",
-  "english_name": "New Station",
-  "latitude": 18.7875,
-  "longitude": 99.0045,
-  "geohash": "w5q6uuhvfcfp25",
-  "status": "active"
-}
-```
-
-### **Update Station Information**
-```bash
-PUT /stations/{station_id}
-Content-Type: application/json
-
-{
-  "thai_name": "ชื่อใหม่",
-  "english_name": "Updated Name",
-  "latitude": 18.8000,
-  "longitude": 99.0100
-}
-```
-
-### **Delete Station**
-```bash
-DELETE /stations/{station_id}
-```
-
-## 🧪 **Testing Station Management**
-
-Use the provided test script to verify station management functionality:
-
-```bash
-# Test all station management endpoints
-python test_station_management.py
-```
-
-This will:
-1. List existing stations
-2. Create a test station
-3. Retrieve station details
-4. Update station information
-5. Verify changes
-6. Delete the test station
-7. Confirm deletion
-
-## 📊 **Station Data Model**
-
-### **Required Fields**
-- `station_code`: Unique identifier (e.g., "P.1", "P.20")
-- `thai_name`: Thai language name
-- `english_name`: English language name
-
-### **Optional Fields**
-- `latitude`: GPS latitude coordinate (-90 to 90)
-- `longitude`: GPS longitude coordinate (-180 to 180)
-- `geohash`: Geohash string for location
-- `status`: Station status ("active", "inactive", "maintenance", "error")
-
-### **Validation Rules**
-- Station codes must be unique
-- Latitude must be between -90 and 90
-- Longitude must be between -180 and 180
-- Names cannot be empty
-- Status must be valid enum value
-
-## 🌐 **Web Interface**
-
-Access the station management interface through the web dashboard:
-
-1. **Start the API server:**
-   ```bash
-   python run.py --web-api
-   ```
-
-2. **Open your browser:**
-   - Dashboard: http://localhost:8000
-   - API Documentation: http://localhost:8000/docs
-
-3. **Use the interactive API docs** to test station management endpoints
-
-## 🔄 **Integration with Data Collection**
-
-- **Dynamic Station Discovery**: New stations are automatically included in data collection
-- **Real-time Updates**: Station information changes are reflected immediately
-- **Data Continuity**: Historical data is preserved when updating station details
-- **Error Handling**: Invalid stations are skipped during data collection
-
-## 📍 **Geographic Coverage**
-
-The Ping River Basin monitoring network covers:
-
-- **Total Distance**: ~400 km from Chiang Dao to Nakhon Sawan
-- **Elevation Range**: 300m to 1,200m above sea level
-- **Catchment Area**: ~25,000 km²
-- **Major Cities**: Chiang Mai, Lamphun, Tak, Nakhon Sawan
-
-## 🚀 **Usage Examples**
-
-### **Add a New Upstream Station**
-```bash
-curl -X POST "http://localhost:8000/stations" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "station_code": "P.UPSTREAM",
-    "thai_name": "สถานีต้นน้ำ",
-    "english_name": "Upstream Station",
-    "latitude": 19.5000,
-    "longitude": 98.9000,
-    "status": "active"
-  }'
-```
-
-### **Update Station Coordinates**
-```bash
-curl -X PUT "http://localhost:8000/stations/1" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "latitude": 19.3700,
-    "longitude": 98.9700
-  }'
-```
-
-### **Mark Station for Maintenance**
-```bash
-curl -X PUT "http://localhost:8000/stations/5" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "status": "maintenance"
-  }'
-```
-
-## 🔒 **Best Practices**
-
-### **Station Naming**
-- Use consistent code format (P.XX)
-- Include both Thai and English names
-- Use descriptive location names
-
-### **Coordinate Accuracy**
-- Use high-precision GPS coordinates (6+ decimal places)
-- Verify coordinates match actual station location
-- Include geohash for efficient spatial queries
-
-### **Status Management**
-- Set status to "maintenance" during repairs
-- Use "inactive" for temporarily offline stations
-- Use "error" for stations with data quality issues
-
-### **Data Integrity**
-- Test new stations before adding to production
-- Backup station configuration before major changes
-- Monitor data quality after station updates
-
-## 🎯 **Future Enhancements**
-
-Planned improvements for station management:
-
-1. **Bulk Operations** - Import/export multiple stations
-2. **Station Groups** - Organize stations by river section
-3. **Automated Validation** - GPS coordinate verification
-4. **Historical Tracking** - Track station configuration changes
-5. **Alert Integration** - Notifications for station status changes
-6. **Map Interface** - Visual station management on interactive map
-
-## 📞 **Support**
-
-For station management issues:
-
-1. Check the API documentation at `/docs`
-2. Run the test script: `python test_station_management.py`
-3. Review logs for error details
-4. Verify station data format and validation rules
-
-The station management system provides flexible control over your monitoring network while maintaining data integrity and system reliability.

docs/VICTORIAMETRICS_SETUP.md

@@ -1,443 +0,0 @@
-# VictoriaMetrics Setup Guide for Thailand Water Monitor
-
-This guide provides comprehensive instructions for setting up VictoriaMetrics as the time-series database backend for the Thailand Water Monitor.
-
-## Why VictoriaMetrics?
-
-VictoriaMetrics is an excellent choice for water monitoring data because:
-
-- **High Performance**: Up to 10x faster than InfluxDB
-- **Low Resource Usage**: Uses 10x less RAM than Prometheus
-- **Better Compression**: 70x better compression than Prometheus
-- **Prometheus Compatible**: Drop-in replacement for Prometheus
-- **Easy to Deploy**: Single binary, no dependencies
-- **Cost Effective**: Open source with commercial support available
-
-## Quick Start
-
-### 1. Environment Variables
-
-Set these environment variables to configure VictoriaMetrics:
-
-```bash
-# Windows (PowerShell)
-$env:DB_TYPE="victoriametrics"
-$env:VM_HOST="localhost"
-$env:VM_PORT="8428"
-
-# Linux/Mac
-export DB_TYPE=victoriametrics
-export VM_HOST=localhost
-export VM_PORT=8428
-```
-
-### 2. Start VictoriaMetrics with Docker
-
-```bash
-# Simple setup
-docker run -d \
-  --name victoriametrics \
-  -p 8428:8428 \
-  -v victoria-metrics-data:/victoria-metrics-data \
-  victoriametrics/victoria-metrics:latest \
-  --storageDataPath=/victoria-metrics-data \
-  --retentionPeriod=2y \
-  --httpListenAddr=:8428
-
-# Verify it's running
-curl http://localhost:8428/health
-```
-
-### 3. Run the Water Monitor
-
-```bash
-python water_scraper_v3.py
-```
-
-### 4. Access Grafana Dashboard
-
-```bash
-# Start with Docker Compose (includes Grafana)
-docker-compose -f docker-compose.victoriametrics.yml up -d
-
-# Access Grafana at http://localhost:3000
-# Username: admin
-# Password: admin_password
-```
-
-## Production Setup
-
-### Docker Compose Configuration
-
-Use the provided `docker-compose.victoriametrics.yml` file:
-
-```bash
-# Start the complete stack
-docker-compose -f docker-compose.victoriametrics.yml up -d
-
-# Check status
-docker-compose -f docker-compose.victoriametrics.yml ps
-
-# View logs
-docker-compose -f docker-compose.victoriametrics.yml logs -f
-```
-
-### Manual VictoriaMetrics Configuration
-
-#### High-Performance Configuration
-
-```bash
-docker run -d \
-  --name victoriametrics \
-  -p 8428:8428 \
-  -v victoria-metrics-data:/victoria-metrics-data \
-  victoriametrics/victoria-metrics:latest \
-  --storageDataPath=/victoria-metrics-data \
-  --retentionPeriod=2y \
-  --httpListenAddr=:8428 \
-  --maxConcurrentInserts=32 \
-  --search.maxQueryDuration=60s \
-  --search.maxConcurrentRequests=16 \
-  --dedup.minScrapeInterval=30s \
-  --memory.allowedPercent=80 \
-  --loggerLevel=INFO \
-  --loggerFormat=json \
-  --search.maxSeries=1000000 \
-  --search.maxPointsPerTimeseries=100000
-```
-
-#### Configuration Parameters Explained
-
-| Parameter | Description | Recommended Value |
-|-----------|-------------|-------------------|
-| `--storageDataPath` | Data storage directory | `/victoria-metrics-data` |
-| `--retentionPeriod` | How long to keep data | `2y` (2 years) |
-| `--httpListenAddr` | HTTP listen address | `:8428` |
-| `--maxConcurrentInserts` | Max concurrent inserts | `32` |
-| `--search.maxQueryDuration` | Max query duration | `60s` |
-| `--search.maxConcurrentRequests` | Max concurrent queries | `16` |
-| `--dedup.minScrapeInterval` | Deduplication interval | `30s` |
-| `--memory.allowedPercent` | Max memory usage | `80` |
-| `--loggerLevel` | Log level | `INFO` |
-| `--search.maxSeries` | Max time series | `1000000` |
-
-### Kubernetes Deployment
-
-```yaml
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: victoriametrics
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: victoriametrics
-  template:
-    metadata:
-      labels:
-        app: victoriametrics
-    spec:
-      containers:
-      - name: victoriametrics
-        image: victoriametrics/victoria-metrics:latest
-        ports:
-        - containerPort: 8428
-        args:
-        - --storageDataPath=/victoria-metrics-data
-        - --retentionPeriod=2y
-        - --httpListenAddr=:8428
-        - --maxConcurrentInserts=32
-        volumeMounts:
-        - name: storage
-          mountPath: /victoria-metrics-data
-        resources:
-          requests:
-            memory: "512Mi"
-            cpu: "500m"
-          limits:
-            memory: "2Gi"
-            cpu: "2000m"
-      volumes:
-      - name: storage
-        persistentVolumeClaim:
-          claimName: victoriametrics-pvc
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: victoriametrics
-spec:
-  selector:
-    app: victoriametrics
-  ports:
-  - port: 8428
-    targetPort: 8428
-  type: ClusterIP
-```
-
-## Data Queries
-
-### HTTP API Queries
-
-VictoriaMetrics provides a Prometheus-compatible HTTP API:
-
-```bash
-# Current water levels for all stations
-curl "http://localhost:8428/api/v1/query?query=water_level"
-
-# Water levels for specific station
-curl "http://localhost:8428/api/v1/query?query=water_level{station_code=\"P.1\"}"
-
-# Average discharge over last hour
-curl "http://localhost:8428/api/v1/query?query=avg_over_time(water_discharge[1h])"
-
-# High discharge alerts (>80%)
-curl "http://localhost:8428/api/v1/query?query=water_discharge_percent>80"
-
-# Time range query (last 6 hours)
-START=$(date -d '6 hours ago' +%s)
-END=$(date +%s)
-curl "http://localhost:8428/api/v1/query_range?query=water_level&start=${START}&end=${END}&step=300"
-```
-
-### PromQL Examples
-
-```promql
-# Current water levels
-water_level
-
-# Water level trends (last 24h)
-water_level[24h]
-
-# Discharge rates by station
-water_discharge{station_code="P.1"}
-
-# Average discharge across all stations
-avg(water_discharge)
-
-# Stations with high discharge (>80%)
-water_discharge_percent > 80
-
-# Rate of change in water level
-rate(water_level[5m])
-
-# Maximum water level in last hour
-max_over_time(water_level[1h])
-
-# Stations with increasing water levels
-increase(water_level[1h]) > 0
-```
-
-## Grafana Integration
-
-### Data Source Configuration
-
-1. **Add VictoriaMetrics as Prometheus Data Source**:
-   - URL: `http://localhost:8428` (or `http://victoriametrics:8428` in Docker)
-   - Access: Server (default)
-   - HTTP Method: POST
-
-2. **Import Dashboard**:
-   - Use the provided `water-monitoring-dashboard.json`
-   - Or create custom dashboards with the queries above
-
-### Dashboard Panels
-
-The included dashboard provides:
-
-- **Time Series**: Water levels and discharge over time
-- **Table**: Current status of all stations
-- **Pie Chart**: Discharge percentage distribution
-- **Gauge**: Average discharge percentage
-- **Variables**: Filter by station
-
-## Monitoring and Maintenance
-
-### Health Checks
-
-```bash
-# Check VictoriaMetrics health
-curl http://localhost:8428/health
-
-# Check metrics endpoint
-curl http://localhost:8428/metrics
-
-# Check configuration
-curl http://localhost:8428/api/v1/status/config
-```
-
-### Performance Monitoring
-
-```bash
-# Query performance stats
-curl http://localhost:8428/api/v1/status/tsdb
-
-# Memory usage
-curl http://localhost:8428/api/v1/status/runtime
-
-# Active queries
-curl http://localhost:8428/api/v1/status/active_queries
-```
-
-### Backup and Restore
-
-```bash
-# Create backup
-docker exec victoriametrics /usr/bin/vmbackup \
-  -storageDataPath=/victoria-metrics-data \
-  -dst=fs:///backup/$(date +%Y%m%d)
-
-# Restore from backup
-docker exec victoriametrics /usr/bin/vmrestore \
-  -src=fs:///backup/20250724 \
-  -storageDataPath=/victoria-metrics-data
-```
-
-### Log Analysis
-
-```bash
-# View logs
-docker logs victoriametrics
-
-# Follow logs
-docker logs -f victoriametrics
-
-# Search for errors
-docker logs victoriametrics 2>&1 | grep ERROR
-```
-
-## Troubleshooting
-
-### Common Issues
-
-1. **Connection Refused**:
-   ```bash
-   # Check if VictoriaMetrics is running
-   docker ps | grep victoriametrics
-   
-   # Check port binding
-   netstat -tlnp | grep 8428
-   ```
-
-2. **High Memory Usage**:
-   ```bash
-   # Reduce memory limit
-   docker run ... --memory.allowedPercent=60 ...
-   ```
-
-3. **Slow Queries**:
-   ```bash
-   # Increase query timeout
-   docker run ... --search.maxQueryDuration=120s ...
-   ```
-
-4. **Data Not Appearing**:
-   ```bash
-   # Check if data is being written
-   curl "http://localhost:8428/api/v1/query?query=up"
-   
-   # Check water monitor logs
-   tail -f water_monitor.log
-   ```
-
-### Performance Tuning
-
-1. **For High Write Load**:
-   ```bash
-   --maxConcurrentInserts=64
-   --insert.maxQueueDuration=60s
-   ```
-
-2. **For High Query Load**:
-   ```bash
-   --search.maxConcurrentRequests=32
-   --search.maxQueryDuration=120s
-   ```
-
-3. **For Large Datasets**:
-   ```bash
-   --search.maxSeries=10000000
-   --search.maxPointsPerTimeseries=1000000
-   ```
-
-## Security
-
-### Authentication
-
-VictoriaMetrics doesn't have built-in authentication. Use a reverse proxy:
-
-```nginx
-server {
-    listen 80;
-    server_name victoriametrics.example.com;
-    
-    auth_basic "VictoriaMetrics";
-    auth_basic_user_file /etc/nginx/.htpasswd;
-    
-    location / {
-        proxy_pass http://localhost:8428;
-        proxy_set_header Host $host;
-        proxy_set_header X-Real-IP $remote_addr;
-    }
-}
-```
-
-### TLS/SSL
-
-```bash
-# Use nginx or traefik for TLS termination
-# Or use VictoriaMetrics with TLS:
-docker run ... \
-  -v /path/to/cert.pem:/cert.pem \
-  -v /path/to/key.pem:/key.pem \
-  victoriametrics/victoria-metrics:latest \
-  --tls \
-  --tlsCertFile=/cert.pem \
-  --tlsKeyFile=/key.pem
-```
-
-## Scaling
-
-### Cluster Setup
-
-For high availability and horizontal scaling:
-
-```bash
-# Start multiple VictoriaMetrics instances
-docker run -d --name vm1 -p 8428:8428 victoriametrics/victoria-metrics:latest
-docker run -d --name vm2 -p 8429:8428 victoriametrics/victoria-metrics:latest
-
-# Use load balancer to distribute queries
-# Use vminsert/vmselect/vmstorage for true clustering
-```
-
-### Resource Requirements
-
-| Data Points/Hour | RAM | CPU | Storage/Day |
-|------------------|-----|-----|-------------|
-| 1,000 | 100MB | 0.1 CPU | 10MB |
-| 10,000 | 500MB | 0.5 CPU | 100MB |
-| 100,000 | 2GB | 1 CPU | 1GB |
-| 1,000,000 | 8GB | 2 CPU | 10GB |
-
-## Migration
-
-### From InfluxDB
-
-```bash
-# Export from InfluxDB
-influx -database water_monitoring -execute "SELECT * FROM water_data" -format csv > data.csv
-
-# Import to VictoriaMetrics (convert to Prometheus format first)
-# Use vmctl tool for migration
-```
-
-### From Prometheus
-
-```bash
-# Use vmctl for direct migration
-vmctl prometheus --prom-snapshot=/path/to/prometheus/data --vm-addr=http://localhost:8428
-```
-
-This comprehensive setup guide should help you configure VictoriaMetrics for optimal performance with the Thailand Water Monitor system.

docs/references/NOTABLE_DOCUMENTS.md

@@ -34,6 +34,20 @@ This document contains important references and external resources related to th
 - **Usage**: Reference for individual station characteristics and historical data patterns
 - **Station**: P.76 - บ้านแม่อีไฮ (Banb Mae I Hai)
 
+### **ThaiWater / HII (Hydro-Informatics Institute) Resources**
+
+#### **4. ThaiWater Portal**
+- **URL**: https://twa.thaiwater.net
+- **Description**: National water situation portal (rainfall, water level, dams, warnings)
+- **Language**: Thai/English
+- **Usage**: Backed by open and auth-gated APIs — full endpoint catalog in [DATA_SOURCES.md](../DATA_SOURCES.md)
+
+#### **5. ThaiWater Data Standard**
+- **URL**: https://standard.thaiwater.net
+- **Description**: Official water-data standard for exchange and warning — canonical station/basin/province code registries, data formats, warning-level definitions
+- **Language**: Thai
+- **Usage**: Reference for station metadata and warning-level semantics
+
 ## 📊 **Data Sources and APIs**
 
 ### **Primary Data Source**

grafana/dashboards/water-monitoring-dashboard.json

@@ -1,543 +0,0 @@
-{
-  "annotations": {
-    "list": [
-      {
-        "builtIn": 1,
-        "datasource": {
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
-        "enable": true,
-        "hide": true,
-        "iconColor": "rgba(0, 211, 255, 1)",
-        "name": "Annotations & Alerts",
-        "type": "dashboard"
-      }
-    ]
-  },
-  "editable": true,
-  "fiscalYearStartMonth": 0,
-  "graphTooltip": 0,
-  "id": null,
-  "links": [],
-  "liveNow": false,
-  "panels": [
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "victoriametrics-uid"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 10,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "vis": false
-            },
-            "lineInterpolation": "linear",
-            "lineWidth": 2,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "never",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "m"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 0
-      },
-      "id": 1,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom"
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "victoriametrics-uid"
-          },
-          "expr": "water_level",
-          "interval": "",
-          "legendFormat": "{{station_code}} - {{station_name_en}}",
-          "refId": "A"
-        }
-      ],
-      "title": "Water Levels by Station",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "victoriametrics-uid"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 10,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "vis": false
-            },
-            "lineInterpolation": "linear",
-            "lineWidth": 2,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "never",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "cms"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 0
-      },
-      "id": 2,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom"
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "victoriametrics-uid"
-          },
-          "expr": "water_discharge",
-          "interval": "",
-          "legendFormat": "{{station_code}} - {{station_name_en}}",
-          "refId": "A"
-        }
-      ],
-      "title": "Water Discharge by Station",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "victoriametrics-uid"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "custom": {
-            "align": "auto",
-            "displayMode": "auto",
-            "inspect": false
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": [
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Water Level"
-            },
-            "properties": [
-              {
-                "id": "unit",
-                "value": "m"
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Discharge"
-            },
-            "properties": [
-              {
-                "id": "unit",
-                "value": "cms"
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Discharge %"
-            },
-            "properties": [
-              {
-                "id": "unit",
-                "value": "percent"
-              }
-            ]
-          }
-        ]
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 24,
-        "x": 0,
-        "y": 8
-      },
-      "id": 3,
-      "options": {
-        "showHeader": true,
-        "sortBy": [
-          {
-            "desc": false,
-            "displayName": "Station"
-          }
-        ]
-      },
-      "pluginVersion": "8.5.0",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "victoriametrics-uid"
-          },
-          "expr": "water_level",
-          "format": "table",
-          "instant": true,
-          "interval": "",
-          "legendFormat": "",
-          "refId": "A"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "victoriametrics-uid"
-          },
-          "expr": "water_discharge",
-          "format": "table",
-          "instant": true,
-          "interval": "",
-          "legendFormat": "",
-          "refId": "B"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "victoriametrics-uid"
-          },
-          "expr": "water_discharge_percent",
-          "format": "table",
-          "instant": true,
-          "interval": "",
-          "legendFormat": "",
-          "refId": "C"
-        }
-      ],
-      "title": "Current Station Status",
-      "transformations": [
-        {
-          "id": "merge",
-          "options": {}
-        },
-        {
-          "id": "organize",
-          "options": {
-            "excludeByName": {
-              "Time": true,
-              "__name__": true,
-              "job": true,
-              "instance": true
-            },
-            "indexByName": {
-              "station_code": 0,
-              "station_name_en": 1,
-              "station_name_th": 2,
-              "Value #A": 3,
-              "Value #B": 4,
-              "Value #C": 5
-            },
-            "renameByName": {
-              "Value #A": "Water Level",
-              "Value #B": "Discharge",
-              "Value #C": "Discharge %",
-              "station_code": "Station",
-              "station_name_en": "English Name",
-              "station_name_th": "Thai Name"
-            }
-          }
-        }
-      ],
-      "type": "table"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "victoriametrics-uid"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "vis": false
-            }
-          },
-          "mappings": [],
-          "unit": "percent"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 16
-      },
-      "id": 4,
-      "options": {
-        "legend": {
-          "displayMode": "list",
-          "placement": "right"
-        },
-        "pieType": "pie",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "victoriametrics-uid"
-          },
-          "expr": "water_discharge_percent",
-          "interval": "",
-          "legendFormat": "{{station_code}}",
-          "refId": "A"
-        }
-      ],
-      "title": "Discharge Percentage Distribution",
-      "type": "piechart"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "victoriametrics-uid"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "yellow",
-                "value": 70
-              },
-              {
-                "color": "red",
-                "value": 90
-              }
-            ]
-          },
-          "unit": "percent"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 16
-      },
-      "id": 5,
-      "options": {
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "showThresholdLabels": false,
-        "showThresholdMarkers": true,
-        "text": {}
-      },
-      "pluginVersion": "8.5.0",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "victoriametrics-uid"
-          },
-          "expr": "avg(water_discharge_percent)",
-          "interval": "",
-          "legendFormat": "Average Discharge %",
-          "refId": "A"
-        }
-      ],
-      "title": "Average Discharge Percentage",
-      "type": "gauge"
-    }
-  ],
-  "refresh": "30s",
-  "schemaVersion": 36,
-  "style": "dark",
-  "tags": [
-    "water-monitoring",
-    "thailand",
-    "victoriametrics"
-  ],
-  "templating": {
-    "list": [
-      {
-        "current": {
-          "selected": false,
-          "text": "All",
-          "value": "$__all"
-        },
-        "datasource": {
-          "type": "prometheus",
-          "uid": "victoriametrics-uid"
-        },
-        "definition": "label_values(water_level, station_code)",
-        "hide": 0,
-        "includeAll": true,
-        "label": "Station",
-        "multi": true,
-        "name": "station",
-        "options": [],
-        "query": {
-          "query": "label_values(water_level, station_code)",
-          "refId": "StandardVariableQuery"
-        },
-        "refresh": 1,
-        "regex": "",
-        "skipUrlSync": false,
-        "sort": 1,
-        "type": "query"
-      }
-    ]
-  },
-  "time": {
-    "from": "now-6h",
-    "to": "now"
-  },
-  "timepicker": {},
-  "timezone": "Asia/Bangkok",
-  "title": "Thailand Water Level Monitoring",
-  "uid": "water-monitoring-dashboard",
-  "version": 1,
-  "weekStart": ""
-}

grafana/provisioning/dashboards/dashboard.yml

@@ -1,12 +0,0 @@
-apiVersion: 1
-
-providers:
-  - name: 'water-monitoring'
-    orgId: 1
-    folder: 'Water Monitoring'
-    type: file
-    disableDeletion: false
-    updateIntervalSeconds: 10
-    allowUiUpdates: true
-    options:
-      path: /var/lib/grafana/dashboards

grafana/provisioning/datasources/victoriametrics.yml

@@ -1,17 +0,0 @@
-apiVersion: 1
-
-datasources:
-  - name: VictoriaMetrics
-    type: prometheus
-    access: proxy
-    url: http://victoriametrics:8428
-    isDefault: true
-    editable: true
-    jsonData:
-      httpMethod: POST
-      queryTimeout: 60s
-      timeInterval: 30s
-      customQueryParameters: ''
-    secureJsonData: {}
-    version: 1
-    uid: victoriametrics-uid

models/eval_2026-09-12.json

@@ -0,0 +1,723 @@
+[
+ {
+  "station": "P.1",
+  "warn_thr": 3.7,
+  "folds": [
+   {
+    "year": 2021,
+    "n_train": 20024,
+    "n_test": 4392,
+    "events": [],
+    "variants": {
+     "rise_rain": {
+      "mae": 0.07375926701460789,
+      "mae_above_2p5": null,
+      "brier_warn": 0.0,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_fc48": {
+      "mae": 0.07332598842242062,
+      "mae_above_2p5": null,
+      "brier_warn": 0.0,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_quantile": {
+      "mae": 0.07383022350644125,
+      "mae_above_2p5": null,
+      "brier_warn": 1.0850721383440065e-12,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_quantile_uw": {
+      "mae": 0.06977345537342049,
+      "mae_above_2p5": null,
+      "brier_warn": 7.128994064266462e-16,
+      "events": [],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2022,
+    "n_train": 28784,
+    "n_test": 4392,
+    "events": [
+     {
+      "crossing": "2022-10-02T19:00:00",
+      "peak_ts": "2022-10-03T15:00:00",
+      "peak_level": 4.65
+     }
+    ],
+    "variants": {
+     "rise_rain": {
+      "mae": 0.08452847354970178,
+      "mae_above_2p5": 0.25082252888260664,
+      "brier_warn": 0.004300908725927739,
+      "events": [
+       {
+        "crossing": "2022-10-02T19:00:00",
+        "lead_h": 5.0,
+        "peak_level": 4.65,
+        "peak_pred_24h_before": 3.8173954245046406
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_fc48": {
+      "mae": 0.08369773912557228,
+      "mae_above_2p5": 0.24702357745371217,
+      "brier_warn": 0.004325741658698973,
+      "events": [
+       {
+        "crossing": "2022-10-02T19:00:00",
+        "lead_h": 5.0,
+        "peak_level": 4.65,
+        "peak_pred_24h_before": 3.8114190118860223
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_quantile": {
+      "mae": 0.08230165237819607,
+      "mae_above_2p5": 0.2599241552494541,
+      "brier_warn": 0.004191550822623699,
+      "events": [
+       {
+        "crossing": "2022-10-02T19:00:00",
+        "lead_h": 3.0,
+        "peak_level": 4.65,
+        "peak_pred_24h_before": 3.693734826616603
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_quantile_uw": {
+      "mae": 0.08181109208140971,
+      "mae_above_2p5": 0.2639358812546371,
+      "brier_warn": 0.004581181754314636,
+      "events": [
+       {
+        "crossing": "2022-10-02T19:00:00",
+        "lead_h": 2.0,
+        "peak_level": 4.65,
+        "peak_pred_24h_before": 3.620470606696475
+       }
+      ],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2023,
+    "n_train": 37539,
+    "n_test": 4392,
+    "events": [],
+    "variants": {
+     "rise_rain": {
+      "mae": 0.07696637416933236,
+      "mae_above_2p5": 0.10643655855133666,
+      "brier_warn": 1.919860722404625e-15,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_fc48": {
+      "mae": 0.07708101918232377,
+      "mae_above_2p5": 0.09974894450126857,
+      "brier_warn": 1.7028444765622288e-15,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_quantile": {
+      "mae": 0.06968496247786034,
+      "mae_above_2p5": 0.10210094973854984,
+      "brier_warn": 1.0987601008721297e-09,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_quantile_uw": {
+      "mae": 0.06726603345661021,
+      "mae_above_2p5": 0.09716644572204487,
+      "brier_warn": 2.7085590803510675e-09,
+      "events": [],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2024,
+    "n_train": 46323,
+    "n_test": 4392,
+    "events": [
+     {
+      "crossing": "2024-09-24T17:00:00",
+      "peak_ts": "2024-09-26T02:00:00",
+      "peak_level": 4.93
+     },
+     {
+      "crossing": "2024-10-03T09:00:00",
+      "peak_ts": "2024-10-05T12:00:00",
+      "peak_level": 5.3
+     }
+    ],
+    "variants": {
+     "rise_rain": {
+      "mae": 0.0890019157543212,
+      "mae_above_2p5": 0.2093245927883516,
+      "brier_warn": 0.005826169840715695,
+      "events": [
+       {
+        "crossing": "2024-09-24T17:00:00",
+        "lead_h": 10.0,
+        "peak_level": 4.93,
+        "peak_pred_24h_before": 4.817519939833057
+       },
+       {
+        "crossing": "2024-10-03T09:00:00",
+        "lead_h": 21.0,
+        "peak_level": 5.3,
+        "peak_pred_24h_before": 5.546588884631041
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_fc48": {
+      "mae": 0.08875916089292855,
+      "mae_above_2p5": 0.20975114218621593,
+      "brier_warn": 0.006061154593275248,
+      "events": [
+       {
+        "crossing": "2024-09-24T17:00:00",
+        "lead_h": 11.0,
+        "peak_level": 4.93,
+        "peak_pred_24h_before": 4.800924141216692
+       },
+       {
+        "crossing": "2024-10-03T09:00:00",
+        "lead_h": 72.0,
+        "peak_level": 5.3,
+        "peak_pred_24h_before": 5.577164984770105
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_quantile": {
+      "mae": 0.08341835741803395,
+      "mae_above_2p5": 0.18688839374651373,
+      "brier_warn": 0.003188229521816099,
+      "events": [
+       {
+        "crossing": "2024-09-24T17:00:00",
+        "lead_h": 17.0,
+        "peak_level": 4.93,
+        "peak_pred_24h_before": 5.058029430632501
+       },
+       {
+        "crossing": "2024-10-03T09:00:00",
+        "lead_h": 21.0,
+        "peak_level": 5.3,
+        "peak_pred_24h_before": 5.3311787370709975
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_quantile_uw": {
+      "mae": 0.08412336465267245,
+      "mae_above_2p5": 0.19173218504592401,
+      "brier_warn": 0.0034722638888286116,
+      "events": [
+       {
+        "crossing": "2024-09-24T17:00:00",
+        "lead_h": 15.0,
+        "peak_level": 4.93,
+        "peak_pred_24h_before": 5.0118284217314
+       },
+       {
+        "crossing": "2024-10-03T09:00:00",
+        "lead_h": 21.0,
+        "peak_level": 5.3,
+        "peak_pred_24h_before": 5.3520431553190155
+       }
+      ],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2025,
+    "n_train": 55083,
+    "n_test": 4392,
+    "events": [
+     {
+      "crossing": "2025-09-27T18:00:00",
+      "peak_ts": "2025-09-27T22:00:00",
+      "peak_level": 3.93
+     }
+    ],
+    "variants": {
+     "rise_rain": {
+      "mae": 0.11202835461695825,
+      "mae_above_2p5": 0.2456782496767171,
+      "brier_warn": 0.005178356039745929,
+      "events": [
+       {
+        "crossing": "2025-09-27T18:00:00",
+        "lead_h": 2.0,
+        "peak_level": 3.93,
+        "peak_pred_24h_before": 3.23
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_fc48": {
+      "mae": 0.11083068059655521,
+      "mae_above_2p5": 0.23787013498709667,
+      "brier_warn": 0.005228043825289021,
+      "events": [
+       {
+        "crossing": "2025-09-27T18:00:00",
+        "lead_h": 2.0,
+        "peak_level": 3.93,
+        "peak_pred_24h_before": 3.23
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_quantile": {
+      "mae": 0.10372526043263834,
+      "mae_above_2p5": 0.23846363852091013,
+      "brier_warn": 0.005898259026876986,
+      "events": [
+       {
+        "crossing": "2025-09-27T18:00:00",
+        "lead_h": 1.0,
+        "peak_level": 3.93,
+        "peak_pred_24h_before": 3.23
+       }
+      ],
+      "false_alarm_episodes": 1
+     },
+     "rise_rain_quantile_uw": {
+      "mae": 0.10267267834487567,
+      "mae_above_2p5": 0.2507951319537275,
+      "brier_warn": 0.005020467408392599,
+      "events": [
+       {
+        "crossing": "2025-09-27T18:00:00",
+        "lead_h": 2.0,
+        "peak_level": 3.93,
+        "peak_pred_24h_before": 3.2417205711942434
+       }
+      ],
+      "false_alarm_episodes": 1
+     }
+    }
+   }
+  ]
+ },
+ {
+  "station": "P.103",
+  "warn_thr": 5.95,
+  "folds": [
+   {
+    "year": 2021,
+    "n_train": 20009,
+    "n_test": 4392,
+    "events": [],
+    "variants": {
+     "rise_rain": {
+      "mae": 0.14438683486790602,
+      "mae_above_2p5": null,
+      "brier_warn": 0.0,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_fc48": {
+      "mae": 0.14305613024043953,
+      "mae_above_2p5": null,
+      "brier_warn": 0.0,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_quantile": {
+      "mae": 0.13512418754716052,
+      "mae_above_2p5": null,
+      "brier_warn": 6.937198832251013e-10,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_quantile_uw": {
+      "mae": 0.10270057685061172,
+      "mae_above_2p5": null,
+      "brier_warn": 6.565945930999852e-14,
+      "events": [],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2022,
+    "n_train": 28769,
+    "n_test": 4392,
+    "events": [
+     {
+      "crossing": "2022-08-14T04:00:00",
+      "peak_ts": "2022-08-14T08:00:00",
+      "peak_level": 6.09
+     },
+     {
+      "crossing": "2022-10-02T16:00:00",
+      "peak_ts": "2022-10-03T16:00:00",
+      "peak_level": 7.54
+     }
+    ],
+    "variants": {
+     "rise_rain": {
+      "mae": 0.14392334606606189,
+      "mae_above_2p5": 0.41075382386412596,
+      "brier_warn": 0.00764679988213082,
+      "events": [
+       {
+        "crossing": "2022-08-14T04:00:00",
+        "lead_h": 6.0,
+        "peak_level": 6.09,
+        "peak_pred_24h_before": 4.825270553204425
+       },
+       {
+        "crossing": "2022-10-02T16:00:00",
+        "lead_h": 9.0,
+        "peak_level": 7.54,
+        "peak_pred_24h_before": 7.194064117976157
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_fc48": {
+      "mae": 0.14284722696431765,
+      "mae_above_2p5": 0.39401132538096667,
+      "brier_warn": 0.007427375799450148,
+      "events": [
+       {
+        "crossing": "2022-08-14T04:00:00",
+        "lead_h": 6.0,
+        "peak_level": 6.09,
+        "peak_pred_24h_before": 4.845896993132048
+       },
+       {
+        "crossing": "2022-10-02T16:00:00",
+        "lead_h": 9.0,
+        "peak_level": 7.54,
+        "peak_pred_24h_before": 7.194661123502819
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_quantile": {
+      "mae": 0.1458954690776336,
+      "mae_above_2p5": 0.4674051188369517,
+      "brier_warn": 0.00899991317044724,
+      "events": [
+       {
+        "crossing": "2022-08-14T04:00:00",
+        "lead_h": 1.0,
+        "peak_level": 6.09,
+        "peak_pred_24h_before": 4.82714042795344
+       },
+       {
+        "crossing": "2022-10-02T16:00:00",
+        "lead_h": 5.0,
+        "peak_level": 7.54,
+        "peak_pred_24h_before": 6.59871451008115
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_quantile_uw": {
+      "mae": 0.13203958422483053,
+      "mae_above_2p5": 0.4902773906613983,
+      "brier_warn": 0.008514527559601331,
+      "events": [
+       {
+        "crossing": "2022-08-14T04:00:00",
+        "lead_h": 4.0,
+        "peak_level": 6.09,
+        "peak_pred_24h_before": 4.895369771408423
+       },
+       {
+        "crossing": "2022-10-02T16:00:00",
+        "lead_h": 3.0,
+        "peak_level": 7.54,
+        "peak_pred_24h_before": 6.294082735853337
+       }
+      ],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2023,
+    "n_train": 37524,
+    "n_test": 4392,
+    "events": [],
+    "variants": {
+     "rise_rain": {
+      "mae": 0.16195301512514512,
+      "mae_above_2p5": null,
+      "brier_warn": 3.0208441147560167e-07,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_fc48": {
+      "mae": 0.16334224973870387,
+      "mae_above_2p5": null,
+      "brier_warn": 1.1626187140381066e-08,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_quantile": {
+      "mae": 0.14755024879522577,
+      "mae_above_2p5": null,
+      "brier_warn": 3.850104714388072e-05,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_quantile_uw": {
+      "mae": 0.10589805327026759,
+      "mae_above_2p5": null,
+      "brier_warn": 1.8896757611081001e-07,
+      "events": [],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2024,
+    "n_train": 46308,
+    "n_test": 4058,
+    "events": [
+     {
+      "crossing": "2024-09-24T10:00:00",
+      "peak_ts": "2024-09-26T00:00:00",
+      "peak_level": 8.27
+     },
+     {
+      "crossing": "2024-09-30T03:00:00",
+      "peak_ts": "2024-09-30T06:00:00",
+      "peak_level": 5.99
+     },
+     {
+      "crossing": "2024-10-03T06:00:00",
+      "peak_ts": "2024-10-05T07:00:00",
+      "peak_level": 9.93
+     }
+    ],
+    "variants": {
+     "rise_rain": {
+      "mae": 0.13901842325128816,
+      "mae_above_2p5": 0.2887416310966684,
+      "brier_warn": 0.0072040857753137046,
+      "events": [
+       {
+        "crossing": "2024-09-24T10:00:00",
+        "lead_h": 19.0,
+        "peak_level": 8.27,
+        "peak_pred_24h_before": 8.212734363860193
+       },
+       {
+        "crossing": "2024-09-30T03:00:00",
+        "lead_h": 9.0,
+        "peak_level": 5.99,
+        "peak_pred_24h_before": 5.47565489914091
+       },
+       {
+        "crossing": "2024-10-03T06:00:00",
+        "lead_h": 55.0,
+        "peak_level": 9.93,
+        "peak_pred_24h_before": 8.780278464915938
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_fc48": {
+      "mae": 0.13850733053801131,
+      "mae_above_2p5": 0.29817392926900865,
+      "brier_warn": 0.007928846574280278,
+      "events": [
+       {
+        "crossing": "2024-09-24T10:00:00",
+        "lead_h": 19.0,
+        "peak_level": 8.27,
+        "peak_pred_24h_before": 8.166020251020889
+       },
+       {
+        "crossing": "2024-09-30T03:00:00",
+        "lead_h": 10.0,
+        "peak_level": 5.99,
+        "peak_pred_24h_before": 5.524584037259288
+       },
+       {
+        "crossing": "2024-10-03T06:00:00",
+        "lead_h": 69.0,
+        "peak_level": 9.93,
+        "peak_pred_24h_before": 8.595120917488185
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_quantile": {
+      "mae": 0.12640546558686208,
+      "mae_above_2p5": 0.27944115421093924,
+      "brier_warn": 0.007730268539879654,
+      "events": [
+       {
+        "crossing": "2024-09-24T10:00:00",
+        "lead_h": 20.0,
+        "peak_level": 8.27,
+        "peak_pred_24h_before": 8.336487732683672
+       },
+       {
+        "crossing": "2024-09-30T03:00:00",
+        "lead_h": 4.0,
+        "peak_level": 5.99,
+        "peak_pred_24h_before": 5.45284915024346
+       },
+       {
+        "crossing": "2024-10-03T06:00:00",
+        "lead_h": 69.0,
+        "peak_level": 9.93,
+        "peak_pred_24h_before": 8.575915226697406
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_quantile_uw": {
+      "mae": 0.12480975852168202,
+      "mae_above_2p5": 0.2857575557415695,
+      "brier_warn": 0.005894928998647136,
+      "events": [
+       {
+        "crossing": "2024-09-24T10:00:00",
+        "lead_h": 21.0,
+        "peak_level": 8.27,
+        "peak_pred_24h_before": 8.444720326882825
+       },
+       {
+        "crossing": "2024-09-30T03:00:00",
+        "lead_h": 9.0,
+        "peak_level": 5.99,
+        "peak_pred_24h_before": 5.42
+       },
+       {
+        "crossing": "2024-10-03T06:00:00",
+        "lead_h": 32.0,
+        "peak_level": 9.93,
+        "peak_pred_24h_before": 8.781001847167515
+       }
+      ],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2025,
+    "n_train": 54577,
+    "n_test": 4392,
+    "events": [
+     {
+      "crossing": "2025-09-26T06:00:00",
+      "peak_ts": "2025-09-27T21:00:00",
+      "peak_level": 6.64
+     },
+     {
+      "crossing": "2025-10-03T06:00:00",
+      "peak_ts": "2025-10-03T12:00:00",
+      "peak_level": 6.14
+     }
+    ],
+    "variants": {
+     "rise_rain": {
+      "mae": 0.17206685418009837,
+      "mae_above_2p5": 0.3840520085986099,
+      "brier_warn": 0.015824852891785323,
+      "events": [
+       {
+        "crossing": "2025-09-26T06:00:00",
+        "lead_h": 6.0,
+        "peak_level": 6.64,
+        "peak_pred_24h_before": 5.73043665401637
+       },
+       {
+        "crossing": "2025-10-03T06:00:00",
+        "lead_h": 8.0,
+        "peak_level": 6.14,
+        "peak_pred_24h_before": 5.609919760117381
+       }
+      ],
+      "false_alarm_episodes": 2
+     },
+     "rise_rain_fc48": {
+      "mae": 0.1722274451362234,
+      "mae_above_2p5": 0.3872216974630654,
+      "brier_warn": 0.016245727130063177,
+      "events": [
+       {
+        "crossing": "2025-09-26T06:00:00",
+        "lead_h": 4.0,
+        "peak_level": 6.64,
+        "peak_pred_24h_before": 5.7312263707793685
+       },
+       {
+        "crossing": "2025-10-03T06:00:00",
+        "lead_h": 8.0,
+        "peak_level": 6.14,
+        "peak_pred_24h_before": 5.685511824758637
+       }
+      ],
+      "false_alarm_episodes": 2
+     },
+     "rise_rain_quantile": {
+      "mae": 0.15912275848686555,
+      "mae_above_2p5": 0.3845695612674703,
+      "brier_warn": 0.014797606329811499,
+      "events": [
+       {
+        "crossing": "2025-09-26T06:00:00",
+        "lead_h": 11.0,
+        "peak_level": 6.64,
+        "peak_pred_24h_before": 5.73
+       },
+       {
+        "crossing": "2025-10-03T06:00:00",
+        "lead_h": 10.0,
+        "peak_level": 6.14,
+        "peak_pred_24h_before": 5.58133012298031
+       }
+      ],
+      "false_alarm_episodes": 2
+     },
+     "rise_rain_quantile_uw": {
+      "mae": 0.15385355845103685,
+      "mae_above_2p5": 0.41314645854725585,
+      "brier_warn": 0.016528116336109958,
+      "events": [
+       {
+        "crossing": "2025-09-26T06:00:00",
+        "lead_h": 9.0,
+        "peak_level": 6.64,
+        "peak_pred_24h_before": 5.741254234340573
+       },
+       {
+        "crossing": "2025-10-03T06:00:00",
+        "lead_h": 8.0,
+        "peak_level": 6.14,
+        "peak_pred_24h_before": 5.418105405306207
+       }
+      ],
+      "false_alarm_episodes": 2
+     }
+    }
+   }
+  ]
+ }
+]

models/eval_2026-09-12_qsigma.json

@@ -0,0 +1,439 @@
+[
+ {
+  "station": "P.1",
+  "warn_thr": 3.7,
+  "folds": [
+   {
+    "year": 2021,
+    "n_train": 20024,
+    "n_test": 4392,
+    "events": [],
+    "variants": {
+     "rise_rain": {
+      "mae": 0.07375926701460789,
+      "mae_above_2p5": null,
+      "brier_warn": 0.0,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_qsigma": {
+      "mae": 0.07375926701460789,
+      "mae_above_2p5": null,
+      "brier_warn": 7.852802408474157e-14,
+      "events": [],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2022,
+    "n_train": 28784,
+    "n_test": 4392,
+    "events": [
+     {
+      "crossing": "2022-10-02T19:00:00",
+      "peak_ts": "2022-10-03T15:00:00",
+      "peak_level": 4.65
+     }
+    ],
+    "variants": {
+     "rise_rain": {
+      "mae": 0.08452847354970178,
+      "mae_above_2p5": 0.25082252888260664,
+      "brier_warn": 0.004300908725927739,
+      "events": [
+       {
+        "crossing": "2022-10-02T19:00:00",
+        "lead_h": 5.0,
+        "peak_level": 4.65,
+        "peak_pred_24h_before": 3.8173954245046406
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_qsigma": {
+      "mae": 0.08452847354970178,
+      "mae_above_2p5": 0.25082252888260664,
+      "brier_warn": 0.0039815091186836665,
+      "events": [
+       {
+        "crossing": "2022-10-02T19:00:00",
+        "lead_h": 5.0,
+        "peak_level": 4.65,
+        "peak_pred_24h_before": 3.8173954245046406
+       }
+      ],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2023,
+    "n_train": 37539,
+    "n_test": 4392,
+    "events": [],
+    "variants": {
+     "rise_rain": {
+      "mae": 0.07696637416933236,
+      "mae_above_2p5": 0.10643655855133666,
+      "brier_warn": 1.919860722404625e-15,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_qsigma": {
+      "mae": 0.07696637416933236,
+      "mae_above_2p5": 0.10643655855133666,
+      "brier_warn": 4.0844243581898366e-08,
+      "events": [],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2024,
+    "n_train": 46323,
+    "n_test": 4392,
+    "events": [
+     {
+      "crossing": "2024-09-24T17:00:00",
+      "peak_ts": "2024-09-26T02:00:00",
+      "peak_level": 4.93
+     },
+     {
+      "crossing": "2024-10-03T09:00:00",
+      "peak_ts": "2024-10-05T12:00:00",
+      "peak_level": 5.3
+     }
+    ],
+    "variants": {
+     "rise_rain": {
+      "mae": 0.0890019157543212,
+      "mae_above_2p5": 0.2093245927883516,
+      "brier_warn": 0.005826169840715695,
+      "events": [
+       {
+        "crossing": "2024-09-24T17:00:00",
+        "lead_h": 10.0,
+        "peak_level": 4.93,
+        "peak_pred_24h_before": 4.817519939833057
+       },
+       {
+        "crossing": "2024-10-03T09:00:00",
+        "lead_h": 21.0,
+        "peak_level": 5.3,
+        "peak_pred_24h_before": 5.546588884631041
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_qsigma": {
+      "mae": 0.0890019157543212,
+      "mae_above_2p5": 0.2093245927883516,
+      "brier_warn": 0.005475787026695418,
+      "events": [
+       {
+        "crossing": "2024-09-24T17:00:00",
+        "lead_h": 10.0,
+        "peak_level": 4.93,
+        "peak_pred_24h_before": 4.817519939833057
+       },
+       {
+        "crossing": "2024-10-03T09:00:00",
+        "lead_h": 21.0,
+        "peak_level": 5.3,
+        "peak_pred_24h_before": 5.546588884631041
+       }
+      ],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2025,
+    "n_train": 55083,
+    "n_test": 4392,
+    "events": [
+     {
+      "crossing": "2025-09-27T18:00:00",
+      "peak_ts": "2025-09-27T22:00:00",
+      "peak_level": 3.93
+     }
+    ],
+    "variants": {
+     "rise_rain": {
+      "mae": 0.11202835461695825,
+      "mae_above_2p5": 0.2456782496767171,
+      "brier_warn": 0.005178356039745929,
+      "events": [
+       {
+        "crossing": "2025-09-27T18:00:00",
+        "lead_h": 2.0,
+        "peak_level": 3.93,
+        "peak_pred_24h_before": 3.23
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_qsigma": {
+      "mae": 0.11202835461695825,
+      "mae_above_2p5": 0.2456782496767171,
+      "brier_warn": 0.00491346243876589,
+      "events": [
+       {
+        "crossing": "2025-09-27T18:00:00",
+        "lead_h": 2.0,
+        "peak_level": 3.93,
+        "peak_pred_24h_before": 3.23
+       }
+      ],
+      "false_alarm_episodes": 0
+     }
+    }
+   }
+  ]
+ },
+ {
+  "station": "P.103",
+  "warn_thr": 5.95,
+  "folds": [
+   {
+    "year": 2021,
+    "n_train": 20009,
+    "n_test": 4392,
+    "events": [],
+    "variants": {
+     "rise_rain": {
+      "mae": 0.14438683486790602,
+      "mae_above_2p5": null,
+      "brier_warn": 0.0,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_qsigma": {
+      "mae": 0.14438683486790602,
+      "mae_above_2p5": null,
+      "brier_warn": 3.5349670949872053e-13,
+      "events": [],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2022,
+    "n_train": 28769,
+    "n_test": 4392,
+    "events": [
+     {
+      "crossing": "2022-08-14T04:00:00",
+      "peak_ts": "2022-08-14T08:00:00",
+      "peak_level": 6.09
+     },
+     {
+      "crossing": "2022-10-02T16:00:00",
+      "peak_ts": "2022-10-03T16:00:00",
+      "peak_level": 7.54
+     }
+    ],
+    "variants": {
+     "rise_rain": {
+      "mae": 0.14392334606606189,
+      "mae_above_2p5": 0.41075382386412596,
+      "brier_warn": 0.00764679988213082,
+      "events": [
+       {
+        "crossing": "2022-08-14T04:00:00",
+        "lead_h": 6.0,
+        "peak_level": 6.09,
+        "peak_pred_24h_before": 4.825270553204425
+       },
+       {
+        "crossing": "2022-10-02T16:00:00",
+        "lead_h": 9.0,
+        "peak_level": 7.54,
+        "peak_pred_24h_before": 7.194064117976157
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_qsigma": {
+      "mae": 0.14392334606606189,
+      "mae_above_2p5": 0.41075382386412596,
+      "brier_warn": 0.006975493312169236,
+      "events": [
+       {
+        "crossing": "2022-08-14T04:00:00",
+        "lead_h": 6.0,
+        "peak_level": 6.09,
+        "peak_pred_24h_before": 4.825270553204425
+       },
+       {
+        "crossing": "2022-10-02T16:00:00",
+        "lead_h": 9.0,
+        "peak_level": 7.54,
+        "peak_pred_24h_before": 7.194064117976157
+       }
+      ],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2023,
+    "n_train": 37524,
+    "n_test": 4392,
+    "events": [],
+    "variants": {
+     "rise_rain": {
+      "mae": 0.16195301512514512,
+      "mae_above_2p5": null,
+      "brier_warn": 3.0208441147560167e-07,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_qsigma": {
+      "mae": 0.16195301512514512,
+      "mae_above_2p5": null,
+      "brier_warn": 3.2176039819636275e-05,
+      "events": [],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2024,
+    "n_train": 46308,
+    "n_test": 4058,
+    "events": [
+     {
+      "crossing": "2024-09-24T10:00:00",
+      "peak_ts": "2024-09-26T00:00:00",
+      "peak_level": 8.27
+     },
+     {
+      "crossing": "2024-09-30T03:00:00",
+      "peak_ts": "2024-09-30T06:00:00",
+      "peak_level": 5.99
+     },
+     {
+      "crossing": "2024-10-03T06:00:00",
+      "peak_ts": "2024-10-05T07:00:00",
+      "peak_level": 9.93
+     }
+    ],
+    "variants": {
+     "rise_rain": {
+      "mae": 0.13901842325128816,
+      "mae_above_2p5": 0.2887416310966684,
+      "brier_warn": 0.0072040857753137046,
+      "events": [
+       {
+        "crossing": "2024-09-24T10:00:00",
+        "lead_h": 19.0,
+        "peak_level": 8.27,
+        "peak_pred_24h_before": 8.212734363860193
+       },
+       {
+        "crossing": "2024-09-30T03:00:00",
+        "lead_h": 9.0,
+        "peak_level": 5.99,
+        "peak_pred_24h_before": 5.47565489914091
+       },
+       {
+        "crossing": "2024-10-03T06:00:00",
+        "lead_h": 55.0,
+        "peak_level": 9.93,
+        "peak_pred_24h_before": 8.780278464915938
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_qsigma": {
+      "mae": 0.13901842325128816,
+      "mae_above_2p5": 0.2887416310966684,
+      "brier_warn": 0.006910847607098417,
+      "events": [
+       {
+        "crossing": "2024-09-24T10:00:00",
+        "lead_h": 19.0,
+        "peak_level": 8.27,
+        "peak_pred_24h_before": 8.212734363860193
+       },
+       {
+        "crossing": "2024-09-30T03:00:00",
+        "lead_h": 9.0,
+        "peak_level": 5.99,
+        "peak_pred_24h_before": 5.47565489914091
+       },
+       {
+        "crossing": "2024-10-03T06:00:00",
+        "lead_h": 55.0,
+        "peak_level": 9.93,
+        "peak_pred_24h_before": 8.780278464915938
+       }
+      ],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2025,
+    "n_train": 54577,
+    "n_test": 4392,
+    "events": [
+     {
+      "crossing": "2025-09-26T06:00:00",
+      "peak_ts": "2025-09-27T21:00:00",
+      "peak_level": 6.64
+     },
+     {
+      "crossing": "2025-10-03T06:00:00",
+      "peak_ts": "2025-10-03T12:00:00",
+      "peak_level": 6.14
+     }
+    ],
+    "variants": {
+     "rise_rain": {
+      "mae": 0.17206685418009837,
+      "mae_above_2p5": 0.3840520085986099,
+      "brier_warn": 0.015824852891785323,
+      "events": [
+       {
+        "crossing": "2025-09-26T06:00:00",
+        "lead_h": 6.0,
+        "peak_level": 6.64,
+        "peak_pred_24h_before": 5.73043665401637
+       },
+       {
+        "crossing": "2025-10-03T06:00:00",
+        "lead_h": 8.0,
+        "peak_level": 6.14,
+        "peak_pred_24h_before": 5.609919760117381
+       }
+      ],
+      "false_alarm_episodes": 2
+     },
+     "rise_rain_qsigma": {
+      "mae": 0.17206685418009837,
+      "mae_above_2p5": 0.3840520085986099,
+      "brier_warn": 0.015889933586185904,
+      "events": [
+       {
+        "crossing": "2025-09-26T06:00:00",
+        "lead_h": 6.0,
+        "peak_level": 6.64,
+        "peak_pred_24h_before": 5.73043665401637
+       },
+       {
+        "crossing": "2025-10-03T06:00:00",
+        "lead_h": 8.0,
+        "peak_level": 6.14,
+        "peak_pred_24h_before": 5.609919760117381
+       }
+      ],
+      "false_alarm_episodes": 2
+     }
+    }
+   }
+  ]
+ }
+]

models/eval_2026-10-18_fc24.json

@@ -0,0 +1,737 @@
+[
+ {
+  "station": "P.1",
+  "warn_thr": 3.7,
+  "folds": [
+   {
+    "year": 2021,
+    "n_train": 20024,
+    "n_test": 4392,
+    "events": [],
+    "variants": {
+     "rise_rain": {
+      "mae": 0.07375926701460789,
+      "mae_above_2p5": null,
+      "brier_warn": 0.0,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_fc24_missing": {
+      "mae": 0.07349391292098781,
+      "mae_above_2p5": null,
+      "brier_warn": 0.0,
+      "events": [],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2022,
+    "n_train": 28784,
+    "n_test": 4392,
+    "events": [
+     {
+      "crossing": "2022-10-02T19:00:00",
+      "peak_ts": "2022-10-03T15:00:00",
+      "peak_level": 4.65
+     }
+    ],
+    "variants": {
+     "rise_rain": {
+      "mae": 0.08452847354970178,
+      "mae_above_2p5": 0.25082252888260664,
+      "brier_warn": 0.004300908725927739,
+      "events": [
+       {
+        "crossing": "2022-10-02T19:00:00",
+        "lead_h": 5.0,
+        "peak_level": 4.65,
+        "peak_pred_24h_before": 3.8173954245046406
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_fc24_missing": {
+      "mae": 0.08511964468019045,
+      "mae_above_2p5": 0.2527703432441232,
+      "brier_warn": 0.004315341268867166,
+      "events": [
+       {
+        "crossing": "2022-10-02T19:00:00",
+        "lead_h": 5.0,
+        "peak_level": 4.65,
+        "peak_pred_24h_before": 3.8102898827310137
+       }
+      ],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2023,
+    "n_train": 37539,
+    "n_test": 4392,
+    "events": [],
+    "variants": {
+     "rise_rain": {
+      "mae": 0.07696637416933236,
+      "mae_above_2p5": 0.10643655855133666,
+      "brier_warn": 1.919860722404625e-15,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_fc24_missing": {
+      "mae": 0.07281928830426289,
+      "mae_above_2p5": 0.12255431002563086,
+      "brier_warn": 1.859416740909973e-15,
+      "events": [],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2024,
+    "n_train": 46323,
+    "n_test": 4392,
+    "events": [
+     {
+      "crossing": "2024-09-24T17:00:00",
+      "peak_ts": "2024-09-26T02:00:00",
+      "peak_level": 4.93
+     },
+     {
+      "crossing": "2024-10-03T09:00:00",
+      "peak_ts": "2024-10-05T12:00:00",
+      "peak_level": 5.3
+     }
+    ],
+    "variants": {
+     "rise_rain": {
+      "mae": 0.0890019157543212,
+      "mae_above_2p5": 0.2093245927883516,
+      "brier_warn": 0.005826169840715695,
+      "events": [
+       {
+        "crossing": "2024-09-24T17:00:00",
+        "lead_h": 10.0,
+        "peak_level": 4.93,
+        "peak_pred_24h_before": 4.817519939833057
+       },
+       {
+        "crossing": "2024-10-03T09:00:00",
+        "lead_h": 21.0,
+        "peak_level": 5.3,
+        "peak_pred_24h_before": 5.546588884631041
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_fc24_missing": {
+      "mae": 0.0896559193295349,
+      "mae_above_2p5": 0.2184968282693981,
+      "brier_warn": 0.00611626529479383,
+      "events": [
+       {
+        "crossing": "2024-09-24T17:00:00",
+        "lead_h": 9.0,
+        "peak_level": 4.93,
+        "peak_pred_24h_before": 4.7051072221287855
+       },
+       {
+        "crossing": "2024-10-03T09:00:00",
+        "lead_h": 21.0,
+        "peak_level": 5.3,
+        "peak_pred_24h_before": 5.563518687086929
+       }
+      ],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2025,
+    "n_train": 55083,
+    "n_test": 4392,
+    "events": [
+     {
+      "crossing": "2025-09-27T18:00:00",
+      "peak_ts": "2025-09-27T22:00:00",
+      "peak_level": 3.93
+     }
+    ],
+    "variants": {
+     "rise_rain": {
+      "mae": 0.11202835461695825,
+      "mae_above_2p5": 0.2456782496767171,
+      "brier_warn": 0.005178356039745929,
+      "events": [
+       {
+        "crossing": "2025-09-27T18:00:00",
+        "lead_h": 2.0,
+        "peak_level": 3.93,
+        "peak_pred_24h_before": 3.23
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_fc24_missing": {
+      "mae": 0.11232147044143151,
+      "mae_above_2p5": 0.2457513740974484,
+      "brier_warn": 0.004988582804599792,
+      "events": [
+       {
+        "crossing": "2025-09-27T18:00:00",
+        "lead_h": 4.0,
+        "peak_level": 3.93,
+        "peak_pred_24h_before": 3.3264027753671597
+       }
+      ],
+      "false_alarm_episodes": 0
+     }
+    }
+   }
+  ]
+ },
+ {
+  "station": "P.103",
+  "warn_thr": 5.95,
+  "folds": [
+   {
+    "year": 2021,
+    "n_train": 20009,
+    "n_test": 4392,
+    "events": [],
+    "variants": {
+     "rise_rain": {
+      "mae": 0.14438683486790602,
+      "mae_above_2p5": null,
+      "brier_warn": 0.0,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_fc24_missing": {
+      "mae": 0.14435605468178428,
+      "mae_above_2p5": null,
+      "brier_warn": 0.0,
+      "events": [],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2022,
+    "n_train": 28769,
+    "n_test": 4392,
+    "events": [
+     {
+      "crossing": "2022-08-14T04:00:00",
+      "peak_ts": "2022-08-14T08:00:00",
+      "peak_level": 6.09
+     },
+     {
+      "crossing": "2022-10-02T16:00:00",
+      "peak_ts": "2022-10-03T16:00:00",
+      "peak_level": 7.54
+     }
+    ],
+    "variants": {
+     "rise_rain": {
+      "mae": 0.14392334606606189,
+      "mae_above_2p5": 0.41075382386412596,
+      "brier_warn": 0.00764679988213082,
+      "events": [
+       {
+        "crossing": "2022-08-14T04:00:00",
+        "lead_h": 6.0,
+        "peak_level": 6.09,
+        "peak_pred_24h_before": 4.825270553204425
+       },
+       {
+        "crossing": "2022-10-02T16:00:00",
+        "lead_h": 9.0,
+        "peak_level": 7.54,
+        "peak_pred_24h_before": 7.194064117976157
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_fc24_missing": {
+      "mae": 0.1438805977972288,
+      "mae_above_2p5": 0.41655333933205874,
+      "brier_warn": 0.007670614570637225,
+      "events": [
+       {
+        "crossing": "2022-08-14T04:00:00",
+        "lead_h": 5.0,
+        "peak_level": 6.09,
+        "peak_pred_24h_before": 4.822220911656332
+       },
+       {
+        "crossing": "2022-10-02T16:00:00",
+        "lead_h": 9.0,
+        "peak_level": 7.54,
+        "peak_pred_24h_before": 7.187888665288963
+       }
+      ],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2023,
+    "n_train": 37524,
+    "n_test": 4392,
+    "events": [],
+    "variants": {
+     "rise_rain": {
+      "mae": 0.16195301512514512,
+      "mae_above_2p5": null,
+      "brier_warn": 3.0208441147560167e-07,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_fc24_missing": {
+      "mae": 0.15117839869986002,
+      "mae_above_2p5": null,
+      "brier_warn": 6.508469941612771e-08,
+      "events": [],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2024,
+    "n_train": 46308,
+    "n_test": 4058,
+    "events": [
+     {
+      "crossing": "2024-09-24T10:00:00",
+      "peak_ts": "2024-09-26T00:00:00",
+      "peak_level": 8.27
+     },
+     {
+      "crossing": "2024-09-30T03:00:00",
+      "peak_ts": "2024-09-30T06:00:00",
+      "peak_level": 5.99
+     },
+     {
+      "crossing": "2024-10-03T06:00:00",
+      "peak_ts": "2024-10-05T07:00:00",
+      "peak_level": 9.93
+     }
+    ],
+    "variants": {
+     "rise_rain": {
+      "mae": 0.13901842325128816,
+      "mae_above_2p5": 0.2887416310966684,
+      "brier_warn": 0.0072040857753137046,
+      "events": [
+       {
+        "crossing": "2024-09-24T10:00:00",
+        "lead_h": 19.0,
+        "peak_level": 8.27,
+        "peak_pred_24h_before": 8.212734363860193
+       },
+       {
+        "crossing": "2024-09-30T03:00:00",
+        "lead_h": 9.0,
+        "peak_level": 5.99,
+        "peak_pred_24h_before": 5.47565489914091
+       },
+       {
+        "crossing": "2024-10-03T06:00:00",
+        "lead_h": 55.0,
+        "peak_level": 9.93,
+        "peak_pred_24h_before": 8.780278464915938
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_fc24_missing": {
+      "mae": 0.15101681306989473,
+      "mae_above_2p5": 0.3221006670360109,
+      "brier_warn": 0.008559364941781224,
+      "events": [
+       {
+        "crossing": "2024-09-24T10:00:00",
+        "lead_h": 18.0,
+        "peak_level": 8.27,
+        "peak_pred_24h_before": 7.997091483400909
+       },
+       {
+        "crossing": "2024-09-30T03:00:00",
+        "lead_h": 9.0,
+        "peak_level": 5.99,
+        "peak_pred_24h_before": 5.502287543201456
+       },
+       {
+        "crossing": "2024-10-03T06:00:00",
+        "lead_h": 62.0,
+        "peak_level": 9.93,
+        "peak_pred_24h_before": 9.103449390810866
+       }
+      ],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2025,
+    "n_train": 54577,
+    "n_test": 4392,
+    "events": [
+     {
+      "crossing": "2025-09-26T06:00:00",
+      "peak_ts": "2025-09-27T21:00:00",
+      "peak_level": 6.64
+     },
+     {
+      "crossing": "2025-10-03T06:00:00",
+      "peak_ts": "2025-10-03T12:00:00",
+      "peak_level": 6.14
+     }
+    ],
+    "variants": {
+     "rise_rain": {
+      "mae": 0.17206685418009837,
+      "mae_above_2p5": 0.3840520085986099,
+      "brier_warn": 0.015824852891785323,
+      "events": [
+       {
+        "crossing": "2025-09-26T06:00:00",
+        "lead_h": 6.0,
+        "peak_level": 6.64,
+        "peak_pred_24h_before": 5.73043665401637
+       },
+       {
+        "crossing": "2025-10-03T06:00:00",
+        "lead_h": 8.0,
+        "peak_level": 6.14,
+        "peak_pred_24h_before": 5.609919760117381
+       }
+      ],
+      "false_alarm_episodes": 2
+     },
+     "rise_rain_fc24_missing": {
+      "mae": 0.17368174393168262,
+      "mae_above_2p5": 0.3756076666801506,
+      "brier_warn": 0.014794520234147523,
+      "events": [
+       {
+        "crossing": "2025-09-26T06:00:00",
+        "lead_h": 5.0,
+        "peak_level": 6.64,
+        "peak_pred_24h_before": 5.8809952440025
+       },
+       {
+        "crossing": "2025-10-03T06:00:00",
+        "lead_h": 35.0,
+        "peak_level": 6.14,
+        "peak_pred_24h_before": 5.6215400644356235
+       }
+      ],
+      "false_alarm_episodes": 2
+     }
+    }
+   }
+  ]
+ },
+ {
+  "station": "P.21",
+  "warn_thr": 3.2,
+  "folds": [
+   {
+    "year": 2021,
+    "n_train": 19099,
+    "n_test": 4392,
+    "events": [],
+    "variants": {
+     "rise_rain": {
+      "mae": 0.31976019104736464,
+      "mae_above_2p5": 0.49280187683998045,
+      "brier_warn": 4.599376306869306e-06,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_fc24_missing": {
+      "mae": 0.3238133154815793,
+      "mae_above_2p5": 0.49187453544981,
+      "brier_warn": 4.609640062881583e-06,
+      "events": [],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2022,
+    "n_train": 27860,
+    "n_test": 4392,
+    "events": [
+     {
+      "crossing": "2022-09-12T05:00:00",
+      "peak_ts": "2022-09-12T05:00:00",
+      "peak_level": 3.21
+     },
+     {
+      "crossing": "2022-10-02T04:00:00",
+      "peak_ts": "2022-10-03T21:00:00",
+      "peak_level": 3.92
+     }
+    ],
+    "variants": {
+     "rise_rain": {
+      "mae": 0.188862265489962,
+      "mae_above_2p5": 0.22738565836625405,
+      "brier_warn": 0.009674093199871036,
+      "events": [
+       {
+        "crossing": "2022-09-12T05:00:00",
+        "lead_h": 0.0,
+        "peak_level": 3.21,
+        "peak_pred_24h_before": 2.49615362708868
+       },
+       {
+        "crossing": "2022-10-02T04:00:00",
+        "lead_h": 1.0,
+        "peak_level": 3.92,
+        "peak_pred_24h_before": 3.907087039152045
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_fc24_missing": {
+      "mae": 0.19791868293297726,
+      "mae_above_2p5": 0.2739869896442161,
+      "brier_warn": 0.0103072884296978,
+      "events": [
+       {
+        "crossing": "2022-09-12T05:00:00",
+        "lead_h": 0.0,
+        "peak_level": 3.21,
+        "peak_pred_24h_before": 2.4722245571508683
+       },
+       {
+        "crossing": "2022-10-02T04:00:00",
+        "lead_h": 0.0,
+        "peak_level": 3.92,
+        "peak_pred_24h_before": 3.845235629130954
+       }
+      ],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2023,
+    "n_train": 36614,
+    "n_test": 4392,
+    "events": [
+     {
+      "crossing": "2023-10-10T06:00:00",
+      "peak_ts": "2023-10-10T06:00:00",
+      "peak_level": 3.2
+     }
+    ],
+    "variants": {
+     "rise_rain": {
+      "mae": 0.17283257678933697,
+      "mae_above_2p5": 0.39237224972369483,
+      "brier_warn": 0.012908395601661503,
+      "events": [
+       {
+        "crossing": "2023-10-10T06:00:00",
+        "lead_h": 15.0,
+        "peak_level": 3.2,
+        "peak_pred_24h_before": 1.8019095357375217
+       }
+      ],
+      "false_alarm_episodes": 3
+     },
+     "rise_rain_fc24_missing": {
+      "mae": 0.1953027570197896,
+      "mae_above_2p5": 0.37345466270918226,
+      "brier_warn": 0.005336247430915456,
+      "events": [
+       {
+        "crossing": "2023-10-10T06:00:00",
+        "lead_h": 14.0,
+        "peak_level": 3.2,
+        "peak_pred_24h_before": 1.8796882667592647
+       }
+      ],
+      "false_alarm_episodes": 3
+     }
+    }
+   },
+   {
+    "year": 2024,
+    "n_train": 45398,
+    "n_test": 4385,
+    "events": [
+     {
+      "crossing": "2024-09-01T03:00:00",
+      "peak_ts": "2024-09-01T08:00:00",
+      "peak_level": 4.3
+     },
+     {
+      "crossing": "2024-09-23T06:00:00",
+      "peak_ts": "2024-09-23T15:00:00",
+      "peak_level": 4.68
+     },
+     {
+      "crossing": "2024-10-03T05:00:00",
+      "peak_ts": "2024-10-04T20:00:00",
+      "peak_level": 5.21
+     }
+    ],
+    "variants": {
+     "rise_rain": {
+      "mae": 0.20733600140331027,
+      "mae_above_2p5": 0.4173432645880738,
+      "brier_warn": 0.013730524899775963,
+      "events": [
+       {
+        "crossing": "2024-09-01T03:00:00",
+        "lead_h": 0.0,
+        "peak_level": 4.3,
+        "peak_pred_24h_before": 1.7973215452596953
+       },
+       {
+        "crossing": "2024-09-23T06:00:00",
+        "lead_h": 2.0,
+        "peak_level": 4.68,
+        "peak_pred_24h_before": 2.670881094688805
+       },
+       {
+        "crossing": "2024-10-03T05:00:00",
+        "lead_h": 33.0,
+        "peak_level": 5.21,
+        "peak_pred_24h_before": 4.999142187693036
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain_fc24_missing": {
+      "mae": 0.2222046566812092,
+      "mae_above_2p5": 0.43423994130380844,
+      "brier_warn": 0.01611358822165686,
+      "events": [
+       {
+        "crossing": "2024-09-01T03:00:00",
+        "lead_h": 0.0,
+        "peak_level": 4.3,
+        "peak_pred_24h_before": 1.7847018175539242
+       },
+       {
+        "crossing": "2024-09-23T06:00:00",
+        "lead_h": 1.0,
+        "peak_level": 4.68,
+        "peak_pred_24h_before": 2.640820102168781
+       },
+       {
+        "crossing": "2024-10-03T05:00:00",
+        "lead_h": 32.0,
+        "peak_level": 5.21,
+        "peak_pred_24h_before": 4.749354278189956
+       }
+      ],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2025,
+    "n_train": 54150,
+    "n_test": 4392,
+    "events": [
+     {
+      "crossing": "2025-07-23T15:00:00",
+      "peak_ts": "2025-07-24T04:00:00",
+      "peak_level": 3.63
+     },
+     {
+      "crossing": "2025-08-26T19:00:00",
+      "peak_ts": "2025-08-26T23:00:00",
+      "peak_level": 3.48
+     },
+     {
+      "crossing": "2025-09-12T21:00:00",
+      "peak_ts": "2025-09-12T22:00:00",
+      "peak_level": 3.21
+     },
+     {
+      "crossing": "2025-09-22T20:00:00",
+      "peak_ts": "2025-09-22T21:00:00",
+      "peak_level": 3.27
+     }
+    ],
+    "variants": {
+     "rise_rain": {
+      "mae": 0.1851599197920176,
+      "mae_above_2p5": 0.29411810271343447,
+      "brier_warn": 0.021849546400589632,
+      "events": [
+       {
+        "crossing": "2025-07-23T15:00:00",
+        "lead_h": 3.0,
+        "peak_level": 3.63,
+        "peak_pred_24h_before": 2.4024371626847
+       },
+       {
+        "crossing": "2025-08-26T19:00:00",
+        "lead_h": 4.0,
+        "peak_level": 3.48,
+        "peak_pred_24h_before": 2.492383184457359
+       },
+       {
+        "crossing": "2025-09-12T21:00:00",
+        "lead_h": 18.0,
+        "peak_level": 3.21,
+        "peak_pred_24h_before": 2.5690976144688706
+       },
+       {
+        "crossing": "2025-09-22T20:00:00",
+        "lead_h": 1.0,
+        "peak_level": 3.27,
+        "peak_pred_24h_before": 2.382265050508778
+       }
+      ],
+      "false_alarm_episodes": 2
+     },
+     "rise_rain_fc24_missing": {
+      "mae": 0.21936157981899435,
+      "mae_above_2p5": 0.3121582241464527,
+      "brier_warn": 0.025980409031233827,
+      "events": [
+       {
+        "crossing": "2025-07-23T15:00:00",
+        "lead_h": 2.0,
+        "peak_level": 3.63,
+        "peak_pred_24h_before": 2.096100137296138
+       },
+       {
+        "crossing": "2025-08-26T19:00:00",
+        "lead_h": 69.0,
+        "peak_level": 3.48,
+        "peak_pred_24h_before": 2.3678295230657067
+       },
+       {
+        "crossing": "2025-09-12T21:00:00",
+        "lead_h": 19.0,
+        "peak_level": 3.21,
+        "peak_pred_24h_before": 2.783241642058761
+       },
+       {
+        "crossing": "2025-09-22T20:00:00",
+        "lead_h": 2.0,
+        "peak_level": 3.27,
+        "peak_pred_24h_before": 2.279238629032059
+       }
+      ],
+      "false_alarm_episodes": 7
+     }
+    }
+   }
+  ]
+ }
+]

models/eval_rain.json

@@ -0,0 +1,439 @@
+[
+ {
+  "station": "P.1",
+  "warn_thr": 3.7,
+  "folds": [
+   {
+    "year": 2021,
+    "n_train": 20024,
+    "n_test": 4392,
+    "events": [],
+    "variants": {
+     "rise": {
+      "mae": 0.070970681677648,
+      "mae_above_2p5": null,
+      "brier_warn": 0.0,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain": {
+      "mae": 0.07247641391827915,
+      "mae_above_2p5": null,
+      "brier_warn": 0.0,
+      "events": [],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2022,
+    "n_train": 28782,
+    "n_test": 4392,
+    "events": [
+     {
+      "crossing": "2022-10-02T19:00:00",
+      "peak_ts": "2022-10-03T15:00:00",
+      "peak_level": 4.65
+     }
+    ],
+    "variants": {
+     "rise": {
+      "mae": 0.08489734638010188,
+      "mae_above_2p5": 0.24517428534843191,
+      "brier_warn": 0.004136576477574926,
+      "events": [
+       {
+        "crossing": "2022-10-02T19:00:00",
+        "lead_h": 6.0,
+        "peak_level": 4.65,
+        "peak_pred_24h_before": 3.870617057762467
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain": {
+      "mae": 0.08491621780630447,
+      "mae_above_2p5": 0.24539225527181602,
+      "brier_warn": 0.004209435091817237,
+      "events": [
+       {
+        "crossing": "2022-10-02T19:00:00",
+        "lead_h": 5.0,
+        "peak_level": 4.65,
+        "peak_pred_24h_before": 3.844081593978701
+       }
+      ],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2023,
+    "n_train": 37537,
+    "n_test": 4392,
+    "events": [],
+    "variants": {
+     "rise": {
+      "mae": 0.07647507344443248,
+      "mae_above_2p5": 0.12593127745781565,
+      "brier_warn": 3.19529506149809e-11,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain": {
+      "mae": 0.07491688056430452,
+      "mae_above_2p5": 0.09506457784884237,
+      "brier_warn": 6.639993512164624e-14,
+      "events": [],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2024,
+    "n_train": 46321,
+    "n_test": 4392,
+    "events": [
+     {
+      "crossing": "2024-09-24T17:00:00",
+      "peak_ts": "2024-09-26T02:00:00",
+      "peak_level": 4.93
+     },
+     {
+      "crossing": "2024-10-03T09:00:00",
+      "peak_ts": "2024-10-05T12:00:00",
+      "peak_level": 5.3
+     }
+    ],
+    "variants": {
+     "rise": {
+      "mae": 0.10075626719251345,
+      "mae_above_2p5": 0.26384939692286363,
+      "brier_warn": 0.011608221543810462,
+      "events": [
+       {
+        "crossing": "2024-09-24T17:00:00",
+        "lead_h": 6.0,
+        "peak_level": 4.93,
+        "peak_pred_24h_before": 4.534709676862131
+       },
+       {
+        "crossing": "2024-10-03T09:00:00",
+        "lead_h": 3.0,
+        "peak_level": 5.3,
+        "peak_pred_24h_before": 5.056619694027486
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain": {
+      "mae": 0.08874847709708966,
+      "mae_above_2p5": 0.20967242138010514,
+      "brier_warn": 0.005862181747890141,
+      "events": [
+       {
+        "crossing": "2024-09-24T17:00:00",
+        "lead_h": 11.0,
+        "peak_level": 4.93,
+        "peak_pred_24h_before": 4.863137825109792
+       },
+       {
+        "crossing": "2024-10-03T09:00:00",
+        "lead_h": 21.0,
+        "peak_level": 5.3,
+        "peak_pred_24h_before": 5.557547530221961
+       }
+      ],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2025,
+    "n_train": 55081,
+    "n_test": 4392,
+    "events": [
+     {
+      "crossing": "2025-09-27T18:00:00",
+      "peak_ts": "2025-09-27T22:00:00",
+      "peak_level": 3.93
+     }
+    ],
+    "variants": {
+     "rise": {
+      "mae": 0.13102686839694494,
+      "mae_above_2p5": 0.2729944665011028,
+      "brier_warn": 0.010889835530782944,
+      "events": [
+       {
+        "crossing": "2025-09-27T18:00:00",
+        "lead_h": 46.0,
+        "peak_level": 3.93,
+        "peak_pred_24h_before": 3.232029710676319
+       }
+      ],
+      "false_alarm_episodes": 1
+     },
+     "rise_rain": {
+      "mae": 0.11160288394028639,
+      "mae_above_2p5": 0.24838881498904114,
+      "brier_warn": 0.005150122823202273,
+      "events": [
+       {
+        "crossing": "2025-09-27T18:00:00",
+        "lead_h": 2.0,
+        "peak_level": 3.93,
+        "peak_pred_24h_before": 3.239754736790134
+       }
+      ],
+      "false_alarm_episodes": 0
+     }
+    }
+   }
+  ]
+ },
+ {
+  "station": "P.103",
+  "warn_thr": 5.95,
+  "folds": [
+   {
+    "year": 2021,
+    "n_train": 20009,
+    "n_test": 4392,
+    "events": [],
+    "variants": {
+     "rise": {
+      "mae": 0.15279228710793116,
+      "mae_above_2p5": null,
+      "brier_warn": 0.0,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain": {
+      "mae": 0.15114584578006643,
+      "mae_above_2p5": null,
+      "brier_warn": 0.0,
+      "events": [],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2022,
+    "n_train": 28769,
+    "n_test": 4392,
+    "events": [
+     {
+      "crossing": "2022-08-14T04:00:00",
+      "peak_ts": "2022-08-14T08:00:00",
+      "peak_level": 6.09
+     },
+     {
+      "crossing": "2022-10-02T16:00:00",
+      "peak_ts": "2022-10-03T16:00:00",
+      "peak_level": 7.54
+     }
+    ],
+    "variants": {
+     "rise": {
+      "mae": 0.15389446229226025,
+      "mae_above_2p5": 0.41887094569113953,
+      "brier_warn": 0.007740077230532647,
+      "events": [
+       {
+        "crossing": "2022-08-14T04:00:00",
+        "lead_h": 6.0,
+        "peak_level": 6.09,
+        "peak_pred_24h_before": 4.837975953559253
+       },
+       {
+        "crossing": "2022-10-02T16:00:00",
+        "lead_h": 9.0,
+        "peak_level": 7.54,
+        "peak_pred_24h_before": 7.168225721504108
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain": {
+      "mae": 0.15303667736579823,
+      "mae_above_2p5": 0.38833713105646916,
+      "brier_warn": 0.0074511589478895475,
+      "events": [
+       {
+        "crossing": "2022-08-14T04:00:00",
+        "lead_h": 6.0,
+        "peak_level": 6.09,
+        "peak_pred_24h_before": 4.8804746828604815
+       },
+       {
+        "crossing": "2022-10-02T16:00:00",
+        "lead_h": 10.0,
+        "peak_level": 7.54,
+        "peak_pred_24h_before": 7.226685294045882
+       }
+      ],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2023,
+    "n_train": 37524,
+    "n_test": 4392,
+    "events": [],
+    "variants": {
+     "rise": {
+      "mae": 0.16106550376855028,
+      "mae_above_2p5": null,
+      "brier_warn": 0.0001779564366636896,
+      "events": [],
+      "false_alarm_episodes": 1
+     },
+     "rise_rain": {
+      "mae": 0.15541158498896382,
+      "mae_above_2p5": null,
+      "brier_warn": 4.103077380009017e-09,
+      "events": [],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2024,
+    "n_train": 46308,
+    "n_test": 4058,
+    "events": [
+     {
+      "crossing": "2024-09-24T10:00:00",
+      "peak_ts": "2024-09-26T00:00:00",
+      "peak_level": 8.27
+     },
+     {
+      "crossing": "2024-09-30T03:00:00",
+      "peak_ts": "2024-09-30T06:00:00",
+      "peak_level": 5.99
+     },
+     {
+      "crossing": "2024-10-03T06:00:00",
+      "peak_ts": "2024-10-05T07:00:00",
+      "peak_level": 9.93
+     }
+    ],
+    "variants": {
+     "rise": {
+      "mae": 0.17314177863843333,
+      "mae_above_2p5": 0.48294427141283425,
+      "brier_warn": 0.016091510788709233,
+      "events": [
+       {
+        "crossing": "2024-09-24T10:00:00",
+        "lead_h": 10.0,
+        "peak_level": 8.27,
+        "peak_pred_24h_before": 7.167139790234238
+       },
+       {
+        "crossing": "2024-09-30T03:00:00",
+        "lead_h": 10.0,
+        "peak_level": 5.99,
+        "peak_pred_24h_before": 5.618725525893519
+       },
+       {
+        "crossing": "2024-10-03T06:00:00",
+        "lead_h": 69.0,
+        "peak_level": 9.93,
+        "peak_pred_24h_before": 7.889819144742388
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain": {
+      "mae": 0.1400555603272253,
+      "mae_above_2p5": 0.2920105854094151,
+      "brier_warn": 0.007294262727198402,
+      "events": [
+       {
+        "crossing": "2024-09-24T10:00:00",
+        "lead_h": 19.0,
+        "peak_level": 8.27,
+        "peak_pred_24h_before": 8.04142923647258
+       },
+       {
+        "crossing": "2024-09-30T03:00:00",
+        "lead_h": 10.0,
+        "peak_level": 5.99,
+        "peak_pred_24h_before": 5.42
+       },
+       {
+        "crossing": "2024-10-03T06:00:00",
+        "lead_h": 55.0,
+        "peak_level": 9.93,
+        "peak_pred_24h_before": 8.655375706947945
+       }
+      ],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2025,
+    "n_train": 54577,
+    "n_test": 4392,
+    "events": [
+     {
+      "crossing": "2025-09-26T06:00:00",
+      "peak_ts": "2025-09-27T21:00:00",
+      "peak_level": 6.64
+     },
+     {
+      "crossing": "2025-10-03T06:00:00",
+      "peak_ts": "2025-10-03T12:00:00",
+      "peak_level": 6.14
+     }
+    ],
+    "variants": {
+     "rise": {
+      "mae": 0.21617454799997243,
+      "mae_above_2p5": 0.406245211140301,
+      "brier_warn": 0.012587550711857222,
+      "events": [
+       {
+        "crossing": "2025-09-26T06:00:00",
+        "lead_h": 14.0,
+        "peak_level": 6.64,
+        "peak_pred_24h_before": 5.73
+       },
+       {
+        "crossing": "2025-10-03T06:00:00",
+        "lead_h": 47.0,
+        "peak_level": 6.14,
+        "peak_pred_24h_before": 5.953951787788988
+       }
+      ],
+      "false_alarm_episodes": 1
+     },
+     "rise_rain": {
+      "mae": 0.17651730883692462,
+      "mae_above_2p5": 0.38926658786456825,
+      "brier_warn": 0.015873258461498164,
+      "events": [
+       {
+        "crossing": "2025-09-26T06:00:00",
+        "lead_h": 6.0,
+        "peak_level": 6.64,
+        "peak_pred_24h_before": 5.760150202082536
+       },
+       {
+        "crossing": "2025-10-03T06:00:00",
+        "lead_h": 8.0,
+        "peak_level": 6.14,
+        "peak_pred_24h_before": 5.605274163405912
+       }
+      ],
+      "false_alarm_episodes": 2
+     }
+    }
+   }
+  ]
+ }
+]

models/eval_variants.json

@@ -0,0 +1,723 @@
+[
+ {
+  "station": "P.1",
+  "warn_thr": 3.7,
+  "folds": [
+   {
+    "year": 2021,
+    "n_train": 20024,
+    "n_test": 4392,
+    "events": [],
+    "variants": {
+     "baseline_abs": {
+      "mae": 0.073069548799388,
+      "mae_above_2p5": null,
+      "brier_warn": 0.0,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise": {
+      "mae": 0.070970681677648,
+      "mae_above_2p5": null,
+      "brier_warn": 0.0,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_weighted": {
+      "mae": 0.07394501467643348,
+      "mae_above_2p5": null,
+      "brier_warn": 0.0,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_quantile": {
+      "mae": 0.07289422294875283,
+      "mae_above_2p5": null,
+      "brier_warn": 6.870660393452143e-17,
+      "events": [],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2022,
+    "n_train": 28782,
+    "n_test": 4392,
+    "events": [
+     {
+      "crossing": "2022-10-02T19:00:00",
+      "peak_ts": "2022-10-03T15:00:00",
+      "peak_level": 4.65
+     }
+    ],
+    "variants": {
+     "baseline_abs": {
+      "mae": 0.09128484356553154,
+      "mae_above_2p5": 0.2856025611084259,
+      "brier_warn": 0.005440790049399641,
+      "events": [
+       {
+        "crossing": "2022-10-02T19:00:00",
+        "lead_h": 0.0,
+        "peak_level": 4.65,
+        "peak_pred_24h_before": 3.4
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise": {
+      "mae": 0.08489734638010188,
+      "mae_above_2p5": 0.24517428534843191,
+      "brier_warn": 0.004136576477574926,
+      "events": [
+       {
+        "crossing": "2022-10-02T19:00:00",
+        "lead_h": 6.0,
+        "peak_level": 4.65,
+        "peak_pred_24h_before": 3.870617057762467
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_weighted": {
+      "mae": 0.08451609397908402,
+      "mae_above_2p5": 0.24677443475763222,
+      "brier_warn": 0.0041231071254735656,
+      "events": [
+       {
+        "crossing": "2022-10-02T19:00:00",
+        "lead_h": 6.0,
+        "peak_level": 4.65,
+        "peak_pred_24h_before": 3.9060484870307466
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_quantile": {
+      "mae": 0.08412283956084679,
+      "mae_above_2p5": 0.2653552568120815,
+      "brier_warn": 0.004349073127074364,
+      "events": [
+       {
+        "crossing": "2022-10-02T19:00:00",
+        "lead_h": 3.0,
+        "peak_level": 4.65,
+        "peak_pred_24h_before": 3.641048749261635
+       }
+      ],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2023,
+    "n_train": 37537,
+    "n_test": 4392,
+    "events": [],
+    "variants": {
+     "baseline_abs": {
+      "mae": 0.08161389876263064,
+      "mae_above_2p5": 0.1335546690803318,
+      "brier_warn": 3.284466170848282e-09,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise": {
+      "mae": 0.07647507344443248,
+      "mae_above_2p5": 0.12593127745781565,
+      "brier_warn": 3.19529506149809e-11,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_weighted": {
+      "mae": 0.07633405851952908,
+      "mae_above_2p5": 0.12234064428755373,
+      "brier_warn": 9.093694748477163e-10,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_quantile": {
+      "mae": 0.07460579730428984,
+      "mae_above_2p5": 0.11243109915383911,
+      "brier_warn": 8.148092046865233e-08,
+      "events": [],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2024,
+    "n_train": 46321,
+    "n_test": 4392,
+    "events": [
+     {
+      "crossing": "2024-09-24T17:00:00",
+      "peak_ts": "2024-09-26T02:00:00",
+      "peak_level": 4.93
+     },
+     {
+      "crossing": "2024-10-03T09:00:00",
+      "peak_ts": "2024-10-05T12:00:00",
+      "peak_level": 5.3
+     }
+    ],
+    "variants": {
+     "baseline_abs": {
+      "mae": 0.11011679980584997,
+      "mae_above_2p5": 0.28849258735632527,
+      "brier_warn": 0.010963910189916666,
+      "events": [
+       {
+        "crossing": "2024-09-24T17:00:00",
+        "lead_h": 0.0,
+        "peak_level": 4.93,
+        "peak_pred_24h_before": 4.39
+       },
+       {
+        "crossing": "2024-10-03T09:00:00",
+        "lead_h": 0.0,
+        "peak_level": 5.3,
+        "peak_pred_24h_before": 4.9
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise": {
+      "mae": 0.10075626719251345,
+      "mae_above_2p5": 0.26384939692286363,
+      "brier_warn": 0.011608221543810462,
+      "events": [
+       {
+        "crossing": "2024-09-24T17:00:00",
+        "lead_h": 6.0,
+        "peak_level": 4.93,
+        "peak_pred_24h_before": 4.534709676862131
+       },
+       {
+        "crossing": "2024-10-03T09:00:00",
+        "lead_h": 3.0,
+        "peak_level": 5.3,
+        "peak_pred_24h_before": 5.056619694027486
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_weighted": {
+      "mae": 0.10644718990351541,
+      "mae_above_2p5": 0.2848978622398853,
+      "brier_warn": 0.013872775238121434,
+      "events": [
+       {
+        "crossing": "2024-09-24T17:00:00",
+        "lead_h": 9.0,
+        "peak_level": 4.93,
+        "peak_pred_24h_before": 4.502016075573465
+       },
+       {
+        "crossing": "2024-10-03T09:00:00",
+        "lead_h": 72.0,
+        "peak_level": 5.3,
+        "peak_pred_24h_before": 5.123356409745132
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_quantile": {
+      "mae": 0.09617759284936986,
+      "mae_above_2p5": 0.271753261978628,
+      "brier_warn": 0.011660822672241142,
+      "events": [
+       {
+        "crossing": "2024-09-24T17:00:00",
+        "lead_h": 4.0,
+        "peak_level": 4.93,
+        "peak_pred_24h_before": 4.532042588397363
+       },
+       {
+        "crossing": "2024-10-03T09:00:00",
+        "lead_h": 2.0,
+        "peak_level": 5.3,
+        "peak_pred_24h_before": 4.956335200519836
+       }
+      ],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2025,
+    "n_train": 55081,
+    "n_test": 4392,
+    "events": [
+     {
+      "crossing": "2025-09-27T18:00:00",
+      "peak_ts": "2025-09-27T22:00:00",
+      "peak_level": 3.93
+     }
+    ],
+    "variants": {
+     "baseline_abs": {
+      "mae": 0.15985415338980086,
+      "mae_above_2p5": 0.277881996645181,
+      "brier_warn": 0.009824391159106445,
+      "events": [
+       {
+        "crossing": "2025-09-27T18:00:00",
+        "lead_h": 0.0,
+        "peak_level": 3.93,
+        "peak_pred_24h_before": 3.2561560254201525
+       }
+      ],
+      "false_alarm_episodes": 2
+     },
+     "rise": {
+      "mae": 0.13102686839694494,
+      "mae_above_2p5": 0.2729944665011028,
+      "brier_warn": 0.010889835530782944,
+      "events": [
+       {
+        "crossing": "2025-09-27T18:00:00",
+        "lead_h": 46.0,
+        "peak_level": 3.93,
+        "peak_pred_24h_before": 3.232029710676319
+       }
+      ],
+      "false_alarm_episodes": 1
+     },
+     "rise_weighted": {
+      "mae": 0.13121120517634968,
+      "mae_above_2p5": 0.27796411411759725,
+      "brier_warn": 0.009393099541680463,
+      "events": [
+       {
+        "crossing": "2025-09-27T18:00:00",
+        "lead_h": 46.0,
+        "peak_level": 3.93,
+        "peak_pred_24h_before": 3.23
+       }
+      ],
+      "false_alarm_episodes": 3
+     },
+     "rise_quantile": {
+      "mae": 0.11678707793162903,
+      "mae_above_2p5": 0.28528124941530186,
+      "brier_warn": 0.013009605253899563,
+      "events": [
+       {
+        "crossing": "2025-09-27T18:00:00",
+        "lead_h": 45.0,
+        "peak_level": 3.93,
+        "peak_pred_24h_before": 3.23
+       }
+      ],
+      "false_alarm_episodes": 3
+     }
+    }
+   }
+  ]
+ },
+ {
+  "station": "P.103",
+  "warn_thr": 5.95,
+  "folds": [
+   {
+    "year": 2021,
+    "n_train": 20009,
+    "n_test": 4392,
+    "events": [],
+    "variants": {
+     "baseline_abs": {
+      "mae": 0.1517830652111977,
+      "mae_above_2p5": null,
+      "brier_warn": 0.0,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise": {
+      "mae": 0.15279228710793116,
+      "mae_above_2p5": null,
+      "brier_warn": 0.0,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_weighted": {
+      "mae": 0.17632375724441426,
+      "mae_above_2p5": null,
+      "brier_warn": 0.0,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_quantile": {
+      "mae": 0.13593873552532,
+      "mae_above_2p5": null,
+      "brier_warn": 1.6481640122270367e-09,
+      "events": [],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2022,
+    "n_train": 28769,
+    "n_test": 4392,
+    "events": [
+     {
+      "crossing": "2022-08-14T04:00:00",
+      "peak_ts": "2022-08-14T08:00:00",
+      "peak_level": 6.09
+     },
+     {
+      "crossing": "2022-10-02T16:00:00",
+      "peak_ts": "2022-10-03T16:00:00",
+      "peak_level": 7.54
+     }
+    ],
+    "variants": {
+     "baseline_abs": {
+      "mae": 0.15960776318568,
+      "mae_above_2p5": 0.5240325509357986,
+      "brier_warn": 0.010134410144044625,
+      "events": [
+       {
+        "crossing": "2022-08-14T04:00:00",
+        "lead_h": 0.0,
+        "peak_level": 6.09,
+        "peak_pred_24h_before": 5.01930531142127
+       },
+       {
+        "crossing": "2022-10-02T16:00:00",
+        "lead_h": 0.0,
+        "peak_level": 7.54,
+        "peak_pred_24h_before": 6.01
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise": {
+      "mae": 0.15389446229226025,
+      "mae_above_2p5": 0.41887094569113953,
+      "brier_warn": 0.007740077230532647,
+      "events": [
+       {
+        "crossing": "2022-08-14T04:00:00",
+        "lead_h": 6.0,
+        "peak_level": 6.09,
+        "peak_pred_24h_before": 4.837975953559253
+       },
+       {
+        "crossing": "2022-10-02T16:00:00",
+        "lead_h": 9.0,
+        "peak_level": 7.54,
+        "peak_pred_24h_before": 7.168225721504108
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_weighted": {
+      "mae": 0.1498671765711928,
+      "mae_above_2p5": 0.40088103598066704,
+      "brier_warn": 0.00801963186023152,
+      "events": [
+       {
+        "crossing": "2022-08-14T04:00:00",
+        "lead_h": 6.0,
+        "peak_level": 6.09,
+        "peak_pred_24h_before": 4.81
+       },
+       {
+        "crossing": "2022-10-02T16:00:00",
+        "lead_h": 8.0,
+        "peak_level": 7.54,
+        "peak_pred_24h_before": 7.002000385945215
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_quantile": {
+      "mae": 0.13833442255794418,
+      "mae_above_2p5": 0.4529809369770209,
+      "brier_warn": 0.009090606171021184,
+      "events": [
+       {
+        "crossing": "2022-08-14T04:00:00",
+        "lead_h": 1.0,
+        "peak_level": 6.09,
+        "peak_pred_24h_before": 4.8698313890426705
+       },
+       {
+        "crossing": "2022-10-02T16:00:00",
+        "lead_h": 5.0,
+        "peak_level": 7.54,
+        "peak_pred_24h_before": 6.650326949981244
+       }
+      ],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2023,
+    "n_train": 37524,
+    "n_test": 4392,
+    "events": [],
+    "variants": {
+     "baseline_abs": {
+      "mae": 0.1489843074454942,
+      "mae_above_2p5": null,
+      "brier_warn": 1.882373500761472e-08,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise": {
+      "mae": 0.16106550376855028,
+      "mae_above_2p5": null,
+      "brier_warn": 0.0001779564366636896,
+      "events": [],
+      "false_alarm_episodes": 1
+     },
+     "rise_weighted": {
+      "mae": 0.1688589840007776,
+      "mae_above_2p5": null,
+      "brier_warn": 0.0012753245446952602,
+      "events": [],
+      "false_alarm_episodes": 1
+     },
+     "rise_quantile": {
+      "mae": 0.1510624979748974,
+      "mae_above_2p5": null,
+      "brier_warn": 0.0005959001115900152,
+      "events": [],
+      "false_alarm_episodes": 1
+     }
+    }
+   },
+   {
+    "year": 2024,
+    "n_train": 46308,
+    "n_test": 4058,
+    "events": [
+     {
+      "crossing": "2024-09-24T10:00:00",
+      "peak_ts": "2024-09-26T00:00:00",
+      "peak_level": 8.27
+     },
+     {
+      "crossing": "2024-09-30T03:00:00",
+      "peak_ts": "2024-09-30T06:00:00",
+      "peak_level": 5.99
+     },
+     {
+      "crossing": "2024-10-03T06:00:00",
+      "peak_ts": "2024-10-05T07:00:00",
+      "peak_level": 9.93
+     }
+    ],
+    "variants": {
+     "baseline_abs": {
+      "mae": 0.193584781859895,
+      "mae_above_2p5": 0.5381754200484403,
+      "brier_warn": 0.018292133172974026,
+      "events": [
+       {
+        "crossing": "2024-09-24T10:00:00",
+        "lead_h": 5.0,
+        "peak_level": 8.27,
+        "peak_pred_24h_before": 7.11
+       },
+       {
+        "crossing": "2024-09-30T03:00:00",
+        "lead_h": 0.0,
+        "peak_level": 5.99,
+        "peak_pred_24h_before": 5.42
+       },
+       {
+        "crossing": "2024-10-03T06:00:00",
+        "lead_h": 0.0,
+        "peak_level": 9.93,
+        "peak_pred_24h_before": 7.86
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise": {
+      "mae": 0.17314177863843333,
+      "mae_above_2p5": 0.48294427141283425,
+      "brier_warn": 0.016091510788709233,
+      "events": [
+       {
+        "crossing": "2024-09-24T10:00:00",
+        "lead_h": 10.0,
+        "peak_level": 8.27,
+        "peak_pred_24h_before": 7.167139790234238
+       },
+       {
+        "crossing": "2024-09-30T03:00:00",
+        "lead_h": 10.0,
+        "peak_level": 5.99,
+        "peak_pred_24h_before": 5.618725525893519
+       },
+       {
+        "crossing": "2024-10-03T06:00:00",
+        "lead_h": 69.0,
+        "peak_level": 9.93,
+        "peak_pred_24h_before": 7.889819144742388
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_weighted": {
+      "mae": 0.18833560689452924,
+      "mae_above_2p5": 0.4948752445774323,
+      "brier_warn": 0.01622076553986666,
+      "events": [
+       {
+        "crossing": "2024-09-24T10:00:00",
+        "lead_h": 11.0,
+        "peak_level": 8.27,
+        "peak_pred_24h_before": 7.283961881370446
+       },
+       {
+        "crossing": "2024-09-30T03:00:00",
+        "lead_h": 10.0,
+        "peak_level": 5.99,
+        "peak_pred_24h_before": 5.783590096006894
+       },
+       {
+        "crossing": "2024-10-03T06:00:00",
+        "lead_h": 69.0,
+        "peak_level": 9.93,
+        "peak_pred_24h_before": 7.922098265471745
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_quantile": {
+      "mae": 0.1644888402154109,
+      "mae_above_2p5": 0.46461119464587947,
+      "brier_warn": 0.013172046828124688,
+      "events": [
+       {
+        "crossing": "2024-09-24T10:00:00",
+        "lead_h": 11.0,
+        "peak_level": 8.27,
+        "peak_pred_24h_before": 7.348489352370365
+       },
+       {
+        "crossing": "2024-09-30T03:00:00",
+        "lead_h": 8.0,
+        "peak_level": 5.99,
+        "peak_pred_24h_before": 5.42
+       },
+       {
+        "crossing": "2024-10-03T06:00:00",
+        "lead_h": 2.0,
+        "peak_level": 9.93,
+        "peak_pred_24h_before": 7.91343423984052
+       }
+      ],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2025,
+    "n_train": 54577,
+    "n_test": 4392,
+    "events": [
+     {
+      "crossing": "2025-09-26T06:00:00",
+      "peak_ts": "2025-09-27T21:00:00",
+      "peak_level": 6.64
+     },
+     {
+      "crossing": "2025-10-03T06:00:00",
+      "peak_ts": "2025-10-03T12:00:00",
+      "peak_level": 6.14
+     }
+    ],
+    "variants": {
+     "baseline_abs": {
+      "mae": 0.2036944006891808,
+      "mae_above_2p5": 0.40787978637525985,
+      "brier_warn": 0.014130122065259989,
+      "events": [
+       {
+        "crossing": "2025-09-26T06:00:00",
+        "lead_h": 14.0,
+        "peak_level": 6.64,
+        "peak_pred_24h_before": 5.73
+       },
+       {
+        "crossing": "2025-10-03T06:00:00",
+        "lead_h": 41.0,
+        "peak_level": 6.14,
+        "peak_pred_24h_before": 5.703778873833753
+       }
+      ],
+      "false_alarm_episodes": 2
+     },
+     "rise": {
+      "mae": 0.21617454799997243,
+      "mae_above_2p5": 0.406245211140301,
+      "brier_warn": 0.012587550711857222,
+      "events": [
+       {
+        "crossing": "2025-09-26T06:00:00",
+        "lead_h": 14.0,
+        "peak_level": 6.64,
+        "peak_pred_24h_before": 5.73
+       },
+       {
+        "crossing": "2025-10-03T06:00:00",
+        "lead_h": 47.0,
+        "peak_level": 6.14,
+        "peak_pred_24h_before": 5.953951787788988
+       }
+      ],
+      "false_alarm_episodes": 1
+     },
+     "rise_weighted": {
+      "mae": 0.21768864574559124,
+      "mae_above_2p5": 0.41071503357135775,
+      "brier_warn": 0.014605838473436519,
+      "events": [
+       {
+        "crossing": "2025-09-26T06:00:00",
+        "lead_h": 13.0,
+        "peak_level": 6.64,
+        "peak_pred_24h_before": 5.73
+       },
+       {
+        "crossing": "2025-10-03T06:00:00",
+        "lead_h": 49.0,
+        "peak_level": 6.14,
+        "peak_pred_24h_before": 5.606011827978066
+       }
+      ],
+      "false_alarm_episodes": 1
+     },
+     "rise_quantile": {
+      "mae": 0.18196250028633115,
+      "mae_above_2p5": 0.3987262421172889,
+      "brier_warn": 0.011560937993250782,
+      "events": [
+       {
+        "crossing": "2025-09-26T06:00:00",
+        "lead_h": 13.0,
+        "peak_level": 6.64,
+        "peak_pred_24h_before": 5.73
+       },
+       {
+        "crossing": "2025-10-03T06:00:00",
+        "lead_h": 42.0,
+        "peak_level": 6.14,
+        "peak_pred_24h_before": 6.00597561680636
+       }
+      ],
+      "false_alarm_episodes": 1
+     }
+    }
+   }
+  ]
+ }
+]

pyproject.toml

@@ -0,0 +1,77 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "northern-thailand-ping-river-monitor"
+version = "3.1.3"
+description = "Ping River Monitor: flood-model training, evaluation and archive backfills (the service, ping-collector, is in rust/)"
+readme = "README.md"
+license = {text = "MIT"}
+authors = [
+    {name = "buildfor.life"}
+]
+keywords = [
+    "water monitoring",
+    "hydrology",
+    "flood forecasting",
+    "thailand",
+    "ping river",
+    "time series",
+]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Science/Research",
+    "Topic :: Scientific/Engineering :: Hydrology",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Operating System :: OS Independent",
+]
+# 3.11 only: pandas 2.0.3 ships no 3.12 wheels; psycopg2-binary 2.9.9 breaks on 3.13
+requires-python = ">=3.11,<3.12"
+dependencies = [
+    "requests==2.34.2",
+    "pandas==2.0.3",
+    "numpy>=1.24,<2",  # pandas 2.0.3 wheels are ABI-incompatible with numpy 2.x
+    "scikit-learn==1.9.0",
+    "sqlalchemy==2.0.23",
+    "psycopg2-binary==2.9.9",
+    # .env loading; rust/collector/src/envfile.rs ports exactly this version
+    "python-dotenv==1.1.1",
+]
+
+[project.urls]
+Homepage = "https://water.buildfor.life/"
+Repository = "https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor"
+Issues = "https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/issues"
+
+[dependency-groups]
+dev = [
+    "pytest==9.1.1",
+    "pytest-cov==4.1.0",
+    "black==26.5.1",
+    "flake8==6.1.0",
+    "isort==5.12.0",
+    "pre-commit==3.5.0",
+    "ipython==8.17.2",
+    "jupyter==1.0.0",
+]
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-dir]
+"" = "src"
+
+# One formatting contract for CI, pre-commit and editors. Black's default 88
+# columns; isort in black-compatible mode. Run `make format` before committing.
+[tool.black]
+line-length = 88
+target-version = ["py311"]
+extend-exclude = '/(\.venv|venv|models|\.claude-flow|\.swarm)/'
+
+[tool.isort]
+profile = "black"
+line_length = 88
+known_first_party = ["src"]
+skip_gitignore = true

requirements-dev.txt

@@ -1,29 +1,8 @@
-# Development dependencies
 -r requirements.txt
 
-# Testing
-pytest==7.4.3
+pytest==9.1.1
 pytest-cov==4.1.0
-pytest-asyncio==0.21.1
-
-# Code formatting and linting
-black==23.11.0
+black==26.5.1
 flake8==6.1.0
 isort==5.12.0
-mypy==1.7.1
-
-# Pre-commit hooks
 pre-commit==3.5.0
-
-# Documentation
-sphinx==7.2.6
-sphinx-rtd-theme==1.3.0
-sphinx-autodoc-typehints==1.25.2
-
-# Development tools
-ipython==8.17.2
-jupyter==1.0.0
-
-# Type stubs
-types-requests==2.31.0.10
-types-python-dateutil==2.8.19.14

requirements.txt

@@ -1,30 +1,9 @@
-# Core dependencies
-requests==2.31.0
-schedule==1.2.0
+# Python side: flood-model training, evaluation, backfills (pyproject.toml is
+# the source; uv.lock pins the rest). The service is ping-collector (rust/).
+requests==2.34.2
 pandas==2.0.3
-
-# Web API framework
-fastapi==0.104.1
-uvicorn[standard]==0.24.0
-pydantic==2.5.0
-
-# Database adapters
+numpy>=1.24,<2  # pandas 2.0.3 wheels are ABI-incompatible with numpy 2.x
+scikit-learn==1.9.0
 sqlalchemy==2.0.23
-influxdb==5.3.1
-pymysql==1.1.0
 psycopg2-binary==2.9.9
-
-# Monitoring and metrics
-psutil==5.9.6
-
-# Development dependencies (optional)
-pytest==7.4.3
-pytest-cov==4.1.0
-black==23.11.0
-flake8==6.1.0
-mypy==1.7.1
-pre-commit==3.5.0
-
-# Documentation
-sphinx==7.2.6
-sphinx-rtd-theme==1.3.0
+python-dotenv==1.1.1

run.py

@@ -1,14 +0,0 @@
-#!/usr/bin/env python3
-"""
-Simple startup script for Thailand Water Monitor
-"""
-
-import sys
-import os
-
-# Add src directory to Python path
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src'))
-
-if __name__ == "__main__":
-    from src.main import main
-    main()

rust/.gitignore

@@ -0,0 +1 @@
+target/

rust/Cargo.toml

@@ -0,0 +1,39 @@
+[workspace]
+resolver = "3"
+members = ["collector", "forecast"]
+
+[workspace.package]
+# 1.88: edition 2024 let-chains, and dependencies (home, icu_*) need it. Distro
+# cargo packages are older: install the toolchain with rustup (rust/README.md).
+edition = "2024"
+rust-version = "1.88"
+license = "MIT"
+publish = false
+repository = "https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor"
+
+[workspace.dependencies]
+anyhow = "1"
+axum = { version = "0.8", default-features = false, features = ["http1", "tokio", "query", "json", "matched-path", "original-uri"] }
+chrono = { version = "0.4", default-features = false, features = ["std", "clock", "serde"] }
+clap = { version = "4", features = ["derive"] }
+flate2 = "1"
+form_urlencoded = "1"
+log = "0.4"
+# native-tls (OpenSSL on Linux, SChannel on Windows), not rustls: app.rid.go.th (dams)
+# offers only TLS 1.2 CBC suites (ECDHE-RSA-AES256-SHA384), which rustls refuses.
+reqwest = { version = "0.12", default-features = false, features = ["native-tls", "json"] }
+serde = { version = "1", features = ["derive"] }
+# preserve_order: JSON objects keep the order they were built in (API
+# responses list their fields in the same order as the FastAPI app did)
+serde_json = { version = "1", features = ["float_roundtrip", "preserve_order"] }
+sqlx = { version = "0.8", default-features = false, features = ["runtime-tokio", "tls-rustls", "postgres", "chrono", "macros"] }
+tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "signal", "net", "sync", "fs"] }
+tower = { version = "0.5", default-features = false, features = ["util"] }
+tower-http = { version = "0.6", features = ["compression-gzip", "cors", "fs"] }
+tracing = "0.1"
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+
+[profile.release]
+lto = "thin"
+codegen-units = 1
+strip = true

rust/README.md

@@ -0,0 +1,145 @@
+# Rust: the service
+
+Since October 2026 the whole Ping River Monitor service is one Rust binary,
+`ping-collector`; Python is left with model training and evaluation
+(`src/ml`) and the archive backfills. How it got here, and the parity rules
+the rewrite followed: [docs/RUST_REWRITE.md](../docs/RUST_REWRITE.md).
+
+| Crate | What |
+| --- | --- |
+| `collector` (`ping-collector`) | collection (RID, HII/ThaiWater, RID dams, Open-Meteo), the forecast run, ntfy + Matrix notifications, the website and API (`collector/src/web`) |
+| `forecast` (`ping-forecast`, library) | flood-forecast inference from the models trained in Python: features, HII gap fill, gradient-boosted trees, heuristic fallback |
+
+## ping-collector
+
+```text
+ping-collector run                  # the service (systemd): startup gap fill, RID at :05 (+10 min
+                                    # retries while RID is behind), HII/dams/rain once per clock
+                                    # hour; FORECAST_IN_COLLECTOR=true: forecast + ntfy/Matrix
+                                    # after each cycle with newer readings; WEB_LISTEN: website/API
+ping-collector serve [--listen A] [--no-analytics]
+                                    # the website and API alone (default 127.0.0.1:8001)
+ping-collector once [--dry-run]     # one cycle of every source; --dry-run writes nothing
+                                    # and compares the RID fetch with the database
+ping-collector fill-gaps <days|all> # re-fetch RID reports with holes
+ping-collector check-db             # connect, create missing tables, row counts
+ping-collector forecast [--json] [--save] [--now T] [--rain-file F]
+                                    # forecast from the database and print it
+ping-collector notify --dry-run     # the readings the notifier judges, and what it would send
+```
+
+Configuration: the environment, or `.env` in the working directory
+(`--env-file` for another), every key with its default in
+[.env.example](../.env.example). `.env` is parsed with python-dotenv's rules
+(`collector/src/envfile.rs`, verified on 614 generated inputs) because the
+Python tools read the same file: an unparsable line is skipped with a warning
+giving only its line number, never its value. Variables already set (e.g. by
+systemd's `EnvironmentFile=`) win. All timestamps are Asia/Bangkok wall-clock,
+whatever the server's time zone.
+
+### Collection
+
+The tables, upserts and parsers are the ones the Python collectors had (the
+DDL is in `collector/src/db.rs`; the Python backfill stores create the same).
+RID's evening report often lags 1-2 h: every poll stores every reading the
+report has, yesterday's report is re-fetched while it has holes, and the
+startup gap fill repairs the last `GAP_FILL_DAYS_ON_START` days.
+
+### Forecast and notifications
+
+After every cycle that brought newer RID or HII readings (and at least
+hourly) the forecast runs over the last 336 h, is stored in `forecast_history`
+(for skill tracking) and `forecast_current`, and the notifier compares the
+readings and forecasts with `notification_state`.
+
+- **Models** are trained in Python. `src/ml/export.py` writes
+  `models/flood_<station>.json.gz` next to each joblib bundle (`train_all()`
+  does it; `python -m src.ml.export` for existing bundles); the service
+  reloads changed files on its next run, so a retrain needs no restart.
+- **Shared constants** (thresholds, flood stages, upstream leads, notifier
+  settings) are compiled in from `forecast/flood_config.json`, generated by
+  `uv run python -m src.ml.export --config`. `tests/test_model_export.py`
+  fails while it differs from Python; a change needs a rebuild.
+- **ntfy** topics, texts and hysteresis: [docs/NOTIFICATIONS.md](../docs/NOTIFICATIONS.md).
+  For the stations whose ThaiWater twin is the same sensor, a reading newer
+  than RID's (RID lags 2-10 h at night) drives the level alerts, marked as
+  ThaiWater telemetry. "Gauge feed stale" means no source has a reading for
+  3 h; RID alone lagging is reported once after `RID_STALE_ALERT_HOURS`.
+- **Matrix** (`MATRIX_*`): the room gets the same transitions (per station,
+  outlook, feed; not ntfy's per-topic summaries), each once, tracked under
+  its own `matrix:` keys in `notification_state`, so a Matrix outage is
+  retried on Matrix only. [docs/MATRIX_QUICK_START.md](../docs/MATRIX_QUICK_START.md).
+- Memory: under 100 MB resident in production, all 15 models loaded
+  (24-byte tree nodes); the unit allows 512 MB.
+
+### Website and API
+
+`collector/src/web` (axum) serves the dashboard (`STATIC_DIR`/dashboard.html,
+default `src/static`, loaded at start), `/static`, the crawler files,
+`/docs` + `/redoc` + `/openapi.json`, and the read API, with gzip,
+Cache-Control per path, CORS (`CORS_ALLOW_ORIGINS`), Umami API tracking
+(`UMAMI_*`) and stale-while-revalidate response caches with byte budgets.
+Station editing, `/config` and `POST /scrape/trigger` need `ADMIN_API_KEY`
+(`X-API-Key` header).
+
+- Parameters are declared once per route (`web/params.rs`): the same list
+  validates the request (pydantic-style 422s) and builds the OpenAPI
+  document (`web/openapi.rs`), so `/docs` cannot drift from what is accepted.
+- The response formats are the retired FastAPI app's, kept so clients did
+  not notice the switch: field order, `isoformat()` timestamps, Python's
+  `round()`, numpy's pairwise sums for the catchment means (`web/pyfmt.rs`),
+  `{"detail": ...}` errors.
+- An old stored forecast is served marked `X-Data-Stale: true`; `/health`
+  counts current ThaiWater telemetry of the same gauges while RID lags.
+
+## Develop
+
+```bash
+cd rust
+cargo fmt --all && cargo clippy --all-targets -- -D warnings
+docker run -d --name ping-pg -e POSTGRES_PASSWORD=ping -e POSTGRES_DB=ping -p 55432:5432 postgres:16
+TEST_DATABASE_URL=postgres://postgres:ping@localhost:55432/ping cargo test
+CARGO_TARGET_DIR=target/msrv cargo +1.88 check --all-targets --locked   # the minimum version
+```
+
+Without `TEST_DATABASE_URL` the database tests skip. CI
+(`.gitea/workflows/rust.yml`) runs them against a PostgreSQL service and
+uploads the Linux binary as the `ping-collector-linux-x86_64` artifact.
+
+- The forecast's parity with the Python reference: `forecast/tests/parity.rs`
+  replays fixtures from the Python production path
+  (`forecast/testdata/gen_forecast_fixtures.py`; regenerate after a change to
+  `src/ml/features.py`, `predict.py`, the HII fill or the dam frame). Features
+  and levels must stay bit-identical, probabilities within a few ulp.
+  `scripts/forecast_parity.py` compares the two on a live database.
+- The notifier's texts are pinned by `collector/testdata/notify_python_cases.json.gz`
+  (751 messages recorded from the Python notifier before it was retired).
+
+HTTP uses native-tls (OpenSSL on Linux), not rustls: app.rid.go.th (dams)
+only offers TLS 1.2 CBC cipher suites, which rustls refuses. Building on
+Linux needs `pkg-config` and `libssl-dev`.
+
+## Deploy
+
+Needs Rust >= 1.88. Distro `cargo` packages are older (`resolver 3 is not
+valid` / `edition 2024` errors), so install the toolchain once with rustup:
+
+```bash
+apt-get install -y build-essential pkg-config libssl-dev curl
+curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal
+. "$HOME/.cargo/env"        # puts ~/.cargo/bin before /usr/bin/cargo
+cargo --version             # >= 1.88
+```
+
+Then on the server, in the checkout:
+
+```bash
+git pull
+cargo build --release --locked --manifest-path rust/Cargo.toml   # or BIN=<CI artifact>
+sudo bash scripts/install.sh
+journalctl -u ping-collector -f
+```
+
+`scripts/install.sh` installs the binary and the units, syncs the Python
+training environment, runs a dry run of every source and the forecast check,
+restarts the service and waits for the website (details in its header).

rust/collector/Cargo.toml

@@ -0,0 +1,30 @@
+[package]
+name = "ping-collector"
+version = "0.1.0"
+description = "Ping River Monitor service: collection (RID, HII/ThaiWater, RID dams, Open-Meteo), forecast, notifications and the website/API"
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+publish.workspace = true
+repository.workspace = true
+
+[dependencies]
+anyhow.workspace = true
+axum.workspace = true
+chrono.workspace = true
+clap.workspace = true
+form_urlencoded.workspace = true
+log.workspace = true
+ping-forecast = { path = "../forecast" }
+reqwest.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+sqlx.workspace = true
+tokio.workspace = true
+tower.workspace = true
+tower-http.workspace = true
+tracing.workspace = true
+tracing-subscriber.workspace = true
+
+[dev-dependencies]
+flate2.workspace = true

rust/collector/src/config.rs

@@ -0,0 +1,299 @@
+//! Configuration from the environment / `.env` file
+//! (`/opt/thailand-water-monitor/.env`), which the Python training tools
+//! (src/config.py) read too. Names and defaults are the ones the Python app
+//! used.
+
+use anyhow::{Context, Result, bail};
+use std::env;
+use std::path::PathBuf;
+
+pub const RID_API_URL: &str =
+    "https://hyd-app-db.rid.go.th/webservice/getGroupHourlyWaterLevelReportAllHL.ashx";
+pub const HII_API_BASE: &str = "https://api-v3.thaiwater.net/api/v1/thaiwater30/public";
+pub const RID_DAMS_URL: &str = "https://app.rid.go.th/reservoir/api/dams";
+pub const OPENMETEO_URL: &str = "https://api.open-meteo.com/v1/forecast";
+/// The RID endpoint serves the same JSON to any client; keep the UA the
+/// Python collector has used for years.
+pub const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 \
+     (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36";
+
+#[derive(Clone, Debug)]
+pub struct Config {
+    pub database_url: String,
+    pub rid_api_url: String,
+    pub hii_api_base: String,
+    pub dams_url: String,
+    pub openmeteo_url: String,
+    pub hii_enabled: bool,
+    pub dams_enabled: bool,
+    pub rain_enabled: bool,
+    pub hii_basin_code: i64,
+    /// RID is polled this many minutes past each hour.
+    pub poll_offset_minutes: u32,
+    /// While the current hour is not published yet, retry this often.
+    pub retry_minutes: u32,
+    /// Days of RID holes re-fetched at startup (0 = off).
+    pub gap_fill_days_on_start: u32,
+    pub station_config_path: PathBuf,
+    /// Status JSON for /scraping/status when the website runs in a separate
+    /// process (`ping-collector serve`).
+    pub status_file: Option<PathBuf>,
+    /// FORECAST_IN_COLLECTOR: forecast and notify (off: collect only).
+    pub forecast_enabled: bool,
+    /// Exported bundles (`flood_<station>.json.gz`, src/ml/export.py).
+    pub models_dir: PathBuf,
+    /// Where to POST ntfy messages; `None` (NTFY_SERVER unset) = off.
+    pub ntfy_publish_url: Option<String>,
+    pub ntfy_token: Option<String>,
+    pub ntfy_topic_prefix: String,
+    /// Dashboard link in notifications.
+    pub public_url: String,
+    /// Report RID lagging this long while ThaiWater is current (0 = never).
+    pub rid_stale_alert_hours: f64,
+    /// Public ntfy URL (what subscribers use), shown by /api/notifications.
+    pub ntfy_server: Option<String>,
+    /// MATRIX_*: the notifications also go to this room (off when unset).
+    pub matrix: Option<MatrixConfig>,
+    pub web: WebConfig,
+}
+
+#[derive(Clone, Debug)]
+pub struct MatrixConfig {
+    pub homeserver: String,
+    pub access_token: String,
+    pub room_id: String,
+}
+
+/// The website and read API (setting names as the retired Python app had them).
+#[derive(Clone, Debug)]
+pub struct WebConfig {
+    /// WEB_LISTEN: `run` also serves the website on this address
+    /// (e.g. 0.0.0.0:8000); unset = collection only.
+    pub listen: Option<String>,
+    /// Dashboard, robots/llms/sitemap and /static files.
+    pub static_dir: PathBuf,
+    /// X-API-Key for the station editing, /config and /scrape/trigger;
+    /// unset = those endpoints are disabled.
+    pub admin_api_key: Option<String>,
+    pub cors_allow_origins: Vec<String>,
+    pub umami_api_url: String,
+    /// Empty = no server-side analytics.
+    pub umami_website_id: String,
+    pub umami_track_api: bool,
+    pub hii_cache_ttl_s: u64,
+    pub latest_cache_ttl_s: u64,
+    pub health_cache_ttl_s: u64,
+    /// A stored forecast older than this is served marked `X-Data-Stale`.
+    pub forecast_max_age_s: u64,
+    pub thaiwater_api_key: Option<String>,
+    pub request_timeout_s: u64,
+}
+
+/// Variable lookup (the process environment, or a map in tests).
+struct Vars<'a>(&'a dyn Fn(&str) -> Option<String>);
+
+impl Vars<'_> {
+    fn var(&self, name: &str) -> Option<String> {
+        (self.0)(name)
+            .map(|v| v.trim().to_string())
+            .filter(|v| !v.is_empty())
+    }
+
+    /// `os.getenv(name, default)`: a variable set to "" stays "".
+    fn raw_var(&self, name: &str, default: &str) -> String {
+        (self.0)(name).map_or_else(|| default.to_string(), |v| v.trim().to_string())
+    }
+
+    fn flag(&self, name: &str, default: bool) -> bool {
+        self.var(name).map_or(default, |v| {
+            matches!(v.to_lowercase().as_str(), "1" | "true" | "yes")
+        })
+    }
+
+    fn number<T: std::str::FromStr>(&self, name: &str, default: T) -> Result<T>
+    where
+        T::Err: std::fmt::Display,
+    {
+        match self.var(name) {
+            None => Ok(default),
+            Some(v) => v.parse().map_err(|e| anyhow::anyhow!("{name}={v:?}: {e}")),
+        }
+    }
+}
+
+/// Percent-encode everything outside RFC 3986 "unreserved" (as Python's
+/// `urllib.parse.quote(password, safe="")`).
+fn encode_userinfo(s: &str) -> String {
+    s.bytes()
+        .map(|b| match b {
+            b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'.' | b'_' | b'~' => {
+                (b as char).to_string()
+            }
+            _ => format!("%{b:02X}"),
+        })
+        .collect()
+}
+
+/// SQLAlchemy URLs carry a driver suffix (`postgresql+psycopg2://`) that
+/// sqlx does not understand.
+pub fn normalize_database_url(url: &str) -> String {
+    match url.split_once("://") {
+        Some((scheme, rest))
+            if scheme.starts_with("postgresql") || scheme.starts_with("postgres") =>
+        {
+            format!("postgres://{rest}")
+        }
+        _ => url.to_string(),
+    }
+}
+
+impl Config {
+    pub fn from_env() -> Result<Self> {
+        Self::from_vars(&|name| env::var(name).ok())
+    }
+
+    pub fn from_vars(get: &dyn Fn(&str) -> Option<String>) -> Result<Self> {
+        let v = Vars(get);
+        let database_url = match v.var("POSTGRES_CONNECTION_STRING") {
+            Some(url) => normalize_database_url(&url),
+            None => {
+                let password = v
+                    .var("POSTGRES_PASSWORD")
+                    .context("POSTGRES_CONNECTION_STRING or POSTGRES_PASSWORD must be set")?;
+                format!(
+                    "postgres://{}:{}@{}:{}/{}",
+                    encode_userinfo(&v.var("POSTGRES_USER").unwrap_or_else(|| "postgres".into())),
+                    encode_userinfo(&password),
+                    v.var("POSTGRES_HOST").unwrap_or_else(|| "localhost".into()),
+                    v.var("POSTGRES_PORT").unwrap_or_else(|| "5432".into()),
+                    v.var("POSTGRES_DB")
+                        .unwrap_or_else(|| "water_monitoring".into()),
+                )
+            }
+        };
+        if let Some(db_type) = v.var("DB_TYPE")
+            && db_type != "postgresql"
+        {
+            bail!("ping-collector supports PostgreSQL only (DB_TYPE={db_type})");
+        }
+        let cfg = Config {
+            database_url,
+            rid_api_url: v.var("RID_API_URL").unwrap_or_else(|| RID_API_URL.into()),
+            hii_api_base: v.var("HII_API_BASE").unwrap_or_else(|| HII_API_BASE.into()),
+            dams_url: v.var("RID_DAMS_URL").unwrap_or_else(|| RID_DAMS_URL.into()),
+            openmeteo_url: v
+                .var("OPENMETEO_URL")
+                .unwrap_or_else(|| OPENMETEO_URL.into()),
+            hii_enabled: v.flag("ENABLE_HII_COLLECTION", true),
+            dams_enabled: v.flag("ENABLE_RESERVOIR_COLLECTION", true),
+            rain_enabled: v.flag("ENABLE_RAIN_COLLECTION", true),
+            hii_basin_code: v.number("HII_BASIN_CODE", 6)?,
+            poll_offset_minutes: v.number("RID_POLL_OFFSET_MINUTES", 5)?,
+            retry_minutes: v.number("RID_RETRY_MINUTES", 10)?,
+            gap_fill_days_on_start: v.number("GAP_FILL_DAYS_ON_START", 14)?,
+            station_config_path: v
+                .var("STATION_CONFIG_PATH")
+                .unwrap_or_else(|| "stations.json".into())
+                .into(),
+            status_file: v.var("SCRAPING_STATUS_FILE").map(PathBuf::from),
+            forecast_enabled: v.flag("FORECAST_IN_COLLECTOR", false),
+            models_dir: v
+                .var("MODELS_DIR")
+                .unwrap_or_else(|| "models".into())
+                .into(),
+            // as src/config.py: off unless NTFY_SERVER is set
+            ntfy_publish_url: v
+                .var("NTFY_SERVER")
+                .map(|server| v.var("NTFY_PUBLISH_URL").unwrap_or(server)),
+            ntfy_token: v.var("NTFY_TOKEN"),
+            ntfy_topic_prefix: v.var("NTFY_TOPIC_PREFIX").unwrap_or_else(|| "ping".into()),
+            public_url: v
+                .var("PUBLIC_URL")
+                .unwrap_or_else(|| "https://water.buildfor.life/".into()),
+            rid_stale_alert_hours: v.number("RID_STALE_ALERT_HOURS", 18.0)?,
+            ntfy_server: v.var("NTFY_SERVER"),
+            matrix: match (v.var("MATRIX_ACCESS_TOKEN"), v.var("MATRIX_ROOM_ID")) {
+                (Some(access_token), Some(room_id)) => Some(MatrixConfig {
+                    homeserver: v
+                        .var("MATRIX_HOMESERVER")
+                        .unwrap_or_else(|| "https://matrix.org".into()),
+                    access_token,
+                    room_id,
+                }),
+                _ => None,
+            },
+            web: WebConfig {
+                listen: v.var("WEB_LISTEN"),
+                static_dir: v
+                    .var("STATIC_DIR")
+                    .unwrap_or_else(|| "src/static".into())
+                    .into(),
+                admin_api_key: v.var("ADMIN_API_KEY"),
+                cors_allow_origins: v
+                    .raw_var("CORS_ALLOW_ORIGINS", "")
+                    .split(',')
+                    .map(str::trim)
+                    .filter(|o| !o.is_empty())
+                    .map(String::from)
+                    .collect(),
+                umami_api_url: v
+                    .var("UMAMI_API_URL")
+                    .unwrap_or_else(|| "https://stats.buildfor.life/api/send".into()),
+                umami_website_id: v
+                    .raw_var("UMAMI_WEBSITE_ID", "00b2be73-8f5f-4400-9029-3be852eb08f7"),
+                umami_track_api: v.flag("UMAMI_TRACK_API", true),
+                hii_cache_ttl_s: v.number("HII_CACHE_TTL_SECONDS", 120)?,
+                latest_cache_ttl_s: v.number("LATEST_CACHE_TTL_SECONDS", 45)?,
+                health_cache_ttl_s: v.number("HEALTH_CACHE_TTL_SECONDS", 30)?,
+                forecast_max_age_s: v.number("FORECAST_MAX_AGE_SECONDS", 7200)?,
+                thaiwater_api_key: v.var("THAIWATER_API_KEY"),
+                request_timeout_s: v.number("REQUEST_TIMEOUT", 30)?,
+            },
+        };
+        if cfg.poll_offset_minutes >= 60 {
+            bail!("RID_POLL_OFFSET_MINUTES must be between 0 and 59");
+        }
+        if cfg.retry_minutes == 0 {
+            bail!("RID_RETRY_MINUTES must be positive");
+        }
+        Ok(cfg)
+    }
+
+    /// The URL with the password masked, for logs.
+    pub fn redacted_database_url(&self) -> String {
+        match (self.database_url.find("://"), self.database_url.rfind('@')) {
+            (Some(s), Some(at)) if at > s => {
+                let creds = &self.database_url[s + 3..at];
+                let user = creds.split(':').next().unwrap_or("");
+                format!(
+                    "{}{user}:***{}",
+                    &self.database_url[..s + 3],
+                    &self.database_url[at..]
+                )
+            }
+            _ => self.database_url.clone(),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn sqlalchemy_urls_are_normalized() {
+        assert_eq!(
+            normalize_database_url("postgresql+psycopg2://u:p@h:5432/db"),
+            "postgres://u:p@h:5432/db"
+        );
+        assert_eq!(
+            normalize_database_url("postgresql://u@h/db"),
+            "postgres://u@h/db"
+        );
+    }
+
+    #[test]
+    fn passwords_are_percent_encoded() {
+        assert_eq!(encode_userinfo("p@ss:w/rd"), "p%40ss%3Aw%2Frd");
+    }
+}

rust/collector/src/dams.rs

@@ -0,0 +1,143 @@
+//! RID large-dam daily status (app.rid.go.th/reservoir): storage, inflow,
+//! outflow for every large dam. Port of the hourly path of
+//! src/rid_reservoir.py (today + yesterday's late revisions). The one-off
+//! archive backfills stay in Python (scripts/backfill_rid_reservoir.py).
+
+use crate::util::{lenient_f64, scalar_string};
+use anyhow::{Context, Result};
+use chrono::NaiveDate;
+use serde_json::Value;
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct DamRecord {
+    pub dam_id: String,
+    pub region: Option<String>,
+    pub name_th: Option<String>,
+    pub latitude: Option<f64>,
+    pub longitude: Option<f64>,
+    pub capacity_max_mcm: Option<f64>,
+    pub capacity_normal_mcm: Option<f64>,
+    pub date: NaiveDate,
+    pub storage_mcm: Option<f64>,
+    pub storage_pct: Option<f64>,
+    pub inflow_mcm: Option<f64>,
+    pub outflow_mcm: Option<f64>,
+    pub level_msl: Option<f64>,
+}
+
+/// Values at or beyond the column's NUMERIC capacity become NULL instead of
+/// failing the whole daily batch (the source publishes e.g. 87798 %).
+fn bounded(v: Option<f64>, limit: f64) -> Option<f64> {
+    v.filter(|x| x.abs() < limit)
+}
+
+fn string(v: Option<&Value>) -> Option<String> {
+    v.and_then(Value::as_str)
+        .filter(|s| !s.is_empty())
+        .map(str::to_string)
+}
+
+pub fn parse_dams(payload: &Value) -> Vec<DamRecord> {
+    let mut out = Vec::new();
+    for region in payload
+        .get("regions")
+        .and_then(Value::as_array)
+        .into_iter()
+        .flatten()
+    {
+        for dam in region
+            .get("dams")
+            .and_then(Value::as_array)
+            .into_iter()
+            .flatten()
+        {
+            let Some(date) = dam
+                .get("DMD_Date")
+                .and_then(Value::as_str)
+                .and_then(|s| NaiveDate::parse_from_str(s, "%Y-%m-%d").ok())
+            else {
+                continue;
+            };
+            let Some(dam_id) = scalar_string(dam.get("DAM_ID")) else {
+                continue;
+            };
+            out.push(DamRecord {
+                dam_id,
+                region: string(region.get("region_name")),
+                name_th: string(dam.get("DAM_Name")),
+                latitude: lenient_f64(dam.get("DAM_Lat")),
+                longitude: lenient_f64(dam.get("DAM_Lon")),
+                capacity_max_mcm: lenient_f64(dam.get("DAM_QMax")),
+                capacity_normal_mcm: lenient_f64(dam.get("DAM_QStore")),
+                date,
+                storage_mcm: bounded(lenient_f64(dam.get("DMD_QUse")), 1e8),
+                storage_pct: bounded(lenient_f64(dam.get("PERCENT_DMD_QUse")), 1e6),
+                inflow_mcm: bounded(lenient_f64(dam.get("DMD_Inflow")), 1e8),
+                outflow_mcm: bounded(lenient_f64(dam.get("DMD_Outflow")), 1e8),
+                level_msl: bounded(lenient_f64(dam.get("DMD_Q")), 1e6),
+            });
+        }
+    }
+    out
+}
+
+pub struct DamsClient {
+    http: reqwest::Client,
+    url: String,
+}
+
+impl DamsClient {
+    pub fn new(http: reqwest::Client, url: String) -> Self {
+        Self { http, url }
+    }
+
+    /// Every large dam on `date` (None = today). POST only; the body may be
+    /// empty but must be sent.
+    pub async fn fetch_day(&self, date: Option<NaiveDate>) -> Result<Vec<DamRecord>> {
+        let date = date
+            .map(|d| d.format("%Y-%m-%d").to_string())
+            .unwrap_or_default();
+        let payload: Value = self
+            .http
+            .post(&self.url)
+            .header("X-Requested-With", "XMLHttpRequest")
+            .form(&[("date", date.as_str())])
+            .timeout(std::time::Duration::from_secs(120))
+            .send()
+            .await
+            .context("RID dams request failed")?
+            .error_for_status()?
+            .json()
+            .await
+            .context("RID dams response is not JSON")?;
+        Ok(parse_dams(&payload))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use serde_json::json;
+
+    #[test]
+    fn parses_regions_and_cleans_numbers() {
+        let payload = json!({"regions": [{"region_name": "ภาคเหนือ", "dams": [
+            {"DAM_ID": "200103", "DAM_Name": "แม่งัด", "DMD_Date": "2026-10-17", "DAM_Lat": "19.17",
+             "DAM_QMax": "265.00", "DMD_QUse": "1,234.5", "PERCENT_DMD_QUse": "87798000",
+             "DMD_Inflow": " - ", "DMD_Outflow": 1.2, "DMD_Q": " - "},
+            {"DAM_ID": "", "DMD_Date": "2026-10-17"},
+            {"DAM_ID": 100602, "DMD_Date": "not a date"}
+        ]}]});
+        let r = parse_dams(&payload);
+        assert_eq!(r.len(), 1);
+        let d = &r[0];
+        assert_eq!(d.dam_id, "200103");
+        assert_eq!(d.region.as_deref(), Some("ภาคเหนือ"));
+        assert_eq!(d.storage_mcm, Some(1234.5));
+        assert_eq!(d.storage_pct, None); // junk beyond NUMERIC(8,2)
+        assert_eq!(d.inflow_mcm, None);
+        assert_eq!(d.outflow_mcm, Some(1.2));
+        assert_eq!(d.level_msl, None);
+        assert_eq!(d.latitude, Some(19.17));
+    }
+}

rust/collector/src/db.rs

@@ -0,0 +1,783 @@
+//! PostgreSQL access: the same tables and upsert semantics as the Python
+//! collectors (src/database_adapters.py, hii_collector.py, rid_reservoir.py,
+//! ml/rain.py), written as one set-based statement per table (UNNEST) instead
+//! of a statement per row.
+
+use crate::dams::DamRecord;
+use crate::hii::{RainRecord, WaterLevelRecord};
+use crate::rain::RainRow;
+use crate::rid::Reading;
+use crate::stations::Stations;
+use anyhow::{Context, Result};
+use chrono::{NaiveDate, NaiveDateTime, NaiveTime};
+use sqlx::PgPool;
+use sqlx::postgres::PgPoolOptions;
+use std::collections::{BTreeMap, HashMap, HashSet};
+
+pub async fn connect(url: &str, max_connections: u32) -> Result<PgPool> {
+    use sqlx::ConnectOptions;
+    use std::str::FromStr;
+    // sqlx warns about any statement over 1 s; the whole-history gap scan
+    // legitimately takes several seconds on ~1M rows.
+    let options = sqlx::postgres::PgConnectOptions::from_str(url)
+        .context("invalid database URL")?
+        .log_slow_statements(log::LevelFilter::Warn, std::time::Duration::from_secs(30));
+    PgPoolOptions::new()
+        .max_connections(max_connections)
+        .acquire_timeout(std::time::Duration::from_secs(30))
+        .connect_with(options)
+        .await
+        .context("connecting to PostgreSQL")
+}
+
+/// Idempotent DDL, identical to what the Python app created and the Python
+/// backfill stores (src/hii_collector.py, src/rid_reservoir.py) still create,
+/// so either can start first against an empty database.
+const SCHEMA: &[&str] = &[
+    "CREATE TABLE IF NOT EXISTS stations (
+        id SERIAL PRIMARY KEY,
+        station_code VARCHAR(10) UNIQUE NOT NULL,
+        thai_name VARCHAR(255) NOT NULL,
+        english_name VARCHAR(255) NOT NULL,
+        latitude DECIMAL(10,8),
+        longitude DECIMAL(11,8),
+        geohash VARCHAR(20),
+        created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+        updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP)",
+    "CREATE TABLE IF NOT EXISTS water_measurements (
+        id BIGSERIAL PRIMARY KEY,
+        timestamp TIMESTAMP NOT NULL,
+        station_id INTEGER NOT NULL,
+        water_level NUMERIC(10,3),
+        discharge NUMERIC(10,2),
+        discharge_percent NUMERIC(5,2),
+        status VARCHAR(20) DEFAULT 'active',
+        created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+        FOREIGN KEY (station_id) REFERENCES stations(id),
+        UNIQUE(timestamp, station_id))",
+    "CREATE INDEX IF NOT EXISTS idx_timestamp ON water_measurements(timestamp)",
+    "CREATE INDEX IF NOT EXISTS idx_station_timestamp ON water_measurements(station_id, timestamp DESC)",
+    "CREATE TABLE IF NOT EXISTS hii_rain_stations (
+        id INTEGER PRIMARY KEY,
+        oldcode VARCHAR(60),
+        name_th VARCHAR(255),
+        name_en VARCHAR(255),
+        latitude NUMERIC(10,6),
+        longitude NUMERIC(10,6),
+        sub_basin_id VARCHAR(10),
+        agency VARCHAR(40),
+        updated_at TIMESTAMP)",
+    "CREATE TABLE IF NOT EXISTS hii_rainfall (
+        station_id INTEGER NOT NULL,
+        timestamp TIMESTAMP NOT NULL,
+        rain_1h NUMERIC(7,2),
+        rain_24h NUMERIC(8,2),
+        created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+        PRIMARY KEY (station_id, timestamp))",
+    "CREATE TABLE IF NOT EXISTS hii_wl_stations (
+        id INTEGER PRIMARY KEY,
+        oldcode VARCHAR(60),
+        rid_code VARCHAR(10),
+        name_th VARCHAR(255),
+        name_en VARCHAR(255),
+        latitude NUMERIC(10,6),
+        longitude NUMERIC(10,6),
+        agency VARCHAR(40),
+        river_name VARCHAR(255),
+        offset_msl NUMERIC(8,3),
+        ground_level_msl NUMERIC(8,3),
+        min_bank_msl NUMERIC(8,3),
+        critical_level_msl NUMERIC(8,3),
+        critical_level_m NUMERIC(8,3),
+        qmax NUMERIC(10,2),
+        is_key_station BOOLEAN,
+        updated_at TIMESTAMP)",
+    "CREATE TABLE IF NOT EXISTS hii_waterlevel (
+        station_id INTEGER NOT NULL,
+        timestamp TIMESTAMP NOT NULL,
+        wl_msl NUMERIC(8,3),
+        wl_m NUMERIC(8,3),
+        discharge NUMERIC(10,2),
+        flow_rate NUMERIC(10,2),
+        storage_percent NUMERIC(6,2),
+        situation_level INTEGER,
+        diff_wl_bank NUMERIC(8,3),
+        created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+        PRIMARY KEY (station_id, timestamp))",
+    "CREATE INDEX IF NOT EXISTS idx_hii_rainfall_ts ON hii_rainfall(timestamp)",
+    "CREATE INDEX IF NOT EXISTS idx_hii_waterlevel_ts ON hii_waterlevel(timestamp)",
+    "CREATE TABLE IF NOT EXISTS rid_dams (
+        dam_id VARCHAR(10) PRIMARY KEY,
+        region VARCHAR(40),
+        name_th VARCHAR(255),
+        latitude NUMERIC(10,6),
+        longitude NUMERIC(10,6),
+        capacity_max_mcm NUMERIC(10,2),
+        capacity_normal_mcm NUMERIC(10,2),
+        updated_at TIMESTAMP)",
+    "CREATE TABLE IF NOT EXISTS rid_reservoir_daily (
+        dam_id VARCHAR(10) NOT NULL,
+        date DATE NOT NULL,
+        storage_mcm NUMERIC(10,2),
+        storage_pct NUMERIC(8,2),
+        inflow_mcm NUMERIC(10,2),
+        outflow_mcm NUMERIC(10,2),
+        level_msl NUMERIC(8,2),
+        created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+        PRIMARY KEY (dam_id, date))",
+    "CREATE INDEX IF NOT EXISTS idx_rid_reservoir_date ON rid_reservoir_daily(date)",
+    "CREATE TABLE IF NOT EXISTS openmeteo_rain (
+        timestamp TIMESTAMP PRIMARY KEY,
+        chiang_dao NUMERIC(6,2), mae_taeng NUMERIC(6,2), mae_ngat NUMERIC(6,2),
+        mae_rim NUMERIC(6,2), chiang_mai NUMERIC(6,2),
+        catchment_mean NUMERIC(6,2),
+        created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP)",
+    // src/forecast_history.py
+    "CREATE TABLE IF NOT EXISTS forecast_history (
+        as_of TIMESTAMP NOT NULL,
+        station_code VARCHAR(10) NOT NULL,
+        horizon_hours INTEGER NOT NULL,
+        predicted_max_level NUMERIC(8,3),
+        p_warning NUMERIC(7,5),
+        p_danger NUMERIC(7,5),
+        current_level NUMERIC(8,3),
+        model_version VARCHAR(64),
+        source VARCHAR(16),
+        created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+        PRIMARY KEY (as_of, station_code, horizon_hours))",
+    "CREATE INDEX IF NOT EXISTS idx_forecast_history_station ON forecast_history(station_code, as_of)",
+    // notify.rs
+    "CREATE TABLE IF NOT EXISTS notification_state (
+        key VARCHAR(64) PRIMARY KEY, state VARCHAR(16) NOT NULL,
+        value NUMERIC(8,3), updated_at TIMESTAMP NOT NULL)",
+    // The current /forecast payload (one row, id 'all'), served by
+    // `ping-collector serve` when the website runs as its own process.
+    "CREATE TABLE IF NOT EXISTS forecast_current (
+        id VARCHAR(16) PRIMARY KEY,
+        computed_at TIMESTAMP NOT NULL,
+        model_versions TEXT,
+        payload TEXT NOT NULL)",
+];
+
+pub async fn ensure_schema(pool: &PgPool) -> Result<()> {
+    for ddl in SCHEMA {
+        sqlx::query(ddl)
+            .execute(pool)
+            .await
+            .with_context(|| format!("DDL failed: {ddl}"))?;
+    }
+    Ok(())
+}
+
+/// Keep the last record per key (a later report revision wins), preserving
+/// first-seen order; Postgres refuses to upsert the same key twice in one
+/// statement.
+fn dedupe_last<T, K: std::hash::Hash + Eq + Clone>(items: &[T], key: impl Fn(&T) -> K) -> Vec<&T> {
+    let mut index: HashMap<K, usize> = HashMap::new();
+    let mut out: Vec<&T> = Vec::new();
+    for item in items {
+        let k = key(item);
+        match index.get(&k) {
+            Some(&i) => out[i] = item,
+            None => {
+                index.insert(k, out.len());
+                out.push(item);
+            }
+        }
+    }
+    out
+}
+
+pub async fn save_rid(pool: &PgPool, readings: &[Reading], stations: &Stations) -> Result<usize> {
+    let rows = dedupe_last(readings, |r| (r.timestamp, r.station_num));
+    let station_nums: Vec<u32> = rows
+        .iter()
+        .map(|r| r.station_num)
+        .collect::<std::collections::BTreeSet<_>>()
+        .into_iter()
+        .collect();
+    let infos: Vec<_> = station_nums.iter().map(|&n| stations.get(n)).collect();
+
+    let mut tx = pool.begin().await?;
+    sqlx::query(
+        "INSERT INTO stations (id, station_code, thai_name, english_name, latitude, longitude, geohash, updated_at)
+         SELECT *, NOW() FROM UNNEST($1::int4[], $2::text[], $3::text[], $4::text[], $5::float8[], $6::float8[], $7::text[])
+         ON CONFLICT (id) DO UPDATE SET
+            thai_name = EXCLUDED.thai_name, english_name = EXCLUDED.english_name,
+            latitude = EXCLUDED.latitude, longitude = EXCLUDED.longitude,
+            geohash = EXCLUDED.geohash, updated_at = NOW()",
+    )
+    .bind(station_nums.iter().map(|&n| n as i32).collect::<Vec<_>>())
+    .bind(infos.iter().map(|s| s.code.clone()).collect::<Vec<_>>())
+    .bind(infos.iter().map(|s| s.thai_name.clone()).collect::<Vec<_>>())
+    .bind(infos.iter().map(|s| s.english_name.clone()).collect::<Vec<_>>())
+    .bind(infos.iter().map(|s| s.latitude).collect::<Vec<_>>())
+    .bind(infos.iter().map(|s| s.longitude).collect::<Vec<_>>())
+    .bind(infos.iter().map(|s| s.geohash.clone()).collect::<Vec<_>>())
+    .execute(&mut *tx)
+    .await
+    .context("upserting stations")?;
+
+    sqlx::query(
+        "INSERT INTO water_measurements (timestamp, station_id, water_level, discharge, discharge_percent, status)
+         SELECT *, 'active' FROM UNNEST($1::timestamp[], $2::int4[], $3::float8[], $4::float8[], $5::float8[])
+         ON CONFLICT (timestamp, station_id) DO UPDATE SET
+            water_level = EXCLUDED.water_level, discharge = EXCLUDED.discharge,
+            discharge_percent = EXCLUDED.discharge_percent, status = EXCLUDED.status",
+    )
+    .bind(rows.iter().map(|r| r.timestamp).collect::<Vec<_>>())
+    .bind(rows.iter().map(|r| r.station_num as i32).collect::<Vec<_>>())
+    .bind(rows.iter().map(|r| r.water_level).collect::<Vec<_>>())
+    .bind(rows.iter().map(|r| r.discharge).collect::<Vec<_>>())
+    .bind(rows.iter().map(|r| r.discharge_percent).collect::<Vec<_>>())
+    .execute(&mut *tx)
+    .await
+    .context("upserting water_measurements")?;
+    tx.commit().await?;
+    Ok(rows.len())
+}
+
+/// Day -> hours (0-23) that have at least one RID reading, for [start, end].
+pub async fn recorded_hours_by_day(
+    pool: &PgPool,
+    start: NaiveDate,
+    end: NaiveDate,
+) -> Result<HashMap<NaiveDate, HashSet<u32>>> {
+    let rows: Vec<(NaiveDate, i32)> = sqlx::query_as(
+        "SELECT CAST(timestamp AS DATE) AS day, CAST(EXTRACT(HOUR FROM timestamp) AS INTEGER) AS hour
+         FROM water_measurements WHERE timestamp >= $1 AND timestamp < $2
+         GROUP BY 1, 2",
+    )
+    .bind(start.and_time(NaiveTime::MIN))
+    .bind((end + chrono::Duration::days(1)).and_time(NaiveTime::MIN))
+    .fetch_all(pool)
+    .await?;
+    let mut out: HashMap<NaiveDate, HashSet<u32>> = HashMap::new();
+    for (day, hour) in rows {
+        out.entry(day).or_default().insert(hour as u32);
+    }
+    Ok(out)
+}
+
+pub async fn earliest_rid_timestamp(pool: &PgPool) -> Result<Option<NaiveDateTime>> {
+    Ok(
+        sqlx::query_scalar("SELECT MIN(timestamp) FROM water_measurements")
+            .fetch_one(pool)
+            .await?,
+    )
+}
+
+pub async fn latest_rid_timestamp(pool: &PgPool) -> Result<Option<NaiveDateTime>> {
+    Ok(
+        sqlx::query_scalar("SELECT MAX(timestamp) FROM water_measurements")
+            .fetch_one(pool)
+            .await?,
+    )
+}
+
+/// How many of `readings` are already stored (dry-run comparison).
+pub async fn count_stored_rid(pool: &PgPool, readings: &[Reading]) -> Result<i64> {
+    Ok(sqlx::query_scalar(
+        "SELECT COUNT(*) FROM water_measurements w
+         JOIN UNNEST($1::timestamp[], $2::int4[]) AS k(ts, sid) ON w.timestamp = k.ts AND w.station_id = k.sid",
+    )
+    .bind(readings.iter().map(|r| r.timestamp).collect::<Vec<_>>())
+    .bind(readings.iter().map(|r| r.station_num as i32).collect::<Vec<_>>())
+    .fetch_one(pool)
+    .await?)
+}
+
+pub async fn save_hii_rain(
+    pool: &PgPool,
+    records: &[RainRecord],
+    now: NaiveDateTime,
+) -> Result<usize> {
+    if records.is_empty() {
+        return Ok(0);
+    }
+    let st = dedupe_last(records, |r| r.station_id);
+    let ms = dedupe_last(records, |r| (r.station_id, r.timestamp));
+    let mut tx = pool.begin().await?;
+    sqlx::query(
+        "INSERT INTO hii_rain_stations (id, oldcode, name_th, name_en, latitude, longitude, sub_basin_id, agency, updated_at)
+         SELECT * FROM UNNEST($1::int4[], $2::text[], $3::text[], $4::text[], $5::float8[], $6::float8[], $7::text[], $8::text[], $9::timestamp[])
+         ON CONFLICT (id) DO UPDATE SET
+            oldcode = EXCLUDED.oldcode, name_th = EXCLUDED.name_th, name_en = EXCLUDED.name_en,
+            latitude = EXCLUDED.latitude, longitude = EXCLUDED.longitude,
+            sub_basin_id = EXCLUDED.sub_basin_id, agency = EXCLUDED.agency, updated_at = EXCLUDED.updated_at",
+    )
+    .bind(st.iter().map(|r| r.station_id as i32).collect::<Vec<_>>())
+    .bind(st.iter().map(|r| r.oldcode.clone()).collect::<Vec<_>>())
+    .bind(st.iter().map(|r| r.name_th.clone()).collect::<Vec<_>>())
+    .bind(st.iter().map(|r| r.name_en.clone()).collect::<Vec<_>>())
+    .bind(st.iter().map(|r| r.latitude).collect::<Vec<_>>())
+    .bind(st.iter().map(|r| r.longitude).collect::<Vec<_>>())
+    .bind(st.iter().map(|r| r.sub_basin_id.clone()).collect::<Vec<_>>())
+    .bind(st.iter().map(|r| r.agency.clone()).collect::<Vec<_>>())
+    .bind(vec![now; st.len()])
+    .execute(&mut *tx)
+    .await
+    .context("upserting hii_rain_stations")?;
+    sqlx::query(
+        "INSERT INTO hii_rainfall (station_id, timestamp, rain_1h, rain_24h)
+         SELECT * FROM UNNEST($1::int4[], $2::timestamp[], $3::float8[], $4::float8[])
+         ON CONFLICT (station_id, timestamp) DO UPDATE SET
+            rain_1h = EXCLUDED.rain_1h, rain_24h = EXCLUDED.rain_24h",
+    )
+    .bind(ms.iter().map(|r| r.station_id as i32).collect::<Vec<_>>())
+    .bind(ms.iter().map(|r| r.timestamp).collect::<Vec<_>>())
+    .bind(ms.iter().map(|r| r.rain_1h).collect::<Vec<_>>())
+    .bind(ms.iter().map(|r| r.rain_24h).collect::<Vec<_>>())
+    .execute(&mut *tx)
+    .await
+    .context("upserting hii_rainfall")?;
+    tx.commit().await?;
+    Ok(ms.len())
+}
+
+pub async fn save_hii_waterlevel(
+    pool: &PgPool,
+    records: &[WaterLevelRecord],
+    now: NaiveDateTime,
+) -> Result<usize> {
+    if records.is_empty() {
+        return Ok(0);
+    }
+    let st = dedupe_last(records, |r| r.station_id);
+    let ms = dedupe_last(records, |r| (r.station_id, r.timestamp));
+    let mut tx = pool.begin().await?;
+    sqlx::query(
+        "INSERT INTO hii_wl_stations (id, oldcode, rid_code, name_th, name_en, latitude, longitude, agency,
+            river_name, offset_msl, ground_level_msl, min_bank_msl, critical_level_msl, critical_level_m,
+            qmax, is_key_station, updated_at)
+         SELECT * FROM UNNEST($1::int4[], $2::text[], $3::text[], $4::text[], $5::text[], $6::float8[],
+            $7::float8[], $8::text[], $9::text[], $10::float8[], $11::float8[], $12::float8[], $13::float8[],
+            $14::float8[], $15::float8[], $16::bool[], $17::timestamp[])
+         ON CONFLICT (id) DO UPDATE SET
+            oldcode = EXCLUDED.oldcode, rid_code = EXCLUDED.rid_code, name_th = EXCLUDED.name_th,
+            name_en = EXCLUDED.name_en, latitude = EXCLUDED.latitude, longitude = EXCLUDED.longitude,
+            agency = EXCLUDED.agency, river_name = EXCLUDED.river_name, offset_msl = EXCLUDED.offset_msl,
+            ground_level_msl = EXCLUDED.ground_level_msl, min_bank_msl = EXCLUDED.min_bank_msl,
+            critical_level_msl = EXCLUDED.critical_level_msl, critical_level_m = EXCLUDED.critical_level_m,
+            qmax = EXCLUDED.qmax, is_key_station = EXCLUDED.is_key_station, updated_at = EXCLUDED.updated_at",
+    )
+    .bind(st.iter().map(|r| r.station_id as i32).collect::<Vec<_>>())
+    .bind(st.iter().map(|r| r.oldcode.clone()).collect::<Vec<_>>())
+    .bind(st.iter().map(|r| r.rid_code.clone()).collect::<Vec<_>>())
+    .bind(st.iter().map(|r| r.name_th.clone()).collect::<Vec<_>>())
+    .bind(st.iter().map(|r| r.name_en.clone()).collect::<Vec<_>>())
+    .bind(st.iter().map(|r| r.latitude).collect::<Vec<_>>())
+    .bind(st.iter().map(|r| r.longitude).collect::<Vec<_>>())
+    .bind(st.iter().map(|r| r.agency.clone()).collect::<Vec<_>>())
+    .bind(st.iter().map(|r| r.river_name.clone()).collect::<Vec<_>>())
+    .bind(st.iter().map(|r| r.offset_msl).collect::<Vec<_>>())
+    .bind(st.iter().map(|r| r.ground_level_msl).collect::<Vec<_>>())
+    .bind(st.iter().map(|r| r.min_bank_msl).collect::<Vec<_>>())
+    .bind(st.iter().map(|r| r.critical_level_msl).collect::<Vec<_>>())
+    .bind(st.iter().map(|r| r.critical_level_m).collect::<Vec<_>>())
+    .bind(st.iter().map(|r| r.qmax).collect::<Vec<_>>())
+    .bind(st.iter().map(|r| r.is_key_station).collect::<Vec<_>>())
+    .bind(vec![now; st.len()])
+    .execute(&mut *tx)
+    .await
+    .context("upserting hii_wl_stations")?;
+    sqlx::query(
+        "INSERT INTO hii_waterlevel (station_id, timestamp, wl_msl, wl_m, discharge, flow_rate,
+            storage_percent, situation_level, diff_wl_bank)
+         SELECT * FROM UNNEST($1::int4[], $2::timestamp[], $3::float8[], $4::float8[], $5::float8[],
+            $6::float8[], $7::float8[], $8::int4[], $9::float8[])
+         ON CONFLICT (station_id, timestamp) DO UPDATE SET
+            wl_msl = EXCLUDED.wl_msl, wl_m = EXCLUDED.wl_m, discharge = EXCLUDED.discharge,
+            flow_rate = EXCLUDED.flow_rate, storage_percent = EXCLUDED.storage_percent,
+            situation_level = EXCLUDED.situation_level, diff_wl_bank = EXCLUDED.diff_wl_bank",
+    )
+    .bind(ms.iter().map(|r| r.station_id as i32).collect::<Vec<_>>())
+    .bind(ms.iter().map(|r| r.timestamp).collect::<Vec<_>>())
+    .bind(ms.iter().map(|r| r.wl_msl).collect::<Vec<_>>())
+    .bind(ms.iter().map(|r| r.wl_m).collect::<Vec<_>>())
+    .bind(ms.iter().map(|r| r.discharge).collect::<Vec<_>>())
+    .bind(ms.iter().map(|r| r.flow_rate).collect::<Vec<_>>())
+    .bind(ms.iter().map(|r| r.storage_percent).collect::<Vec<_>>())
+    .bind(ms.iter().map(|r| r.situation_level).collect::<Vec<_>>())
+    .bind(ms.iter().map(|r| r.diff_wl_bank).collect::<Vec<_>>())
+    .execute(&mut *tx)
+    .await
+    .context("upserting hii_waterlevel")?;
+    tx.commit().await?;
+    Ok(ms.len())
+}
+
+/// rid_dams metadata never blanks a stored value (COALESCE), and the daily
+/// `level_msl` is preserved too: only the range backfill carries a level, the
+/// daily snapshot would NULL it on every cycle.
+pub async fn save_dams(pool: &PgPool, records: &[DamRecord], now: NaiveDateTime) -> Result<usize> {
+    if records.is_empty() {
+        return Ok(0);
+    }
+    let dams = dedupe_last(records, |r| r.dam_id.clone());
+    let ms = dedupe_last(records, |r| (r.dam_id.clone(), r.date));
+    let mut tx = pool.begin().await?;
+    sqlx::query(
+        "INSERT INTO rid_dams (dam_id, region, name_th, latitude, longitude, capacity_max_mcm,
+            capacity_normal_mcm, updated_at)
+         SELECT * FROM UNNEST($1::text[], $2::text[], $3::text[], $4::float8[], $5::float8[],
+            $6::float8[], $7::float8[], $8::timestamp[])
+         ON CONFLICT (dam_id) DO UPDATE SET
+            region = COALESCE(EXCLUDED.region, rid_dams.region),
+            name_th = COALESCE(EXCLUDED.name_th, rid_dams.name_th),
+            latitude = COALESCE(EXCLUDED.latitude, rid_dams.latitude),
+            longitude = COALESCE(EXCLUDED.longitude, rid_dams.longitude),
+            capacity_max_mcm = COALESCE(EXCLUDED.capacity_max_mcm, rid_dams.capacity_max_mcm),
+            capacity_normal_mcm = COALESCE(EXCLUDED.capacity_normal_mcm, rid_dams.capacity_normal_mcm),
+            updated_at = EXCLUDED.updated_at",
+    )
+    .bind(dams.iter().map(|r| r.dam_id.clone()).collect::<Vec<_>>())
+    .bind(dams.iter().map(|r| r.region.clone()).collect::<Vec<_>>())
+    .bind(dams.iter().map(|r| r.name_th.clone()).collect::<Vec<_>>())
+    .bind(dams.iter().map(|r| r.latitude).collect::<Vec<_>>())
+    .bind(dams.iter().map(|r| r.longitude).collect::<Vec<_>>())
+    .bind(dams.iter().map(|r| r.capacity_max_mcm).collect::<Vec<_>>())
+    .bind(dams.iter().map(|r| r.capacity_normal_mcm).collect::<Vec<_>>())
+    .bind(vec![now; dams.len()])
+    .execute(&mut *tx)
+    .await
+    .context("upserting rid_dams")?;
+    sqlx::query(
+        "INSERT INTO rid_reservoir_daily (dam_id, date, storage_mcm, storage_pct, inflow_mcm, outflow_mcm, level_msl)
+         SELECT * FROM UNNEST($1::text[], $2::date[], $3::float8[], $4::float8[], $5::float8[], $6::float8[], $7::float8[])
+         ON CONFLICT (dam_id, date) DO UPDATE SET
+            storage_mcm = EXCLUDED.storage_mcm, storage_pct = EXCLUDED.storage_pct,
+            inflow_mcm = EXCLUDED.inflow_mcm, outflow_mcm = EXCLUDED.outflow_mcm,
+            level_msl = COALESCE(EXCLUDED.level_msl, rid_reservoir_daily.level_msl)",
+    )
+    .bind(ms.iter().map(|r| r.dam_id.clone()).collect::<Vec<_>>())
+    .bind(ms.iter().map(|r| r.date).collect::<Vec<_>>())
+    .bind(ms.iter().map(|r| r.storage_mcm).collect::<Vec<_>>())
+    .bind(ms.iter().map(|r| r.storage_pct).collect::<Vec<_>>())
+    .bind(ms.iter().map(|r| r.inflow_mcm).collect::<Vec<_>>())
+    .bind(ms.iter().map(|r| r.outflow_mcm).collect::<Vec<_>>())
+    .bind(ms.iter().map(|r| r.level_msl).collect::<Vec<_>>())
+    .execute(&mut *tx)
+    .await
+    .context("upserting rid_reservoir_daily")?;
+    tx.commit().await?;
+    Ok(ms.len())
+}
+
+pub async fn save_rain(pool: &PgPool, rows: &[RainRow]) -> Result<usize> {
+    if rows.is_empty() {
+        return Ok(0);
+    }
+    let rows = dedupe_last(rows, |r| r.timestamp);
+    let col = |i: usize| rows.iter().map(|r| r.points[i]).collect::<Vec<_>>();
+    sqlx::query(
+        "INSERT INTO openmeteo_rain (timestamp, chiang_dao, mae_taeng, mae_ngat, mae_rim, chiang_mai, catchment_mean)
+         SELECT * FROM UNNEST($1::timestamp[], $2::float8[], $3::float8[], $4::float8[], $5::float8[], $6::float8[], $7::float8[])
+         ON CONFLICT (timestamp) DO UPDATE SET
+            chiang_dao = EXCLUDED.chiang_dao, mae_taeng = EXCLUDED.mae_taeng, mae_ngat = EXCLUDED.mae_ngat,
+            mae_rim = EXCLUDED.mae_rim, chiang_mai = EXCLUDED.chiang_mai, catchment_mean = EXCLUDED.catchment_mean",
+    )
+    .bind(rows.iter().map(|r| r.timestamp).collect::<Vec<_>>())
+    .bind(col(0))
+    .bind(col(1))
+    .bind(col(2))
+    .bind(col(3))
+    .bind(col(4))
+    .bind(rows.iter().map(|r| r.catchment_mean).collect::<Vec<_>>())
+    .execute(pool)
+    .await
+    .context("upserting openmeteo_rain")?;
+    Ok(rows.len())
+}
+
+/// Row counts per table, for `check-db`.
+pub async fn table_counts(pool: &PgPool) -> Result<BTreeMap<&'static str, i64>> {
+    let mut out = BTreeMap::new();
+    for table in [
+        "water_measurements",
+        "hii_rainfall",
+        "hii_waterlevel",
+        "rid_reservoir_daily",
+        "openmeteo_rain",
+    ] {
+        let n: i64 = sqlx::query_scalar(&format!("SELECT COUNT(*) FROM {table}"))
+            .fetch_one(pool)
+            .await?;
+        out.insert(table, n);
+    }
+    Ok(out)
+}
+
+// ---------------------------------------------------------------------------
+// Forecast inputs and outputs (FORECAST_IN_COLLECTOR)
+//
+// NUMERIC columns are read as text and parsed: Python's `float(Decimal)` is a
+// correctly rounded parse of the same digits, so both sides get the same f64.
+// Forecast values are written as text (shortest round-trip digits, as Python's
+// repr) so PostgreSQL rounds the same decimal as it does for psycopg2.
+// ---------------------------------------------------------------------------
+
+/// A row of (key, key, NUMERIC as text, NUMERIC as text).
+type TextRow<A, B> = (A, B, Option<String>, Option<String>);
+/// (date, storage_pct, inflow_mcm, outflow_mcm) with NUMERIC as text.
+type DamRow = (NaiveDate, Option<String>, Option<String>, Option<String>);
+
+fn parse_num(v: Option<String>) -> f64 {
+    v.and_then(|s| s.parse().ok()).unwrap_or(f64::NAN)
+}
+
+fn num_text(v: f64) -> Option<String> {
+    v.is_finite().then(|| format!("{v}"))
+}
+
+/// RID readings in [start, end], as `data._fetch_from_db`.
+pub async fn load_rid_window(
+    pool: &PgPool,
+    start: NaiveDateTime,
+    end: NaiveDateTime,
+) -> Result<Vec<ping_forecast::Row>> {
+    let rows: Vec<TextRow<NaiveDateTime, String>> = sqlx::query_as(
+        "SELECT m.timestamp, s.station_code, m.water_level::text, m.discharge::text
+         FROM water_measurements m JOIN stations s ON m.station_id = s.id
+         WHERE m.timestamp >= $1 AND m.timestamp <= $2 ORDER BY m.timestamp",
+    )
+    .bind(start)
+    .bind(end)
+    .fetch_all(pool)
+    .await
+    .context("loading the RID window")?;
+    Ok(rows
+        .into_iter()
+        .map(|(t, code, level, q)| ping_forecast::Row {
+            timestamp: ping_forecast::grid::floor_hour(t),
+            station_code: code,
+            water_level: parse_num(level),
+            discharge: parse_num(q),
+        })
+        .collect())
+}
+
+/// HII mirror readings for `codes` in [start, end], as `data._fetch_hii_levels`.
+pub async fn load_hii_window(
+    pool: &PgPool,
+    codes: &[String],
+    start: NaiveDateTime,
+    end: NaiveDateTime,
+) -> Result<Vec<ping_forecast::hii::HiiRow>> {
+    let rows: Vec<TextRow<NaiveDateTime, String>> = sqlx::query_as(
+        "SELECT m.timestamp, s.rid_code, m.wl_msl::text, m.discharge::text
+         FROM hii_waterlevel m JOIN hii_wl_stations s ON s.id = m.station_id
+         WHERE s.rid_code IS NOT NULL AND m.timestamp >= $1 AND m.timestamp <= $2
+           AND s.rid_code = ANY($3)
+         ORDER BY m.timestamp, m.station_id",
+    )
+    .bind(start)
+    .bind(end)
+    .bind(codes)
+    .fetch_all(pool)
+    .await
+    .context("loading HII mirror levels")?;
+    Ok(rows
+        .into_iter()
+        .map(|(t, code, wl, q)| ping_forecast::hii::HiiRow {
+            timestamp: t,
+            station_code: code,
+            wl_msl: parse_num(wl),
+            discharge: parse_num(q),
+        })
+        .collect())
+}
+
+/// Newest HII mirror reading for `codes` (a forecast trigger).
+pub async fn latest_hii_timestamp(
+    pool: &PgPool,
+    codes: &[String],
+) -> Result<Option<NaiveDateTime>> {
+    Ok(sqlx::query_scalar(
+        "SELECT MAX(m.timestamp) FROM hii_waterlevel m JOIN hii_wl_stations s ON s.id = m.station_id
+         WHERE s.rid_code = ANY($1) AND m.wl_msl IS NOT NULL",
+    )
+    .bind(codes)
+    .fetch_one(pool)
+    .await?)
+}
+
+/// Daily reservoir rows from `start`, as `dam.load_daily`.
+pub async fn load_dam_days(
+    pool: &PgPool,
+    dam_id: &str,
+    start: NaiveDate,
+) -> Result<Vec<ping_forecast::dam::DamDay>> {
+    let rows: Vec<DamRow> = sqlx::query_as(
+        "SELECT date, storage_pct::text, inflow_mcm::text, outflow_mcm::text
+         FROM rid_reservoir_daily WHERE dam_id = $1 AND date >= $2 ORDER BY date",
+    )
+    .bind(dam_id)
+    .bind(start)
+    .fetch_all(pool)
+    .await
+    .context("loading the dam series")?;
+    Ok(rows
+        .into_iter()
+        .map(|(date, s, i, o)| ping_forecast::dam::DamDay {
+            date,
+            storage_pct: parse_num(s),
+            inflow_mcm: parse_num(i),
+            outflow_mcm: parse_num(o),
+        })
+        .collect())
+}
+
+/// The newest RID row per station (`get_latest_measurements(200)`).
+#[derive(Clone, Debug, PartialEq)]
+pub struct LatestReading {
+    pub timestamp: NaiveDateTime,
+    pub station_code: String,
+    pub water_level: Option<f64>,
+    pub discharge_percent: Option<f64>,
+}
+
+pub async fn latest_rid_readings(pool: &PgPool) -> Result<Vec<LatestReading>> {
+    let rows: Vec<TextRow<NaiveDateTime, String>> = sqlx::query_as(
+        "SELECT m.timestamp, s.station_code, m.water_level::text, m.discharge_percent::text
+         FROM water_measurements m
+         JOIN stations s ON m.station_id = s.id
+         INNER JOIN (
+             SELECT station_id, MAX(timestamp) AS max_timestamp
+             FROM water_measurements GROUP BY station_id
+         ) latest ON m.station_id = latest.station_id AND m.timestamp = latest.max_timestamp
+         ORDER BY s.station_code
+         LIMIT 200",
+    )
+    .fetch_all(pool)
+    .await
+    .context("loading the latest readings")?;
+    let num = |v: Option<String>| v.and_then(|s| s.parse().ok());
+    Ok(rows
+        .into_iter()
+        .map(|(t, code, level, pct)| LatestReading {
+            timestamp: t,
+            station_code: code,
+            water_level: num(level),
+            discharge_percent: num(pct),
+        })
+        .collect())
+}
+
+/// `ForecastHistoryStore.save_rows`: one row per (as_of, station, horizon).
+pub async fn save_forecast_history(
+    pool: &PgPool,
+    rows: &[ping_forecast::Forecast],
+) -> Result<usize> {
+    let rows = dedupe_last(rows, |r| (r.as_of, r.station_code.clone(), r.horizon_hours));
+    if rows.is_empty() {
+        return Ok(0);
+    }
+    sqlx::query(
+        "INSERT INTO forecast_history (as_of, station_code, horizon_hours, predicted_max_level,
+            p_warning, p_danger, current_level, model_version, source)
+         SELECT a, c, h, pm::numeric, pw::numeric, pd::numeric, cl::numeric, mv, src
+         FROM UNNEST($1::timestamp[], $2::text[], $3::int4[], $4::text[], $5::text[], $6::text[],
+                     $7::text[], $8::text[], $9::text[])
+              AS t(a, c, h, pm, pw, pd, cl, mv, src)
+         ON CONFLICT (as_of, station_code, horizon_hours) DO UPDATE SET
+            predicted_max_level = EXCLUDED.predicted_max_level, p_warning = EXCLUDED.p_warning,
+            p_danger = EXCLUDED.p_danger, current_level = EXCLUDED.current_level,
+            model_version = EXCLUDED.model_version, source = EXCLUDED.source",
+    )
+    .bind(rows.iter().map(|r| r.as_of).collect::<Vec<_>>())
+    .bind(
+        rows.iter()
+            .map(|r| r.station_code.clone())
+            .collect::<Vec<_>>(),
+    )
+    .bind(
+        rows.iter()
+            .map(|r| r.horizon_hours as i32)
+            .collect::<Vec<_>>(),
+    )
+    .bind(
+        rows.iter()
+            .map(|r| num_text(r.predicted_max_level))
+            .collect::<Vec<_>>(),
+    )
+    .bind(
+        rows.iter()
+            .map(|r| num_text(r.p_warning))
+            .collect::<Vec<_>>(),
+    )
+    .bind(
+        rows.iter()
+            .map(|r| num_text(r.p_danger))
+            .collect::<Vec<_>>(),
+    )
+    .bind(
+        rows.iter()
+            .map(|r| num_text(r.current_level))
+            .collect::<Vec<_>>(),
+    )
+    .bind(
+        rows.iter()
+            .map(|r| r.model_version.clone())
+            .collect::<Vec<_>>(),
+    )
+    .bind(rows.iter().map(|r| r.source.clone()).collect::<Vec<_>>())
+    .execute(pool)
+    .await
+    .context("upserting forecast_history")?;
+    Ok(rows.len())
+}
+
+/// The payload /forecast serves (see `forecast_current` above).
+pub async fn save_forecast_current(
+    pool: &PgPool,
+    computed_at: NaiveDateTime,
+    rows: &[ping_forecast::Forecast],
+) -> Result<()> {
+    let versions: std::collections::BTreeSet<&str> =
+        rows.iter().map(|r| r.model_version.as_str()).collect();
+    sqlx::query(
+        "INSERT INTO forecast_current (id, computed_at, model_versions, payload)
+         VALUES ('all', $1, $2, $3)
+         ON CONFLICT (id) DO UPDATE SET computed_at = EXCLUDED.computed_at,
+            model_versions = EXCLUDED.model_versions, payload = EXCLUDED.payload",
+    )
+    .bind(computed_at)
+    .bind(versions.into_iter().collect::<Vec<_>>().join(","))
+    .bind(serde_json::to_string(rows)?)
+    .execute(pool)
+    .await
+    .context("writing forecast_current")?;
+    Ok(())
+}
+
+/// notification_state as a key -> state map.
+pub async fn notification_states(pool: &PgPool) -> Result<HashMap<String, String>> {
+    let rows: Vec<(String, String)> = sqlx::query_as("SELECT key, state FROM notification_state")
+        .fetch_all(pool)
+        .await
+        .context("reading notification_state")?;
+    Ok(rows.into_iter().collect())
+}
+
+pub async fn set_notification_state(
+    pool: &PgPool,
+    key: &str,
+    state: &str,
+    value: Option<f64>,
+    now: NaiveDateTime,
+) -> Result<()> {
+    sqlx::query(
+        "INSERT INTO notification_state (key, state, value, updated_at)
+         VALUES ($1, $2, $3::numeric, $4) ON CONFLICT (key) DO UPDATE SET
+            state = EXCLUDED.state, value = EXCLUDED.value, updated_at = EXCLUDED.updated_at",
+    )
+    .bind(key)
+    .bind(state)
+    .bind(value.and_then(num_text))
+    .bind(now)
+    .execute(pool)
+    .await
+    .with_context(|| format!("saving notification state {key}"))?;
+    Ok(())
+}

rust/collector/src/db_tests.rs

@@ -0,0 +1,521 @@
+//! Database tests against a real PostgreSQL. Each test runs in its own
+//! throwaway schema. Set TEST_DATABASE_URL to run them, e.g.
+//! `postgres://postgres:ping@localhost:55432/ping`; without it they are
+//! skipped (reported as passed with a note on stderr).
+
+use crate::dams::DamRecord;
+use crate::db;
+use crate::hii::{RainRecord, WaterLevelRecord};
+use crate::rain::RainRow;
+use crate::rid::{self, Reading, RidSource};
+use crate::stations::Stations;
+use anyhow::Result;
+use chrono::{Duration, NaiveDate, NaiveDateTime, NaiveTime};
+use sqlx::PgPool;
+use sqlx::postgres::{PgConnectOptions, PgPoolOptions};
+use std::collections::HashMap;
+use std::str::FromStr;
+use std::sync::Mutex;
+
+pub(crate) struct TestDb {
+    pub(crate) pool: PgPool,
+    admin: PgPool,
+    schema: String,
+}
+
+impl TestDb {
+    pub(crate) async fn new() -> Option<Self> {
+        let Ok(url) = std::env::var("TEST_DATABASE_URL") else {
+            eprintln!("TEST_DATABASE_URL not set: skipping database test");
+            return None;
+        };
+        // the counter: tests run in parallel and the clock is coarse on Windows
+        static N: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0);
+        let schema = format!(
+            "t_{}_{}_{}",
+            std::process::id(),
+            chrono::Utc::now().timestamp_nanos_opt().unwrap(),
+            N.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
+        );
+        let admin = PgPool::connect(&url).await.expect("connect");
+        sqlx::query(&format!("CREATE SCHEMA {schema}"))
+            .execute(&admin)
+            .await
+            .unwrap();
+        let opts = PgConnectOptions::from_str(&url)
+            .unwrap()
+            .options([("search_path", schema.as_str())]);
+        let pool = PgPoolOptions::new()
+            .max_connections(2)
+            .connect_with(opts)
+            .await
+            .unwrap();
+        db::ensure_schema(&pool).await.expect("schema");
+        Some(TestDb {
+            pool,
+            admin,
+            schema,
+        })
+    }
+
+    pub(crate) async fn drop(self) {
+        self.pool.close().await;
+        sqlx::query(&format!("DROP SCHEMA {} CASCADE", self.schema))
+            .execute(&self.admin)
+            .await
+            .unwrap();
+    }
+}
+
+fn d(y: i32, m: u32, day: u32) -> NaiveDate {
+    NaiveDate::from_ymd_opt(y, m, day).unwrap()
+}
+fn t(y: i32, m: u32, day: u32, h: u32, min: u32) -> NaiveDateTime {
+    d(y, m, day).and_hms_opt(h, min, 0).unwrap()
+}
+
+/// Canned RID: date -> last published hour (1..=24); P.1 (column 8) and P.20
+/// (column 1). Records which reports were requested.
+struct FakeRid {
+    published: HashMap<NaiveDate, i64>,
+    calls: Mutex<Vec<NaiveDate>>,
+}
+
+impl FakeRid {
+    fn new(published: &[(NaiveDate, i64)]) -> Self {
+        FakeRid {
+            published: published.iter().copied().collect(),
+            calls: Mutex::new(Vec::new()),
+        }
+    }
+    fn calls(&self) -> Vec<NaiveDate> {
+        self.calls.lock().unwrap().clone()
+    }
+}
+
+impl RidSource for FakeRid {
+    async fn fetch_report(&self, date: NaiveDate) -> Result<Vec<Reading>> {
+        self.calls.lock().unwrap().push(date);
+        let last = self.published.get(&date).copied().unwrap_or(0);
+        Ok((1..=last)
+            .flat_map(|h| {
+                let ts = date.and_time(NaiveTime::MIN) + Duration::hours(h);
+                [8u32, 1].map(|n| Reading {
+                    timestamp: ts,
+                    station_num: n,
+                    water_level: 1.0 + h as f64 / 100.0,
+                    discharge: Some(50.0),
+                    discharge_percent: Some(12.5),
+                })
+            })
+            .collect())
+    }
+}
+
+async fn stored_hours(pool: &PgPool, day: NaiveDate) -> Vec<u32> {
+    let rec = db::recorded_hours_by_day(pool, day, day).await.unwrap();
+    let mut h: Vec<u32> = rec
+        .get(&day)
+        .cloned()
+        .unwrap_or_default()
+        .into_iter()
+        .collect();
+    h.sort();
+    h
+}
+
+#[tokio::test]
+async fn late_report_is_stored_and_reported_behind() {
+    let Some(db) = TestDb::new().await else {
+        return;
+    };
+    let today = d(2026, 10, 17);
+    let yesterday = today - Duration::days(1);
+    let stations = Stations::bundled();
+    // yesterday complete already
+    let full = FakeRid::new(&[(yesterday, 24)]);
+    rid::fill_gaps(&full, &db.pool, &stations, Some(1), t(2026, 10, 17, 0, 30))
+        .await
+        .unwrap();
+
+    // 21:30 and RID has published up to 20:00
+    let src = FakeRid::new(&[(today, 20)]);
+    let o = rid::run_cycle(&src, &db.pool, &stations, t(2026, 10, 17, 21, 30)).await;
+    assert!(!o.current);
+    assert_eq!(o.latest, Some(t(2026, 10, 17, 20, 0)));
+    assert_eq!(o.saved, 40);
+    assert_eq!(src.calls(), vec![today]);
+    assert_eq!(
+        stored_hours(&db.pool, today).await,
+        (0..=20).collect::<Vec<_>>()
+    );
+
+    // the 21:00 reading lands; the next poll is current
+    let src = FakeRid::new(&[(today, 21)]);
+    let o = rid::run_cycle(&src, &db.pool, &stations, t(2026, 10, 17, 21, 40)).await;
+    assert!(o.current);
+    db.drop().await;
+}
+
+#[tokio::test]
+async fn yesterdays_late_hours_are_recovered_after_midnight() {
+    let Some(db) = TestDb::new().await else {
+        return;
+    };
+    let today = d(2026, 10, 17);
+    let yesterday = today - Duration::days(1);
+    let stations = Stations::bundled();
+    // stored up to 21:00 yesterday (the evening RID published late)
+    let partial = FakeRid::new(&[(yesterday, 21)]);
+    rid::run_cycle(&partial, &db.pool, &stations, t(2026, 10, 16, 21, 5)).await;
+    assert_eq!(
+        stored_hours(&db.pool, yesterday).await,
+        (1..=21).collect::<Vec<_>>()
+    );
+
+    // 02:10: RID now has all of yesterday and today up to 02:00
+    let src = FakeRid::new(&[(yesterday, 24), (today, 2)]);
+    let o = rid::run_cycle(&src, &db.pool, &stations, t(2026, 10, 17, 2, 10)).await;
+    assert!(o.current);
+    assert_eq!(src.calls(), vec![yesterday, today]);
+    assert_eq!(
+        stored_hours(&db.pool, yesterday).await,
+        (1..=23).collect::<Vec<_>>()
+    );
+    assert_eq!(stored_hours(&db.pool, today).await, vec![0, 1, 2]);
+
+    // complete now: the next cycle fetches only today's report
+    let src = FakeRid::new(&[(yesterday, 24), (today, 3)]);
+    rid::run_cycle(&src, &db.pool, &stations, t(2026, 10, 17, 3, 5)).await;
+    assert_eq!(src.calls(), vec![today]);
+    db.drop().await;
+}
+
+#[tokio::test]
+async fn rid_upsert_revises_values_and_rounds_like_numeric() {
+    let Some(db) = TestDb::new().await else {
+        return;
+    };
+    let stations = Stations::bundled();
+    let r = |level: f64| Reading {
+        timestamp: t(2026, 10, 17, 9, 0),
+        station_num: 8,
+        water_level: level,
+        discharge: None,
+        discharge_percent: None,
+    };
+    db::save_rid(&db.pool, &[r(1.0700000524520874)], &stations)
+        .await
+        .unwrap();
+    db::save_rid(&db.pool, &[r(1.0900000333786)], &stations)
+        .await
+        .unwrap();
+    let rows: Vec<(String, f64, Option<f64>, String)> = sqlx::query_as(
+        "SELECT s.station_code, w.water_level::float8, w.discharge::float8, w.status
+         FROM water_measurements w JOIN stations s ON s.id = w.station_id",
+    )
+    .fetch_all(&db.pool)
+    .await
+    .unwrap();
+    assert_eq!(rows, vec![("P.1".into(), 1.09, None, "active".into())]);
+    db.drop().await;
+}
+
+#[tokio::test]
+async fn hii_dams_and_rain_round_trip() {
+    let Some(db) = TestDb::new().await else {
+        return;
+    };
+    let now = t(2026, 10, 17, 21, 5);
+    let rain = RainRecord {
+        station_id: 11,
+        oldcode: Some("CHM001".into()),
+        name_th: None,
+        name_en: Some("Chiang Mai 1".into()),
+        latitude: Some(18.79),
+        longitude: Some(99.0),
+        sub_basin_id: Some("601".into()),
+        agency: Some("HII".into()),
+        timestamp: t(2026, 10, 17, 21, 0),
+        rain_1h: Some(1.234),
+        rain_24h: Some(12.0),
+    };
+    assert_eq!(
+        db::save_hii_rain(&db.pool, &[rain.clone(), rain.clone()], now)
+            .await
+            .unwrap(),
+        1
+    );
+    let (r1h,): (f64,) = sqlx::query_as("SELECT rain_1h::float8 FROM hii_rainfall")
+        .fetch_one(&db.pool)
+        .await
+        .unwrap();
+    assert_eq!(r1h, 1.23);
+
+    let wl = WaterLevelRecord {
+        station_id: 5,
+        oldcode: Some("ridhydro_P.1".into()),
+        rid_code: Some("P.1".into()),
+        name_th: Some("สะพานนวรัฐ".into()),
+        name_en: None,
+        latitude: None,
+        longitude: None,
+        agency: None,
+        river_name: None,
+        offset_msl: Some(300.0),
+        ground_level_msl: None,
+        min_bank_msl: None,
+        critical_level_msl: None,
+        critical_level_m: None,
+        qmax: None,
+        is_key_station: true,
+        timestamp: t(2026, 10, 17, 21, 0),
+        wl_msl: Some(301.2),
+        wl_m: None,
+        discharge: None,
+        flow_rate: None,
+        storage_percent: None,
+        situation_level: Some(2),
+        diff_wl_bank: None,
+    };
+    db::save_hii_waterlevel(&db.pool, &[wl], now).await.unwrap();
+    let (code, key, sit): (String, bool, i32) = sqlx::query_as(
+        "SELECT s.rid_code, s.is_key_station, w.situation_level
+         FROM hii_waterlevel w JOIN hii_wl_stations s ON s.id = w.station_id",
+    )
+    .fetch_one(&db.pool)
+    .await
+    .unwrap();
+    assert_eq!((code.as_str(), key, sit), ("P.1", true, 2));
+
+    let dam = |name: Option<&str>, level: Option<f64>| DamRecord {
+        dam_id: "200103".into(),
+        region: Some("north".into()),
+        name_th: name.map(str::to_string),
+        latitude: Some(19.17),
+        longitude: None,
+        capacity_max_mcm: Some(265.0),
+        capacity_normal_mcm: None,
+        date: d(2026, 10, 17),
+        storage_mcm: Some(200.0),
+        storage_pct: Some(75.5),
+        inflow_mcm: None,
+        outflow_mcm: Some(1.2),
+        level_msl: level,
+    };
+    db::save_dams(&db.pool, &[dam(Some("แม่งัด"), Some(395.1))], now)
+        .await
+        .unwrap();
+    // a later snapshot without name / level must not blank them
+    db::save_dams(&db.pool, &[dam(None, None)], now)
+        .await
+        .unwrap();
+    let (name, level): (Option<String>, Option<f64>) = sqlx::query_as(
+        "SELECT d.name_th, r.level_msl::float8 FROM rid_reservoir_daily r JOIN rid_dams d USING (dam_id)",
+    )
+    .fetch_one(&db.pool)
+    .await
+    .unwrap();
+    assert_eq!(name.as_deref(), Some("แม่งัด"));
+    assert_eq!(level, Some(395.1));
+
+    let rows = vec![RainRow {
+        timestamp: t(2026, 10, 18, 3, 0),
+        points: [Some(1.0), None, Some(2.0), None, Some(0.333)],
+        catchment_mean: Some(1.111),
+    }];
+    db::save_rain(&db.pool, &rows).await.unwrap();
+    let (mean, missing): (f64, Option<f64>) =
+        sqlx::query_as("SELECT catchment_mean::float8, mae_taeng::float8 FROM openmeteo_rain")
+            .fetch_one(&db.pool)
+            .await
+            .unwrap();
+    assert_eq!((mean, missing), (1.11, None));
+    db.drop().await;
+}
+
+fn forecast_row(code: &str, h: u32, p: f64) -> ping_forecast::Forecast {
+    ping_forecast::Forecast {
+        station_code: code.into(),
+        horizon_hours: h,
+        p_warning: p,
+        p_danger: p / 2.0,
+        predicted_max_level: 3.123456,
+        current_level: 1.23,
+        as_of: t(2026, 10, 18, 13, 0),
+        model_version: "hgb-v3+test".into(),
+        trained_at: Some("2026-10-01T03:30:00".into()),
+        source: "model".into(),
+        threshold_warning: 3.7,
+        threshold_danger: 4.2,
+        stages: None,
+    }
+}
+
+#[tokio::test]
+async fn forecast_outputs_round_trip() {
+    let Some(db) = TestDb::new().await else {
+        return;
+    };
+    // 0.123455 + 1 ulp: Python's repr gives 17 digits and NUMERIC(7,5)
+    // rounds that up; a float8 -> numeric cast (15 digits) would round down
+    let tricky = f64::from_bits(0.123455f64.to_bits() + 1);
+    let rows = [
+        forecast_row("P.1", 24, tricky),
+        forecast_row("P.1", 24, 0.2),
+        forecast_row("P.20", 6, 1e-7),
+    ];
+    assert_eq!(db::save_forecast_history(&db.pool, &rows).await.unwrap(), 2);
+    db::save_forecast_history(&db.pool, &rows[..1])
+        .await
+        .unwrap();
+    let got: Vec<(String, i32, f64, f64, f64)> = sqlx::query_as(
+        "SELECT station_code, horizon_hours, p_warning::float8, predicted_max_level::float8,
+                current_level::float8 FROM forecast_history ORDER BY station_code",
+    )
+    .fetch_all(&db.pool)
+    .await
+    .unwrap();
+    assert_eq!(
+        got,
+        vec![
+            ("P.1".into(), 24, 0.12346, 3.123, 1.23),
+            ("P.20".into(), 6, 0.0, 3.123, 1.23)
+        ]
+    );
+
+    let now = t(2026, 10, 18, 13, 7);
+    db::save_forecast_current(&db.pool, now, &rows[1..])
+        .await
+        .unwrap();
+    db::save_forecast_current(&db.pool, now, &rows[..1])
+        .await
+        .unwrap();
+    let (at, versions, payload): (NaiveDateTime, String, String) =
+        sqlx::query_as("SELECT computed_at, model_versions, payload FROM forecast_current")
+            .fetch_one(&db.pool)
+            .await
+            .unwrap();
+    assert_eq!((at, versions.as_str()), (now, "hgb-v3+test"));
+    let v: serde_json::Value = serde_json::from_str(&payload).unwrap();
+    assert_eq!(v[0]["as_of"], "2026-10-18T13:00:00");
+    assert_eq!(v[0]["p_warning"].as_f64(), Some(tricky));
+    assert!(v[0].get("stages").is_none());
+    db.drop().await;
+}
+
+#[tokio::test]
+async fn notification_state_round_trip() {
+    let Some(db) = TestDb::new().await else {
+        return;
+    };
+    let now = t(2026, 10, 18, 2, 0);
+    db::set_notification_state(&db.pool, "level:P.1", "warning", Some(3.7512), now)
+        .await
+        .unwrap();
+    db::set_notification_state(&db.pool, "feed", "stale", None, now)
+        .await
+        .unwrap();
+    db::set_notification_state(&db.pool, "level:P.1", "clear", Some(3.5), now)
+        .await
+        .unwrap();
+    let states = db::notification_states(&db.pool).await.unwrap();
+    assert_eq!(states.len(), 2);
+    assert_eq!(states["level:P.1"], "clear");
+    assert_eq!(states["feed"], "stale");
+    db.drop().await;
+}
+
+#[tokio::test]
+async fn forecast_inputs_load_like_python() {
+    let Some(db) = TestDb::new().await else {
+        return;
+    };
+    let stations = Stations::bundled();
+    let r = |h: u32, level: f64| Reading {
+        timestamp: t(2026, 10, 18, h, 0),
+        station_num: 8, // P.1
+        water_level: level,
+        discharge: Some(51.25),
+        discharge_percent: Some(40.5),
+    };
+    db::save_rid(&db.pool, &[r(10, 1.1), r(11, 1.2), r(12, 1.3)], &stations)
+        .await
+        .unwrap();
+    let rows = db::load_rid_window(&db.pool, t(2026, 10, 18, 11, 0), t(2026, 10, 18, 12, 30))
+        .await
+        .unwrap();
+    assert_eq!(rows.len(), 2);
+    assert_eq!(
+        (rows[0].station_code.as_str(), rows[0].water_level),
+        ("P.1", 1.2)
+    );
+    assert_eq!(rows[1].discharge, 51.25);
+
+    let latest = db::latest_rid_readings(&db.pool).await.unwrap();
+    assert_eq!(latest.len(), 1);
+    assert_eq!(
+        (
+            latest[0].timestamp,
+            latest[0].water_level,
+            latest[0].discharge_percent
+        ),
+        (t(2026, 10, 18, 12, 0), Some(1.3), Some(40.5))
+    );
+
+    let wl = |minute: u32, v: f64| WaterLevelRecord {
+        station_id: 5,
+        oldcode: None,
+        rid_code: Some("P.1".into()),
+        name_th: None,
+        name_en: None,
+        latitude: None,
+        longitude: None,
+        agency: None,
+        river_name: None,
+        offset_msl: None,
+        ground_level_msl: None,
+        min_bank_msl: None,
+        critical_level_msl: None,
+        critical_level_m: None,
+        qmax: None,
+        is_key_station: true,
+        timestamp: t(2026, 10, 18, 13, minute),
+        wl_msl: Some(v),
+        wl_m: None,
+        discharge: Some(52.0),
+        flow_rate: None,
+        storage_percent: None,
+        situation_level: None,
+        diff_wl_bank: None,
+    };
+    db::save_hii_waterlevel(
+        &db.pool,
+        &[wl(0, 301.4), wl(40, 301.45)],
+        t(2026, 10, 18, 13, 45),
+    )
+    .await
+    .unwrap();
+    let codes = vec!["P.1".to_string(), "P.103".to_string()];
+    let hii = db::load_hii_window(
+        &db.pool,
+        &codes,
+        t(2026, 10, 18, 0, 0),
+        t(2026, 10, 18, 14, 0),
+    )
+    .await
+    .unwrap();
+    assert_eq!(hii.len(), 2);
+    assert_eq!(ping_forecast::hii::prepare(&hii)[0].wl_msl, 301.45);
+    assert_eq!(
+        db::latest_hii_timestamp(&db.pool, &codes).await.unwrap(),
+        Some(t(2026, 10, 18, 13, 40))
+    );
+    assert!(
+        db::load_dam_days(&db.pool, "200103", d(2026, 10, 1))
+            .await
+            .unwrap()
+            .is_empty()
+    );
+    db.drop().await;
+}

rust/collector/src/envfile.rs

@@ -0,0 +1,489 @@
+//! `.env` loading with python-dotenv 1.1.1 semantics (`load_dotenv()`:
+//! interpolate `${VAR}` / `${VAR:-default}`, never override variables that
+//! are already set).
+//!
+//! The Python tools (src/config.py) read the same file, so the two must agree
+//! on every value.
+//! dotenvy (the usual crate) is stricter: a stray `'` or `"` inside an
+//! unquoted value makes it swallow the rest of the file and abort, and its
+//! error message echoes the line, password included. Here an unparsable
+//! statement is skipped with a warning naming only its line number, exactly
+//! like python-dotenv. Port of dotenv/parser.py + variables.py; the tests
+//! below are python-dotenv's own output for the same inputs.
+
+use std::collections::HashMap;
+use std::path::Path;
+
+/// Python's `\s` (str.isspace), which includes \x1c-\x1f.
+fn is_ws(c: char) -> bool {
+    c.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&c)
+}
+/// `[^\S\r\n]`: whitespace other than line breaks.
+fn is_hspace(c: char) -> bool {
+    is_ws(c) && c != '\r' && c != '\n'
+}
+
+struct Reader {
+    s: Vec<char>,
+    pos: usize,
+}
+
+struct Fail;
+
+impl Reader {
+    fn peek(&self) -> Option<char> {
+        self.s.get(self.pos).copied()
+    }
+    fn at(&self, i: usize) -> Option<char> {
+        self.s.get(i).copied()
+    }
+    fn skip_while(&mut self, f: impl Fn(char) -> bool) {
+        while self.peek().is_some_and(&f) {
+            self.pos += 1;
+        }
+    }
+    fn starts_with(&self, lit: &str) -> bool {
+        lit.chars()
+            .enumerate()
+            .all(|(i, c)| self.at(self.pos + i) == Some(c))
+    }
+    fn line_of(&self, pos: usize) -> usize {
+        // python-dotenv counts \r\n, \n and \r as one newline each
+        let mut line = 1;
+        let mut i = 0;
+        while i < pos {
+            match self.s[i] {
+                '\r' if self.at(i + 1) == Some('\n') && i + 1 < pos => {
+                    line += 1;
+                    i += 1;
+                }
+                '\r' | '\n' => line += 1,
+                _ => {}
+            }
+            i += 1;
+        }
+        line
+    }
+
+    /// `(?:export[^\S\r\n]+)?`
+    fn export(&mut self) {
+        if self.starts_with("export") && self.at(self.pos + 6).is_some_and(is_hspace) {
+            self.pos += 6;
+            self.skip_while(is_hspace);
+        }
+    }
+
+    /// `'([^']+)'` or `([^=\#\s]+)`; None for a comment line.
+    fn key(&mut self) -> Result<Option<String>, Fail> {
+        match self.peek() {
+            Some('#') => Ok(None),
+            Some('\'') => {
+                let start = self.pos + 1;
+                let mut i = start;
+                while self.at(i).is_some_and(|c| c != '\'') {
+                    i += 1;
+                }
+                if i == start || self.at(i) != Some('\'') {
+                    return Err(Fail);
+                }
+                self.pos = i + 1;
+                Ok(Some(self.s[start..i].iter().collect()))
+            }
+            _ => {
+                let start = self.pos;
+                self.skip_while(|c| c != '=' && c != '#' && !is_ws(c));
+                if self.pos == start {
+                    return Err(Fail);
+                }
+                Ok(Some(self.s[start..self.pos].iter().collect()))
+            }
+        }
+    }
+
+    /// `Q((?:\\Q|[^Q])*)Q` with regex backtracking: the greedy scan stops at
+    /// an unescaped quote; at end of input the last `\Q` is re-read as a
+    /// literal backslash followed by the closing quote.
+    fn quoted(&mut self, q: char) -> Result<String, Fail> {
+        let start = self.pos + 1;
+        let mut i = start;
+        let mut last_escaped_quote = None;
+        loop {
+            match self.at(i) {
+                None => {
+                    let Some(bs) = last_escaped_quote else {
+                        return Err(Fail);
+                    };
+                    self.pos = bs + 2;
+                    return Ok(self.s[start..=bs].iter().collect());
+                }
+                Some(c) if c == q => {
+                    self.pos = i + 1;
+                    return Ok(self.s[start..i].iter().collect());
+                }
+                Some('\\') if self.at(i + 1) == Some(q) => {
+                    last_escaped_quote = Some(i);
+                    i += 2;
+                }
+                Some(_) => i += 1,
+            }
+        }
+    }
+
+    /// parse_value
+    fn value(&mut self) -> Result<String, Fail> {
+        match self.peek() {
+            Some('\'') => Ok(decode_escapes(&self.quoted('\'')?, "\\'")),
+            Some('"') => Ok(decode_escapes(&self.quoted('"')?, "\\'\"abfnrtv")),
+            None | Some('\n') | Some('\r') => Ok(String::new()),
+            Some(_) => {
+                let start = self.pos;
+                self.skip_while(|c| c != '\r' && c != '\n');
+                let part: Vec<char> = self.s[start..self.pos].to_vec();
+                Ok(strip_inline_comment(&part))
+            }
+        }
+    }
+
+    /// `(?:[^\S\r\n]*#[^\r\n]*)?` then `[^\S\r\n]*(?:\r\n|\n|\r|$)`
+    fn comment_and_eol(&mut self) -> Result<(), Fail> {
+        let save = self.pos;
+        self.skip_while(is_hspace);
+        if self.peek() == Some('#') {
+            self.skip_while(|c| c != '\r' && c != '\n');
+        } else {
+            self.pos = save;
+        }
+        self.skip_while(is_hspace);
+        match self.peek() {
+            None => Ok(()),
+            Some('\r') => {
+                self.pos += if self.at(self.pos + 1) == Some('\n') {
+                    2
+                } else {
+                    1
+                };
+                Ok(())
+            }
+            Some('\n') => {
+                self.pos += 1;
+                Ok(())
+            }
+            Some(_) => Err(Fail),
+        }
+    }
+
+    /// `[^\r\n]*(?:\r|\n|\r\n)?` (the `\r` alternative wins for `\r\n`)
+    fn rest_of_line(&mut self) {
+        self.skip_while(|c| c != '\r' && c != '\n');
+        if self.peek().is_some() {
+            self.pos += 1;
+        }
+    }
+}
+
+/// Unquoted values: `re.sub(r"\s+#.*", "", part).rstrip()`.
+fn strip_inline_comment(part: &[char]) -> String {
+    let mut end = part.len();
+    let mut i = 0;
+    while i < part.len() {
+        if is_ws(part[i]) {
+            let mut j = i;
+            while j < part.len() && is_ws(part[j]) {
+                j += 1;
+            }
+            if part.get(j) == Some(&'#') {
+                end = i;
+                break;
+            }
+            i = j;
+        } else {
+            i += 1;
+        }
+    }
+    let kept: String = part[..end].iter().collect();
+    kept.trim_end_matches(is_ws).to_string()
+}
+
+/// `\\[<allowed>]` -> codecs 'unicode-escape' of that pair.
+fn decode_escapes(s: &str, allowed: &str) -> String {
+    let mut out = String::with_capacity(s.len());
+    let mut chars = s.chars().peekable();
+    while let Some(c) = chars.next() {
+        if c == '\\'
+            && let Some(&n) = chars.peek()
+            && allowed.contains(n)
+        {
+            chars.next();
+            out.push(match n {
+                'a' => '\u{7}',
+                'b' => '\u{8}',
+                'f' => '\u{c}',
+                'n' => '\n',
+                'r' => '\r',
+                't' => '\t',
+                'v' => '\u{b}',
+                other => other, // \\ \' \"
+            });
+        } else {
+            out.push(c);
+        }
+    }
+    out
+}
+
+/// Statements in file order: (key, value or None for `KEY` without `=`).
+/// Unparsable statements are reported as line numbers in `errors`.
+pub fn parse(content: &str) -> (Vec<(String, Option<String>)>, Vec<usize>) {
+    let mut r = Reader {
+        s: content.chars().collect(),
+        pos: 0,
+    };
+    let mut out = Vec::new();
+    let mut errors = Vec::new();
+    while r.pos < r.s.len() {
+        r.skip_while(is_ws); // _multiline_whitespace
+        if r.pos >= r.s.len() {
+            break;
+        }
+        let statement_start = r.pos;
+        let binding = (|| {
+            r.export();
+            let key = r.key()?;
+            r.skip_while(is_hspace);
+            let value = if r.peek() == Some('=') {
+                r.pos += 1;
+                r.skip_while(is_hspace);
+                Some(r.value()?)
+            } else {
+                None
+            };
+            r.comment_and_eol()?;
+            Ok::<_, Fail>(key.map(|k| (k, value)))
+        })();
+        match binding {
+            Ok(Some(kv)) => out.push(kv),
+            Ok(None) => {}
+            Err(Fail) => {
+                errors.push(r.line_of(statement_start));
+                r.rest_of_line();
+            }
+        }
+    }
+    (out, errors)
+}
+
+/// `${NAME}` / `${NAME:-default}` expansion, python-dotenv `parse_variables`.
+fn interpolate(value: &str, lookup: &impl Fn(&str) -> Option<String>) -> String {
+    let v: Vec<char> = value.chars().collect();
+    let mut out = String::new();
+    let mut i = 0;
+    while i < v.len() {
+        if v[i] == '$' && v.get(i + 1) == Some(&'{') {
+            let mut j = i + 2;
+            while j < v.len() && v[j] != '}' && v[j] != ':' {
+                j += 1;
+            }
+            let name: String = v[i + 2..j].iter().collect();
+            let mut default = None;
+            let mut k = j;
+            if v.get(j) == Some(&':') && v.get(j + 1) == Some(&'-') {
+                k = j + 2;
+                while k < v.len() && v[k] != '}' {
+                    k += 1;
+                }
+                default = Some(v[j + 2..k].iter().collect::<String>());
+            }
+            if v.get(k) == Some(&'}') {
+                out.push_str(&lookup(&name).or(default).unwrap_or_default());
+                i = k + 1;
+                continue;
+            }
+        }
+        out.push(v[i]);
+        i += 1;
+    }
+    out
+}
+
+/// python-dotenv `dotenv_values` with override=False: what `load_dotenv()`
+/// would set. `env` looks up the process environment.
+pub fn resolve(
+    content: &str,
+    env: impl Fn(&str) -> Option<String>,
+) -> (Vec<(String, Option<String>)>, Vec<usize>) {
+    let (raw, errors) = parse(content);
+    let mut resolved: Vec<(String, Option<String>)> = Vec::new();
+    let mut by_key: HashMap<String, usize> = HashMap::new();
+    for (key, value) in raw {
+        let result = value.map(|v| {
+            // override=False: the process environment wins over the file
+            let lookup = |name: &str| {
+                env(name).or_else(|| {
+                    by_key
+                        .get(name)
+                        .and_then(|&i| resolved[i].1.clone())
+                        .or_else(|| {
+                            by_key.contains_key(name).then(String::new) // KEY without '='
+                        })
+                })
+            };
+            interpolate(&v, &lookup)
+        });
+        match by_key.get(&key) {
+            Some(&i) => resolved[i].1 = result,
+            None => {
+                by_key.insert(key.clone(), resolved.len());
+                resolved.push((key, result));
+            }
+        }
+    }
+    (resolved, errors)
+}
+
+/// `load_dotenv(path)`: set every variable that is not set yet. Must run
+/// before any other thread exists (`set_var`). Returns how many were set.
+pub fn load(path: &Path) -> std::io::Result<usize> {
+    let content = std::fs::read_to_string(path)?;
+    let (values, errors) = resolve(&content, |k| std::env::var(k).ok());
+    for line in errors {
+        eprintln!(
+            "warning: {}: could not parse the statement on line {line}; skipped (as python-dotenv does)",
+            path.display()
+        );
+    }
+    let mut set = 0;
+    for (key, value) in values {
+        if std::env::var_os(&key).is_some() {
+            continue;
+        }
+        if let Some(value) = value {
+            // SAFETY: called from main() before the tokio runtime (or any
+            // other thread) is started.
+            unsafe { std::env::set_var(&key, value) };
+            set += 1;
+        }
+    }
+    Ok(set)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn vals(content: &str) -> Vec<(String, Option<String>)> {
+        resolve(content, |k| {
+            (k == "HOME_FROM_ENV").then(|| "/home/x".into())
+        })
+        .0
+    }
+    fn kv(k: &str, v: &str) -> (String, Option<String>) {
+        (k.into(), Some(v.into()))
+    }
+
+    // Expected values below are python-dotenv 1.1.1 `DotEnv(stream,
+    // interpolate=True, override=False).dict()` for the same input.
+
+    #[test]
+    fn stray_quotes_in_unquoted_values_are_literal() {
+        assert_eq!(
+            vals("A=abc'def\nB=x\"y\nC=1\n"),
+            vec![kv("A", "abc'def"), kv("B", "x\"y"), kv("C", "1")]
+        );
+    }
+
+    #[test]
+    fn unterminated_quote_skips_only_that_line() {
+        let (v, errors) = resolve("A=1\nPOSTGRES_PASSWORD='abc\nB=2\n", |_| None);
+        assert_eq!(v, vec![kv("A", "1"), kv("B", "2")]);
+        assert_eq!(errors, vec![2]);
+    }
+
+    #[test]
+    fn whitespace_comments_and_export() {
+        assert_eq!(
+            vals("  export  A = spaced value   # comment\nB=a#b\n# c\n\nC= \nD\n"),
+            vec![
+                kv("A", "spaced value"),
+                kv("B", "a#b"),
+                kv("C", ""),
+                ("D".into(), None)
+            ]
+        );
+    }
+
+    #[test]
+    fn quoted_values_and_escapes() {
+        assert_eq!(
+            vals("A='x\\'y'\nB=\"l1\\nl2\\t\\\"q\\\"\"\nC='multi\nline'\nD='a\\\\b'\n"),
+            vec![
+                kv("A", "x'y"),
+                kv("B", "l1\nl2\t\"q\""),
+                kv("C", "multi\nline"),
+                kv("D", "a\\b")
+            ]
+        );
+    }
+
+    #[test]
+    fn trailing_escaped_quote_backtracks_like_the_regex() {
+        // python: '((?:\\'|[^'])*)' on "'ab\'" matches with value "ab\"
+        assert_eq!(vals("A='ab\\'"), vec![kv("A", "ab\\")]);
+    }
+
+    #[test]
+    fn junk_after_a_quoted_value_is_an_error() {
+        let (v, errors) = resolve("A='x'y\nB=1\n", |_| None);
+        assert_eq!(v, vec![kv("B", "1")]);
+        assert_eq!(errors, vec![1]);
+    }
+
+    #[test]
+    fn interpolation_prefers_the_environment() {
+        assert_eq!(
+            vals(
+                "HOME_FROM_ENV=file\nA=${HOME_FROM_ENV}/x\nB=${A}-${MISSING:-dflt}-${NOPE}\nC=$A ${A:x}\n"
+            ),
+            vec![
+                kv("HOME_FROM_ENV", "file"),
+                kv("A", "/home/x/x"),
+                kv("B", "/home/x/x-dflt-"),
+                kv("C", "$A ${A:x}")
+            ]
+        );
+    }
+
+    /// 614 inputs (the cases above + 600 random ones built from quotes,
+    /// escapes, comments, `${}` and odd separators/line ends) with
+    /// python-dotenv 1.1.1's result; regenerate with testdata/gen_envfile_cases.py.
+    #[test]
+    fn matches_python_dotenv_on_generated_cases() {
+        let fixture: serde_json::Value =
+            serde_json::from_str(include_str!("../testdata/envfile_python_dotenv.json")).unwrap();
+        let cases = fixture["cases"].as_array().unwrap();
+        assert!(cases.len() > 600);
+        for case in cases {
+            let input = case["input"].as_str().unwrap();
+            let want: Vec<(String, Option<String>)> = case["expected"]
+                .as_array()
+                .unwrap()
+                .iter()
+                .map(|kv| {
+                    (
+                        kv[0].as_str().unwrap().to_string(),
+                        kv[1].as_str().map(str::to_string),
+                    )
+                })
+                .collect();
+            let got = resolve(input, |k| (k == "HOME_FROM_ENV").then(|| "/home/x".into())).0;
+            assert_eq!(got, want, "input {input:?}");
+        }
+    }
+
+    #[test]
+    fn crlf_and_duplicate_keys() {
+        assert_eq!(
+            vals("A=1\r\nB=2\r\nA=3\r\n"),
+            vec![kv("A", "3"), kv("B", "2")]
+        );
+    }
+}

rust/collector/src/forecasting.rs

@@ -0,0 +1,286 @@
+//! Forecasting and notifications in the collector (FORECAST_IN_COLLECTOR=true);
+//! the port of the retired Python app's forecast_notify_task.
+//!
+//! After each collection cycle that brought newer RID or HII readings, and at
+//! least once per clock hour: load the last 336 h (RID plus the HII mirror
+//! fill), fetch the rain forecast, run every exported model, store the result
+//! (`forecast_history`, and `forecast_current` which the API serves as
+//! /forecast) and push ntfy transitions.
+
+use crate::config::Config;
+use crate::db;
+use crate::notify::{self, Notifier, Settings};
+use crate::rain::OpenMeteoClient;
+use crate::util::floor_hour;
+use anyhow::{Result, bail};
+use chrono::{Duration, NaiveDateTime};
+use ping_forecast::config::config;
+use ping_forecast::dam::hourly_frame;
+use ping_forecast::hii::{self, FillInfo};
+use ping_forecast::rain::RainSeries;
+use ping_forecast::{Forecast, ModelSet, ModelStore};
+use sqlx::PgPool;
+
+/// Everything one forecast run used and produced.
+pub struct Snapshot {
+    pub now: NaiveDateTime,
+    pub fills: Vec<FillInfo>,
+    pub rain_hours: usize,
+    pub dam_days: usize,
+    pub forecasts: Vec<Forecast>,
+}
+
+/// `predict.get_latest_forecasts` with this process's data access. Source
+/// failures degrade like Python: no HII fill, NaN rain, NaN dam features.
+pub async fn compute(
+    pool: &PgPool,
+    rain_client: &OpenMeteoClient,
+    models: &ModelSet,
+    now: NaiveDateTime,
+    rain_override: Option<RainSeries>,
+) -> Result<Snapshot> {
+    let cfg = config();
+    let start = now - Duration::hours(cfg.latest_window_h);
+    let rid = db::load_rid_window(pool, start, now).await?;
+    let codes = hii::fill_candidates(&rid);
+    let hii_rows = if codes.is_empty() {
+        vec![]
+    } else {
+        db::load_hii_window(pool, &codes, start, now)
+            .await
+            .unwrap_or_else(|e| {
+                tracing::warn!("HII gap-fill skipped (fetch failed): {e:#}");
+                vec![]
+            })
+    };
+    let (merged, fills) = hii::fill(&rid, &hii::prepare(&hii_rows));
+    for f in &fills {
+        tracing::debug!(
+            "HII gap-fill {}: +{} hours (offset {:.3} m)",
+            f.station_code,
+            f.filled_hours,
+            f.offset
+        );
+    }
+    let rain: RainSeries = match rain_override {
+        Some(r) => r,
+        None => match rain_client.fetch_forecast(5, 2).await {
+            Ok(rows) => rows
+                .into_iter()
+                .map(|r| (r.timestamp, r.catchment_mean.unwrap_or(f64::NAN)))
+                .collect(),
+            Err(e) => {
+                tracing::warn!("live rain unavailable, rain features will be NaN: {e:#}");
+                RainSeries::new()
+            }
+        },
+    };
+    let dam_start = now.date() - Duration::days(cfg.dam.serving_days);
+    let days = db::load_dam_days(pool, &cfg.dam.dam_id, dam_start)
+        .await
+        .unwrap_or_else(|e| {
+            tracing::warn!("dam state unavailable, dam features will be NaN: {e:#}");
+            vec![]
+        });
+    let dam = hourly_frame(&days).unwrap_or_default();
+    let forecasts = ping_forecast::forecasts(&merged, models, now, Some(&rain), Some(&dam));
+    Ok(Snapshot {
+        now,
+        fills,
+        rain_hours: rain.values().filter(|v| !v.is_nan()).count(),
+        dam_days: days.len(),
+        forecasts,
+    })
+}
+
+pub fn summarize(s: &Snapshot) -> String {
+    let model = s.forecasts.iter().filter(|f| f.source == "model").count();
+    let p1 = s
+        .forecasts
+        .iter()
+        .find(|f| f.station_code == "P.1" && f.horizon_hours == 24);
+    let filled: Vec<String> = s
+        .fills
+        .iter()
+        .filter(|f| {
+            f.latest
+                .is_some_and(|(t, _)| s.now - t < Duration::hours(24))
+        })
+        .map(|f| format!("{} +{}h", f.station_code, f.filled_hours))
+        .collect();
+    format!(
+        "{} rows ({model} model, {} heuristic){}{}; rain {} h, dam {} days",
+        s.forecasts.len(),
+        s.forecasts.len() - model,
+        p1.map(|f| format!(
+            "; P.1 {} {:.2} m, 24 h peak {:.2} m, p_warning {:.3}",
+            f.as_of.format("%d %H:%M"),
+            f.current_level,
+            f.predicted_max_level,
+            f.p_warning
+        ))
+        .unwrap_or_default(),
+        if filled.is_empty() {
+            String::new()
+        } else {
+            format!("; HII fill {}", filled.join(", "))
+        },
+        s.rain_hours,
+        s.dam_days
+    )
+}
+
+/// ntfy (NTFY_SERVER) and/or Matrix (MATRIX_ACCESS_TOKEN + MATRIX_ROOM_ID);
+/// `None` when neither is configured.
+pub fn notifier(cfg: &Config, http: reqwest::Client) -> Option<Notifier> {
+    let ntfy = cfg
+        .ntfy_publish_url
+        .clone()
+        .map(|publish_url| notify::Ntfy {
+            publish_url,
+            token: cfg.ntfy_token.clone(),
+        });
+    if ntfy.is_none() && cfg.matrix.is_none() {
+        return None;
+    }
+    Some(Notifier {
+        http,
+        ntfy,
+        matrix: cfg.matrix.clone(),
+        dashboard_url: cfg.public_url.clone(),
+        settings: Settings {
+            prefix: cfg.ntfy_topic_prefix.clone(),
+            rid_stale_alert_hours: cfg.rid_stale_alert_hours,
+        },
+    })
+}
+
+/// What the status file reports about the last run.
+#[derive(Clone, Debug)]
+pub struct RunInfo {
+    pub at: NaiveDateTime,
+    pub rows: usize,
+    pub model_rows: usize,
+    pub models: usize,
+    pub notified: usize,
+}
+
+pub struct Forecaster {
+    store: ModelStore,
+    notifier: Option<Notifier>,
+    last_inputs: Option<(Option<NaiveDateTime>, Option<NaiveDateTime>)>,
+    last_slot: Option<NaiveDateTime>,
+}
+
+impl Forecaster {
+    pub fn new(cfg: &Config, http: reqwest::Client) -> Self {
+        let notifier = notifier(cfg, http);
+        match notifier
+            .as_ref()
+            .and_then(|n| n.ntfy.as_ref().map(|t| (n, t)))
+        {
+            Some((n, t)) => tracing::info!(
+                "ntfy notifications: publish to {}, topics {}-*",
+                t.publish_url,
+                n.settings.prefix
+            ),
+            None => tracing::info!("ntfy notifications off (NTFY_SERVER not set)"),
+        }
+        match &cfg.matrix {
+            Some(m) => tracing::info!(
+                "Matrix notifications: room {} on {}",
+                m.room_id,
+                m.homeserver
+            ),
+            None => tracing::info!(
+                "Matrix notifications off (MATRIX_ACCESS_TOKEN/MATRIX_ROOM_ID not set)"
+            ),
+        }
+        Self {
+            store: ModelStore::new(&cfg.models_dir),
+            notifier,
+            last_inputs: None,
+            last_slot: None,
+        }
+    }
+
+    /// Run when RID or the HII mirrors brought newer readings, and once per
+    /// clock hour regardless (so a stopped feed is noticed and reported).
+    pub async fn maybe_run(
+        &mut self,
+        pool: &PgPool,
+        rain: &OpenMeteoClient,
+        now: NaiveDateTime,
+    ) -> Option<RunInfo> {
+        let codes = &config().hii_fill_stations;
+        let inputs = (
+            db::latest_rid_timestamp(pool).await.ok().flatten(),
+            db::latest_hii_timestamp(pool, codes).await.ok().flatten(),
+        );
+        let slot = floor_hour(now);
+        if self.last_inputs == Some(inputs) && self.last_slot == Some(slot) {
+            return None;
+        }
+        match self.run(pool, rain, now).await {
+            Ok(info) => {
+                self.last_inputs = Some(inputs);
+                self.last_slot = Some(slot);
+                Some(info)
+            }
+            Err(e) => {
+                tracing::error!("forecast run failed: {e:#}");
+                None
+            }
+        }
+    }
+
+    pub async fn run(
+        &mut self,
+        pool: &PgPool,
+        rain: &OpenMeteoClient,
+        now: NaiveDateTime,
+    ) -> Result<RunInfo> {
+        let models = self.store.refresh()?;
+        if models.is_empty() {
+            bail!(
+                "no exported models in {} (run `python -m src.ml.export`)",
+                self.store.dir().display()
+            );
+        }
+        let snap = compute(pool, rain, &models, now, None).await?;
+        let saved = db::save_forecast_history(pool, &snap.forecasts).await?;
+        db::save_forecast_current(pool, now, &snap.forecasts).await?;
+        tracing::info!("forecast: {}; {saved} stored", summarize(&snap));
+        let mut notified = 0;
+        if let Some(n) = &self.notifier {
+            let readings =
+                notify::readings_with_hii(&db::latest_rid_readings(pool).await?, &snap.fills);
+            match n.run(pool, &readings, &snap.forecasts, now).await {
+                Ok(sent) => {
+                    notified = sent.len();
+                    if !sent.is_empty() {
+                        tracing::info!(
+                            "notifications: published {}",
+                            sent.iter()
+                                .map(|s| format!("{}: {}", s.topic, s.title))
+                                .collect::<Vec<_>>()
+                                .join(", ")
+                        );
+                    }
+                }
+                Err(e) => tracing::warn!("notify cycle failed: {e:#}"),
+            }
+        }
+        Ok(RunInfo {
+            at: now,
+            rows: snap.forecasts.len(),
+            model_rows: snap
+                .forecasts
+                .iter()
+                .filter(|f| f.source == "model")
+                .count(),
+            models: models.len(),
+            notified,
+        })
+    }
+}

rust/collector/src/hii.rs

@@ -0,0 +1,262 @@
+//! HII / ThaiWater open api-v3 (no key): Ping-basin rain gauges and an
+//! independent water-level network (m MSL). Port of src/hii_collector.py.
+
+use crate::util::{lenient_f64, lenient_i64, scalar_string, truthy};
+use anyhow::{Context, Result};
+use chrono::NaiveDateTime;
+use serde_json::Value;
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct RainRecord {
+    pub station_id: i64,
+    pub oldcode: Option<String>,
+    pub name_th: Option<String>,
+    pub name_en: Option<String>,
+    pub latitude: Option<f64>,
+    pub longitude: Option<f64>,
+    pub sub_basin_id: Option<String>,
+    pub agency: Option<String>,
+    pub timestamp: NaiveDateTime,
+    pub rain_1h: Option<f64>,
+    pub rain_24h: Option<f64>,
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct WaterLevelRecord {
+    pub station_id: i64,
+    pub oldcode: Option<String>,
+    pub rid_code: Option<String>,
+    pub name_th: Option<String>,
+    pub name_en: Option<String>,
+    pub latitude: Option<f64>,
+    pub longitude: Option<f64>,
+    pub agency: Option<String>,
+    pub river_name: Option<String>,
+    pub offset_msl: Option<f64>,
+    pub ground_level_msl: Option<f64>,
+    pub min_bank_msl: Option<f64>,
+    pub critical_level_msl: Option<f64>,
+    pub critical_level_m: Option<f64>,
+    pub qmax: Option<f64>,
+    pub is_key_station: bool,
+    pub timestamp: NaiveDateTime,
+    pub wl_msl: Option<f64>,
+    pub wl_m: Option<f64>,
+    pub discharge: Option<f64>,
+    pub flow_rate: Option<f64>,
+    pub storage_percent: Option<f64>,
+    pub situation_level: Option<i32>,
+    pub diff_wl_bank: Option<f64>,
+}
+
+/// Thai local time, '2026-08-11 13:00' or with seconds.
+fn parse_datetime(v: Option<&Value>) -> Option<NaiveDateTime> {
+    let s = v?.as_str()?;
+    NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M")
+        .or_else(|_| NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S"))
+        .ok()
+}
+
+/// `tele_station_name` is {th, en} or a bare (Thai) string.
+fn name(station: &Value, lang: &str) -> Option<String> {
+    match station.get("tele_station_name")? {
+        Value::Object(o) => o.get(lang).and_then(Value::as_str).map(str::to_string),
+        Value::String(s) if lang == "th" => Some(s.clone()),
+        _ => None,
+    }
+}
+
+fn string(v: Option<&Value>) -> Option<String> {
+    v.and_then(Value::as_str).map(str::to_string)
+}
+
+fn agency(row: &Value) -> Option<String> {
+    string(row.pointer("/agency/agency_shortname/en"))
+}
+
+/// 'P.1', 'ridhydro_P.67' -> 'P.1' / 'P.67'; 'ridtele_TUP.14' -> None.
+/// (Python: `(?:^|_)(P\.\d+[A-Z]?)$`.)
+pub fn rid_code_from_oldcode(oldcode: &str) -> Option<String> {
+    let seg = oldcode.rsplit('_').next()?;
+    let digits = seg.strip_prefix("P.")?;
+    let (num, suffix) = match digits.char_indices().find(|(_, c)| !c.is_ascii_digit()) {
+        Some((i, _)) => digits.split_at(i),
+        None => (digits, ""),
+    };
+    let suffix_ok = suffix.is_empty()
+        || (suffix.chars().count() == 1 && suffix.chars().all(|c| c.is_ascii_uppercase()));
+    (!num.is_empty() && suffix_ok).then(|| seg.to_string())
+}
+
+fn in_basin(row: &Value, basin_code: i64) -> bool {
+    row.pointer("/basin/basin_code").and_then(Value::as_i64) == Some(basin_code)
+}
+
+pub fn parse_rain(payload: &Value, basin_code: i64) -> Vec<RainRecord> {
+    let rows = payload
+        .get("data")
+        .and_then(Value::as_array)
+        .cloned()
+        .unwrap_or_default();
+    rows.iter()
+        .filter(|row| in_basin(row, basin_code))
+        .filter_map(|row| {
+            let station = row.get("station").unwrap_or(&Value::Null);
+            Some(RainRecord {
+                station_id: lenient_i64(station.get("id"))?,
+                timestamp: parse_datetime(row.get("rainfall_datetime"))?,
+                oldcode: string(station.get("tele_station_oldcode")),
+                name_th: name(station, "th"),
+                name_en: name(station, "en"),
+                latitude: lenient_f64(station.get("tele_station_lat")),
+                longitude: lenient_f64(station.get("tele_station_long")),
+                // Python: str(x or "") or None
+                sub_basin_id: truthy(station.get("sub_basin_id"))
+                    .then(|| scalar_string(station.get("sub_basin_id")))
+                    .flatten(),
+                agency: agency(row),
+                rain_1h: lenient_f64(row.get("rain_1h")),
+                rain_24h: lenient_f64(row.get("rain_24h")),
+            })
+        })
+        .collect()
+}
+
+pub fn parse_waterlevel(payload: &Value, basin_code: i64) -> Vec<WaterLevelRecord> {
+    let rows = payload
+        .pointer("/waterlevel_data/data")
+        .and_then(Value::as_array)
+        .cloned()
+        .unwrap_or_default();
+    rows.iter()
+        .filter(|row| in_basin(row, basin_code))
+        .filter_map(|row| {
+            let station = row.get("station").unwrap_or(&Value::Null);
+            let oldcode = string(station.get("tele_station_oldcode"));
+            Some(WaterLevelRecord {
+                station_id: lenient_i64(station.get("id"))?,
+                timestamp: parse_datetime(row.get("waterlevel_datetime"))?,
+                rid_code: oldcode.as_deref().and_then(rid_code_from_oldcode),
+                oldcode,
+                name_th: name(station, "th"),
+                name_en: name(station, "en"),
+                latitude: lenient_f64(station.get("tele_station_lat")),
+                longitude: lenient_f64(station.get("tele_station_long")),
+                agency: agency(row),
+                river_name: string(row.get("river_name")),
+                offset_msl: lenient_f64(station.get("offset")),
+                ground_level_msl: lenient_f64(station.get("ground_level")),
+                min_bank_msl: lenient_f64(station.get("min_bank")),
+                critical_level_msl: lenient_f64(station.get("critical_level_msl")),
+                critical_level_m: lenient_f64(station.get("critical_level_m")),
+                qmax: lenient_f64(station.get("qmax")),
+                is_key_station: truthy(station.get("is_key_station")),
+                wl_msl: lenient_f64(row.get("waterlevel_msl")),
+                wl_m: lenient_f64(row.get("waterlevel_m")),
+                discharge: lenient_f64(row.get("discharge")),
+                flow_rate: lenient_f64(row.get("flow_rate")),
+                storage_percent: lenient_f64(row.get("storage_percent")),
+                situation_level: lenient_i64(row.get("situation_level")).map(|v| v as i32),
+                diff_wl_bank: lenient_f64(row.get("diff_wl_bank")),
+            })
+        })
+        .collect()
+}
+
+pub struct HiiClient {
+    http: reqwest::Client,
+    base: String,
+}
+
+impl HiiClient {
+    pub fn new(http: reqwest::Client, base: String) -> Self {
+        Self {
+            http,
+            base: base.trim_end_matches('/').to_string(),
+        }
+    }
+
+    async fn get(&self, endpoint: &str) -> Result<Value> {
+        self.http
+            .get(format!("{}/{endpoint}", self.base))
+            .timeout(std::time::Duration::from_secs(90))
+            .send()
+            .await
+            .with_context(|| format!("HII {endpoint} request failed"))?
+            .error_for_status()?
+            .json()
+            .await
+            .with_context(|| format!("HII {endpoint} response is not JSON"))
+    }
+
+    pub async fn fetch_rain(&self, basin_code: i64) -> Result<Vec<RainRecord>> {
+        Ok(parse_rain(&self.get("rain_24h").await?, basin_code))
+    }
+
+    pub async fn fetch_waterlevel(&self, basin_code: i64) -> Result<Vec<WaterLevelRecord>> {
+        Ok(parse_waterlevel(
+            &self.get("waterlevel_load").await?,
+            basin_code,
+        ))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use serde_json::json;
+
+    #[test]
+    fn rid_codes() {
+        assert_eq!(rid_code_from_oldcode("P.1").as_deref(), Some("P.1"));
+        assert_eq!(
+            rid_code_from_oldcode("ridhydro_P.67").as_deref(),
+            Some("P.67")
+        );
+        assert_eq!(rid_code_from_oldcode("x_P.4A").as_deref(), Some("P.4A"));
+        assert_eq!(rid_code_from_oldcode("ridtele_TUP.14"), None);
+        assert_eq!(rid_code_from_oldcode("P.4AB"), None);
+        assert_eq!(rid_code_from_oldcode("CHM001"), None);
+    }
+
+    #[test]
+    fn rain_rows_filtered_to_basin() {
+        let payload = json!({"data": [
+            {"basin": {"basin_code": 6}, "station": {"id": 11, "tele_station_oldcode": "CHM001",
+              "tele_station_name": {"th": "ชม1", "en": "Chiang Mai 1"}, "tele_station_lat": "18.79",
+              "tele_station_long": 99.0, "sub_basin_id": 601},
+             "agency": {"agency_shortname": {"en": "HII"}},
+             "rainfall_datetime": "2026-08-11 13:00", "rain_1h": "1.5", "rain_24h": 12},
+            {"basin": {"basin_code": 7}, "station": {"id": 12}, "rainfall_datetime": "2026-08-11 13:00"},
+            {"basin": {"basin_code": 6}, "station": {"id": 13}, "rainfall_datetime": "bad"}
+        ]});
+        let r = parse_rain(&payload, 6);
+        assert_eq!(r.len(), 1);
+        assert_eq!(r[0].station_id, 11);
+        assert_eq!(r[0].name_en.as_deref(), Some("Chiang Mai 1"));
+        assert_eq!(r[0].sub_basin_id.as_deref(), Some("601"));
+        assert_eq!(r[0].agency.as_deref(), Some("HII"));
+        assert_eq!(r[0].rain_1h, Some(1.5));
+        assert_eq!(r[0].latitude, Some(18.79));
+    }
+
+    #[test]
+    fn waterlevel_rows() {
+        let payload = json!({"waterlevel_data": {"data": [
+            {"basin": {"basin_code": 6},
+             "station": {"id": 5, "tele_station_oldcode": "ridhydro_P.1", "tele_station_name": "สะพานนวรัฐ",
+                         "offset": "300.0", "is_key_station": 1},
+             "waterlevel_datetime": "2026-08-11 13:00:00", "waterlevel_msl": "301.2",
+             "situation_level": "2", "river_name": "แม่น้ำปิง"}
+        ]}});
+        let r = parse_waterlevel(&payload, 6);
+        assert_eq!(r.len(), 1);
+        assert_eq!(r[0].rid_code.as_deref(), Some("P.1"));
+        assert_eq!(r[0].name_th.as_deref(), Some("สะพานนวรัฐ"));
+        assert_eq!(r[0].name_en, None);
+        assert!(r[0].is_key_station);
+        assert_eq!(r[0].situation_level, Some(2));
+        assert_eq!(r[0].wl_msl, Some(301.2));
+        assert_eq!(r[0].offset_msl, Some(300.0));
+    }
+}

rust/collector/src/main.rs

@@ -0,0 +1,220 @@
+//! ping-collector: the Ping River Monitor service.
+//!
+//! Collects RID hourly gauges, HII/ThaiWater rain + water level, RID dams
+//! and Open-Meteo rain into PostgreSQL; with FORECAST_IN_COLLECTOR=true runs
+//! the flood forecast (rust/forecast, models trained in Python) and the
+//! ntfy/Matrix notifications; with WEB_LISTEN serves the website and API.
+//! It replaced the Python app in 2026-10 (same tables, routes and messages);
+//! see docs/RUST_REWRITE.md.
+
+mod config;
+mod dams;
+mod db;
+#[cfg(test)]
+mod db_tests;
+mod envfile;
+mod forecasting;
+mod hii;
+mod notify;
+mod rain;
+mod rid;
+mod service;
+mod stations;
+mod util;
+mod web;
+
+use anyhow::{Result, bail};
+use clap::{Parser, Subcommand};
+use std::path::PathBuf;
+
+#[derive(Parser)]
+#[command(version, about)]
+struct Cli {
+    /// .env file to load (default: ./.env when present), read like python-dotenv;
+    /// variables already in the environment take precedence.
+    #[arg(long, global = true)]
+    env_file: Option<PathBuf>,
+    #[command(subcommand)]
+    command: Command,
+}
+
+#[derive(Subcommand)]
+enum Command {
+    /// Run the service (what systemd starts): collection, with
+    /// FORECAST_IN_COLLECTOR=true forecasts and notifications, with WEB_LISTEN
+    /// the website and API
+    Run,
+    /// Serve only the website and API (no collection), e.g. next to the
+    /// running service for a comparison
+    Serve {
+        /// Address to listen on [default: WEB_LISTEN, else 127.0.0.1:8001]
+        #[arg(long)]
+        listen: Option<String>,
+        /// Do not report requests to Umami (comparison runs)
+        #[arg(long)]
+        no_analytics: bool,
+    },
+    /// One cycle of every source, then exit
+    Once {
+        /// Fetch and compare with the database, write nothing
+        #[arg(long)]
+        dry_run: bool,
+    },
+    /// Re-fetch RID reports that have holes: a number of days back, or "all"
+    FillGaps { days: String },
+    /// Connect, create missing tables, print row counts and the newest RID reading
+    CheckDb,
+    /// Forecast from the database now and print it; stores nothing without --save
+    Forecast {
+        /// Print the /forecast JSON instead of a table
+        #[arg(long)]
+        json: bool,
+        /// Store the result (forecast_history and the API's forecast_current)
+        #[arg(long)]
+        save: bool,
+        /// Forecast as of this local time, e.g. 2026-10-18T14:23:05.5 (comparisons)
+        #[arg(long)]
+        now: Option<String>,
+        /// Rain series from a JSON file {"timestamps": [...], "values": [...]}
+        /// instead of a live Open-Meteo fetch (comparisons)
+        #[arg(long)]
+        rain_file: Option<PathBuf>,
+    },
+    /// Show the readings the notifier judges and the ntfy transitions due now
+    Notify {
+        /// Show what would be published; publish nothing, change no state
+        #[arg(long)]
+        dry_run: bool,
+    },
+}
+
+fn read_rain_file(path: &std::path::Path) -> Result<ping_forecast::rain::RainSeries> {
+    #[derive(serde::Deserialize)]
+    struct File {
+        timestamps: Vec<String>,
+        values: Vec<Option<f64>>,
+    }
+    let f: File = serde_json::from_str(&std::fs::read_to_string(path)?)?;
+    if f.timestamps.len() != f.values.len() {
+        bail!("{}: timestamps and values differ in length", path.display());
+    }
+    f.timestamps
+        .iter()
+        .zip(f.values)
+        .map(|(t, v)| Ok((parse_local(t)?, v.unwrap_or(f64::NAN))))
+        .collect()
+}
+
+fn parse_local(s: &str) -> Result<chrono::NaiveDateTime> {
+    chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S%.f")
+        .or_else(|_| chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M"))
+        .map_err(|e| anyhow::anyhow!("{s:?}: {e}"))
+}
+
+fn main() -> Result<()> {
+    let cli = Cli::parse();
+    // .env before the runtime starts: setting variables is only sound while
+    // the process is single-threaded.
+    match &cli.env_file {
+        Some(path) => {
+            envfile::load(path).map_err(|e| anyhow::anyhow!("{}: {e}", path.display()))?;
+        }
+        None => {
+            // Under systemd the variables come from EnvironmentFile=; an
+            // unreadable ./.env is not fatal there.
+            let path = std::path::Path::new(".env");
+            if path.exists()
+                && let Err(e) = envfile::load(path)
+            {
+                eprintln!("warning: .env not loaded: {e}");
+            }
+        }
+    }
+    let level = std::env::var("RUST_LOG")
+        .or_else(|_| std::env::var("LOG_LEVEL").map(|l| l.to_lowercase()))
+        .unwrap_or_else(|_| "info".into());
+    tracing_subscriber::fmt()
+        .with_env_filter(tracing_subscriber::EnvFilter::new(format!(
+            "warn,ping_collector={level},ping_forecast={level}"
+        )))
+        .with_target(false)
+        .init();
+    tokio::runtime::Builder::new_multi_thread()
+        .enable_all()
+        .build()?
+        .block_on(run(cli))
+}
+
+async fn run(cli: Cli) -> Result<()> {
+    let mut cfg = config::Config::from_env()?;
+    if let Command::Serve {
+        no_analytics: true, ..
+    } = cli.command
+    {
+        cfg.web.umami_track_api = false;
+    }
+    tracing::info!(
+        "ping-collector {} -> {}",
+        env!("CARGO_PKG_VERSION"),
+        cfg.redacted_database_url()
+    );
+    // the website answers concurrent requests; collection alone needs few
+    let serving = matches!(cli.command, Command::Serve { .. })
+        || (matches!(cli.command, Command::Run) && cfg.web.listen.is_some());
+    let pool = db::connect(&cfg.database_url, if serving { 12 } else { 4 }).await?;
+    let stations = stations::Stations::load(&cfg.station_config_path)?;
+    let collector = service::Collector::new(cfg, pool, stations)?;
+
+    match cli.command {
+        Command::Run => collector.run().await,
+        Command::Serve { listen, .. } => {
+            let listen = listen
+                .or_else(|| collector.cfg.web.listen.clone())
+                .unwrap_or_else(|| "127.0.0.1:8001".into());
+            collector.serve(listen).await
+        }
+        Command::Once { dry_run } => collector.once(dry_run).await,
+        Command::FillGaps { days } => {
+            let days = match days.as_str() {
+                "all" => None,
+                n => match n.parse::<u32>() {
+                    Ok(d) => Some(d),
+                    Err(_) => bail!("fill-gaps takes a number of days or \"all\", got {n:?}"),
+                },
+            };
+            db::ensure_schema(&collector.pool).await?;
+            let filled = rid::fill_gaps(
+                &collector.rid,
+                &collector.pool,
+                &collector.stations(),
+                days,
+                util::bangkok_now(),
+            )
+            .await?;
+            println!("{filled} readings re-fetched");
+            Ok(())
+        }
+        Command::Forecast {
+            json,
+            save,
+            now,
+            rain_file,
+        } => {
+            let now = now.as_deref().map(parse_local).transpose()?;
+            let rain = rain_file.as_deref().map(read_rain_file).transpose()?;
+            collector.forecast(json, save, now, rain).await
+        }
+        Command::Notify { dry_run } => collector.notify(dry_run).await,
+        Command::CheckDb => {
+            db::ensure_schema(&collector.pool).await?;
+            for (table, n) in db::table_counts(&collector.pool).await? {
+                println!("{table:<22} {n:>10} rows");
+            }
+            println!(
+                "newest RID reading: {:?}",
+                db::latest_rid_timestamp(&collector.pool).await?
+            );
+            Ok(())
+        }
+    }
+}

rust/collector/src/rain.rs

@@ -0,0 +1,128 @@
+//! Open-Meteo hourly precipitation at five upper-Ping catchment points
+//! (recent days + next 48 h) into `openmeteo_rain`. Port of the live path of
+//! src/ml/rain.py (`fetch_forecast` + `save_to_db`). Times are Asia/Bangkok.
+
+use anyhow::{Context, Result};
+use chrono::NaiveDateTime;
+use serde_json::Value;
+use std::collections::BTreeMap;
+
+/// (column, lat, lon), headwaters to city. Order = column order in the table.
+pub const CATCHMENT_POINTS: [(&str, f64, f64); 5] = [
+    ("chiang_dao", 19.37, 98.97),
+    ("mae_taeng", 19.12, 98.94),
+    ("mae_ngat", 19.17, 99.05),
+    ("mae_rim", 18.92, 98.92),
+    ("chiang_mai", 18.79, 99.00),
+];
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct RainRow {
+    pub timestamp: NaiveDateTime,
+    pub points: [Option<f64>; 5],
+    /// Mean of the points that have a value (pandas `mean(axis=1)`).
+    pub catchment_mean: Option<f64>,
+}
+
+/// A multi-coordinate request returns a list, one result per point.
+pub fn parse(payload: &Value) -> Vec<RainRow> {
+    let results: Vec<&Value> = match payload {
+        Value::Array(a) => a.iter().collect(),
+        other => vec![other],
+    };
+    let mut by_time: BTreeMap<NaiveDateTime, [Option<f64>; 5]> = BTreeMap::new();
+    for (i, result) in results.iter().take(CATCHMENT_POINTS.len()).enumerate() {
+        let hourly = result.get("hourly");
+        let times = hourly.and_then(|h| h.get("time")).and_then(Value::as_array);
+        let values = hourly
+            .and_then(|h| h.get("precipitation"))
+            .and_then(Value::as_array);
+        let (Some(times), Some(values)) = (times, values) else {
+            continue;
+        };
+        for (t, v) in times.iter().zip(values) {
+            let Some(ts) = t
+                .as_str()
+                .and_then(|s| NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M").ok())
+            else {
+                continue;
+            };
+            by_time.entry(ts).or_insert([None; 5])[i] = v.as_f64().filter(|x| x.is_finite());
+        }
+    }
+    by_time
+        .into_iter()
+        .map(|(timestamp, points)| RainRow {
+            timestamp,
+            points,
+            catchment_mean: ping_forecast::rain::catchment_mean(&points),
+        })
+        .collect()
+}
+
+pub struct OpenMeteoClient {
+    http: reqwest::Client,
+    url: String,
+}
+
+impl OpenMeteoClient {
+    pub fn new(http: reqwest::Client, url: String) -> Self {
+        Self { http, url }
+    }
+
+    pub async fn fetch_forecast(&self, past_days: u32, forecast_days: u32) -> Result<Vec<RainRow>> {
+        let join = |f: fn(&(&str, f64, f64)) -> f64| {
+            CATCHMENT_POINTS
+                .iter()
+                .map(|p| f(p).to_string())
+                .collect::<Vec<_>>()
+                .join(",")
+        };
+        let payload: Value = self
+            .http
+            .get(&self.url)
+            .query(&[
+                ("latitude", join(|p| p.1)),
+                ("longitude", join(|p| p.2)),
+                ("hourly", "precipitation".into()),
+                ("timezone", "Asia/Bangkok".into()),
+                ("past_days", past_days.to_string()),
+                ("forecast_days", forecast_days.to_string()),
+            ])
+            .timeout(std::time::Duration::from_secs(60))
+            .send()
+            .await
+            .context("Open-Meteo request failed")?
+            .error_for_status()?
+            .json()
+            .await
+            .context("Open-Meteo response is not JSON")?;
+        Ok(parse(&payload))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use serde_json::json;
+
+    #[test]
+    fn aligns_points_and_averages_present_values() {
+        let point = |vals: Value| json!({"hourly": {"time": ["2026-10-17T00:00", "2026-10-17T01:00"], "precipitation": vals}});
+        let payload = json!([
+            point(json!([1.0, null])),
+            point(json!([3.0, null])),
+            point(json!([null, null])),
+            point(json!([2.0, null])),
+            point(json!([0.0, 4.0])),
+        ]);
+        let rows = parse(&payload);
+        assert_eq!(rows.len(), 2);
+        assert_eq!(
+            rows[0].points,
+            [Some(1.0), Some(3.0), None, Some(2.0), Some(0.0)]
+        );
+        assert_eq!(rows[0].catchment_mean, Some(1.5));
+        assert_eq!(rows[1].catchment_mean, Some(4.0));
+    }
+}

rust/collector/src/rid.rs

@@ -0,0 +1,449 @@
+//! RID hourly water-level/discharge report (16 Ping gauges).
+//!
+//! `POST getGroupHourlyWaterLevelReportAllHL.ashx` with a Buddhist-era date
+//! returns 24 rows (`hourlytime` "1.00".."24.00"; hour 24 is 00:00 of the next
+//! day) with `wlvaluesN` / `qvaluesN` / `QPercentN` for report column N=1..16.
+//! Hours not published yet are null.
+//!
+//! The report often lags 1-2 h in the evening. Each cycle therefore stores
+//! every reading it gets (not only "the current hour"), and re-fetches
+//! yesterday's report while the store still has holes in it: hours 22-24 of a
+//! day are reachable only through that day's report.
+
+use crate::db;
+use crate::stations::Stations;
+use crate::util::{floor_hour, lenient_f64};
+use anyhow::{Context, Result};
+use chrono::{Datelike, Duration, NaiveDate, NaiveDateTime, NaiveTime};
+use serde_json::Value;
+use sqlx::PgPool;
+use std::collections::{BTreeSet, HashMap, HashSet};
+use std::future::Future;
+
+pub const N_STATIONS: u32 = 16;
+
+// Plausibility ranges (src/validators.py)
+const WATER_LEVEL_RANGE: (f64, f64) = (-10.0, 50.0);
+const DISCHARGE_RANGE: (f64, f64) = (0.0, 10_000.0);
+// % of channel capacity; Oct 2024 peaked at 226 %. Out of range -> NULL,
+// the reading itself is kept.
+const DISCHARGE_PERCENT_RANGE: (f64, f64) = (0.0, 500.0);
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct Reading {
+    pub timestamp: NaiveDateTime,
+    /// RID report column 1..16 (= stations.id in the database)
+    pub station_num: u32,
+    pub water_level: f64,
+    pub discharge: Option<f64>,
+    pub discharge_percent: Option<f64>,
+}
+
+fn within(x: f64, (lo, hi): (f64, f64)) -> bool {
+    lo <= x && x <= hi
+}
+
+/// Parse one report. Rows with an unparsable hour, a missing/implausible
+/// water level or an implausible discharge are dropped, as in Python.
+pub fn parse_report(report_date: NaiveDate, payload: &Value) -> Vec<Reading> {
+    let mut out = Vec::new();
+    let Some(rows) = payload.get("rows").and_then(Value::as_array) else {
+        return out;
+    };
+    for row in rows {
+        let hour = match row.get("hourlytime") {
+            Some(Value::String(s)) => s.trim().parse::<f64>().ok(),
+            Some(Value::Number(n)) => n.as_f64(),
+            _ => None,
+        };
+        let Some(hour) = hour
+            .map(|h| h.trunc() as i64)
+            .filter(|h| (1..=24).contains(h))
+        else {
+            continue;
+        };
+        let timestamp = report_date.and_time(NaiveTime::MIN) + Duration::hours(hour);
+
+        for n in 1..=N_STATIONS {
+            let Some(water_level) = lenient_f64(row.get(format!("wlvalues{n}"))) else {
+                continue;
+            };
+            if !within(water_level, WATER_LEVEL_RANGE) {
+                tracing::warn!(
+                    "RID {timestamp} column {n}: water level {water_level} out of range"
+                );
+                continue;
+            }
+            let discharge = lenient_f64(row.get(format!("qvalues{n}")));
+            if let Some(q) = discharge
+                && !within(q, DISCHARGE_RANGE)
+            {
+                tracing::warn!("RID {timestamp} column {n}: discharge {q} out of range");
+                continue;
+            }
+            let discharge_percent = discharge
+                .and(lenient_f64(row.get(format!("QPercent{n}"))))
+                .filter(|p| within(*p, DISCHARGE_PERCENT_RANGE));
+            out.push(Reading {
+                timestamp,
+                station_num: n,
+                water_level,
+                discharge,
+                discharge_percent,
+            });
+        }
+    }
+    out
+}
+
+/// Anything that can produce a parsed RID report (HTTP in production,
+/// canned rows in tests).
+pub trait RidSource {
+    fn fetch_report(&self, date: NaiveDate) -> impl Future<Output = Result<Vec<Reading>>> + Send;
+}
+
+pub struct RidClient {
+    http: reqwest::Client,
+    url: String,
+}
+
+impl RidClient {
+    pub fn new(http: reqwest::Client, url: String) -> Self {
+        Self { http, url }
+    }
+}
+
+impl RidSource for RidClient {
+    async fn fetch_report(&self, date: NaiveDate) -> Result<Vec<Reading>> {
+        let thai_date = format!(
+            "{:02}/{:02}/{}",
+            date.day(),
+            date.month(),
+            date.year() + 543
+        );
+        let nd = chrono::Utc::now().timestamp_millis().to_string();
+        let form = [
+            ("DW[UtokID]", "1"),
+            ("DW[BasinID]", "6"),
+            ("DW[TimeCurrent]", thai_date.as_str()),
+            ("_search", "false"),
+            ("nd", nd.as_str()),
+            ("rows", "100"),
+            ("page", "1"),
+            ("sidx", "indexhourly"),
+            ("sord", "asc"),
+        ];
+        let payload: Value = self
+            .http
+            .post(&self.url)
+            .header("X-Requested-With", "XMLHttpRequest")
+            .header("Accept", "application/json, text/javascript, */*; q=0.01")
+            .form(&form)
+            .timeout(std::time::Duration::from_secs(30))
+            .send()
+            .await
+            .context("RID request failed")?
+            .error_for_status()?
+            .json()
+            .await
+            .context("RID response is not JSON")?;
+        Ok(parse_report(date, &payload))
+    }
+}
+
+/// Report dates whose re-fetch would fill holes in [start, end].
+///
+/// Hours 1-23 of day D come from D's report, hour 0 from the previous day's
+/// report ("hour 24"). For today only hours already passed are expected.
+pub fn gap_fetch_dates(
+    start: NaiveDate,
+    end: NaiveDate,
+    now: NaiveDateTime,
+    recorded: &HashMap<NaiveDate, HashSet<u32>>,
+) -> BTreeSet<NaiveDate> {
+    let mut dates = BTreeSet::new();
+    let empty = HashSet::new();
+    let mut day = start;
+    while day <= end {
+        let expected: Vec<u32> = if day == now.date() {
+            (0..chrono::Timelike::hour(&now)).collect()
+        } else {
+            (0..24).collect()
+        };
+        let have = recorded.get(&day).unwrap_or(&empty);
+        let missing: Vec<u32> = expected.into_iter().filter(|h| !have.contains(h)).collect();
+        if missing.iter().any(|&h| h >= 1) {
+            dates.insert(day);
+        }
+        if missing.contains(&0) && day > start {
+            dates.insert(day - Duration::days(1));
+        }
+        day += Duration::days(1);
+    }
+    dates
+}
+
+/// Reports to fetch in one cycle: the one holding the current hour (today's,
+/// or before 01:00 yesterday's) plus yesterday's while it has holes.
+pub async fn cycle_dates(pool: &PgPool, now: NaiveDateTime) -> Vec<NaiveDate> {
+    let today = now.date();
+    let yesterday = today - Duration::days(1);
+    let mut dates = BTreeSet::from([if chrono::Timelike::hour(&now) >= 1 {
+        today
+    } else {
+        yesterday
+    }]);
+    match db::recorded_hours_by_day(pool, yesterday, today).await {
+        Ok(recorded) => dates.extend(gap_fetch_dates(yesterday, today, now, &recorded)),
+        Err(e) => tracing::warn!("gap detection failed, fetching the current report only: {e:#}"),
+    }
+    dates.into_iter().collect()
+}
+
+#[derive(Clone, Debug, Default)]
+pub struct CycleOutcome {
+    pub dates: Vec<NaiveDate>,
+    pub fetched: usize,
+    pub saved: usize,
+    pub latest: Option<NaiveDateTime>,
+    /// The current hour's reading is in: poll again next hour.
+    pub current: bool,
+    pub error: Option<String>,
+}
+
+pub async fn fetch_dates(
+    src: &impl RidSource,
+    dates: &[NaiveDate],
+) -> (Vec<Reading>, Option<String>) {
+    let mut readings = Vec::new();
+    let mut error = None;
+    for &d in dates {
+        match src.fetch_report(d).await {
+            Ok(r) => readings.extend(r),
+            Err(e) => {
+                tracing::error!("RID report {d}: {e:#}");
+                error = Some(format!("{e:#}"));
+            }
+        }
+    }
+    (readings, error)
+}
+
+/// One RID poll: fetch, store everything, report whether the feed is current.
+pub async fn run_cycle(
+    src: &impl RidSource,
+    pool: &PgPool,
+    stations: &Stations,
+    now: NaiveDateTime,
+) -> CycleOutcome {
+    let mut out = CycleOutcome {
+        dates: cycle_dates(pool, now).await,
+        ..Default::default()
+    };
+    let (readings, fetch_error) = fetch_dates(src, &out.dates).await;
+    out.fetched = readings.len();
+    out.error = fetch_error;
+    if readings.is_empty() {
+        out.error.get_or_insert_with(|| "no data fetched".into());
+        tracing::warn!("RID: no data fetched for {:?}", out.dates);
+        return out;
+    }
+    if let Err(e) = db::save_rid(pool, &readings, stations).await {
+        tracing::error!("RID save failed: {e:#}");
+        out.error = Some(format!("save failed: {e:#}"));
+        return out;
+    }
+    out.saved = readings.len();
+    let latest = readings
+        .iter()
+        .map(|r| r.timestamp)
+        .max()
+        .expect("non-empty");
+    let current_hour = floor_hour(now);
+    out.latest = Some(latest);
+    out.current = latest >= current_hour;
+    if out.current {
+        tracing::info!("RID: {} readings saved, latest {latest}", out.saved);
+    } else {
+        tracing::warn!(
+            "RID: {} readings saved up to {latest}; {current_hour} not published yet ({} h behind), will retry",
+            out.saved,
+            (current_hour - latest).num_hours()
+        );
+    }
+    out
+}
+
+/// Re-fetch every report with holes in the last `days` days (None = whole
+/// history). Returns readings re-saved.
+pub async fn fill_gaps(
+    src: &impl RidSource,
+    pool: &PgPool,
+    stations: &Stations,
+    days: Option<u32>,
+    now: NaiveDateTime,
+) -> Result<usize> {
+    let end = now.date();
+    let start = match days {
+        Some(d) => end - Duration::days(d as i64),
+        None => match db::earliest_rid_timestamp(pool).await? {
+            Some(t) => t.date(),
+            None => {
+                tracing::warn!("database is empty; nothing to gap-fill");
+                return Ok(0);
+            }
+        },
+    };
+    let recorded = db::recorded_hours_by_day(pool, start, end).await?;
+    let dates = gap_fetch_dates(start, end, now, &recorded);
+    tracing::info!(
+        "gap fill {start}..{end}: {} report(s) to re-fetch",
+        dates.len()
+    );
+    let mut filled = 0;
+    let mut empty = 0;
+    for (i, d) in dates.iter().enumerate() {
+        if i > 0 {
+            tokio::time::sleep(std::time::Duration::from_secs(1)).await; // be polite
+        }
+        match src.fetch_report(*d).await {
+            Ok(readings) if !readings.is_empty() => {
+                db::save_rid(pool, &readings, stations).await?;
+                filled += readings.len();
+            }
+            Ok(_) => {
+                empty += 1;
+                tracing::debug!("gap fill {d}: RID has no data");
+            }
+            Err(e) => tracing::warn!("gap fill {d}: {e:#}"),
+        }
+        if (i + 1) % 50 == 0 && i + 1 < dates.len() {
+            tracing::info!(
+                "gap fill: {}/{} reports, at {d}, {filled} readings re-saved",
+                i + 1,
+                dates.len()
+            );
+        }
+    }
+    tracing::info!(
+        "gap fill done: {filled} readings re-saved from {} report(s); {empty} had no data at RID",
+        dates.len() - empty
+    );
+    Ok(filled)
+}
+
+/// Next RID poll: `offset` minutes past the next hour when current, else in
+/// `retry` minutes (never later than that hourly poll).
+pub fn next_poll_time(
+    now: NaiveDateTime,
+    current: bool,
+    offset_min: u32,
+    retry_min: u32,
+) -> NaiveDateTime {
+    let on_schedule = floor_hour(now) + Duration::hours(1) + Duration::minutes(offset_min as i64);
+    if current {
+        on_schedule
+    } else {
+        on_schedule.min(now + Duration::minutes(retry_min as i64))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use serde_json::json;
+
+    fn d(y: i32, m: u32, day: u32) -> NaiveDate {
+        NaiveDate::from_ymd_opt(y, m, day).unwrap()
+    }
+    fn t(y: i32, m: u32, day: u32, h: u32, min: u32) -> NaiveDateTime {
+        d(y, m, day).and_hms_opt(h, min, 0).unwrap()
+    }
+
+    #[test]
+    fn parses_levels_discharge_and_percent() {
+        let p = json!({"rows": [{"hourlytime": "9.00", "wlvalues1": "3.50", "qvalues1": "120.5", "QPercent1": 45.2}]});
+        let r = parse_report(d(2026, 7, 22), &p);
+        assert_eq!(
+            r,
+            vec![Reading {
+                timestamp: t(2026, 7, 22, 9, 0),
+                station_num: 1,
+                water_level: 3.5,
+                discharge: Some(120.5),
+                discharge_percent: Some(45.2),
+            }]
+        );
+    }
+
+    #[test]
+    fn hour_24_is_next_day_midnight_and_junk_hours_are_skipped() {
+        let p = json!({"rows": [
+            {"hourlytime": "24.00", "wlvalues1": 3.0},
+            {"hourlytime": "25.00", "wlvalues1": 3.0},
+            {"hourlytime": "0.00", "wlvalues1": 3.0},
+            {"hourlytime": "", "wlvalues1": 3.0}
+        ]});
+        let r = parse_report(d(2026, 7, 22), &p);
+        assert_eq!(r.len(), 1);
+        assert_eq!(r[0].timestamp, t(2026, 7, 23, 0, 0));
+    }
+
+    #[test]
+    fn missing_or_junk_values() {
+        let p = json!({"rows": [{"hourlytime": "10.00",
+            "wlvalues1": null,
+            "wlvalues2": 2.5, "qvalues2": "***", "QPercent2": 10,
+            "wlvalues3": 80.0,
+            "wlvalues4": 1.0, "qvalues4": "20", "QPercent4": 900,
+            "wlvalues5": 1.0, "qvalues5": "-5"}]});
+        let r = parse_report(d(2026, 7, 22), &p);
+        let by: HashMap<u32, &Reading> = r.iter().map(|x| (x.station_num, x)).collect();
+        assert!(!by.contains_key(&1)); // no level
+        assert_eq!(by[&2].discharge, None); // "***"
+        assert_eq!(by[&2].discharge_percent, None); // percent needs a discharge
+        assert!(!by.contains_key(&3)); // level out of range
+        assert_eq!(by[&4].discharge_percent, None); // 900 % nulled, reading kept
+        assert!(!by.contains_key(&5)); // negative discharge drops the reading
+    }
+
+    #[test]
+    fn missing_rows_key_is_empty() {
+        assert!(parse_report(d(2026, 7, 22), &json!({"unexpected": true})).is_empty());
+    }
+
+    fn hours(spec: &[(NaiveDate, std::ops::Range<u32>)]) -> HashMap<NaiveDate, HashSet<u32>> {
+        spec.iter()
+            .map(|(day, r)| (*day, r.clone().collect()))
+            .collect()
+    }
+
+    #[test]
+    fn gap_dates_late_evening_and_midnight() {
+        let today = d(2026, 10, 17);
+        let yesterday = d(2026, 10, 16);
+        // 02:10, yesterday has 0..22, today only hour 1: yesterday's report holds
+        // 22:00, 23:00 and today's 00:00 ("hour 24")
+        let rec = hours(&[(yesterday, 0..22), (today, 1..2)]);
+        let got = gap_fetch_dates(yesterday, today, t(2026, 10, 17, 2, 10), &rec);
+        assert_eq!(got.into_iter().collect::<Vec<_>>(), vec![yesterday]);
+        // today's 01:00 missing as well -> today's report too
+        let rec = hours(&[(yesterday, 0..22)]);
+        let got = gap_fetch_dates(yesterday, today, t(2026, 10, 17, 2, 10), &rec);
+        assert_eq!(got.into_iter().collect::<Vec<_>>(), vec![yesterday, today]);
+        // complete yesterday, today complete up to now -> nothing
+        let rec = hours(&[(yesterday, 0..24), (today, 0..15)]);
+        assert!(gap_fetch_dates(yesterday, today, t(2026, 10, 17, 15, 5), &rec).is_empty());
+    }
+
+    #[test]
+    fn poll_schedule() {
+        let now = t(2026, 10, 17, 21, 6);
+        assert_eq!(next_poll_time(now, true, 5, 10), t(2026, 10, 17, 22, 5));
+        assert_eq!(next_poll_time(now, false, 5, 10), t(2026, 10, 17, 21, 16));
+        assert_eq!(
+            next_poll_time(t(2026, 10, 17, 21, 58), false, 5, 10),
+            t(2026, 10, 17, 22, 5)
+        );
+    }
+}

rust/collector/src/service.rs

@@ -0,0 +1,489 @@
+//! The long-running collection loop and the one-shot commands.
+
+use crate::config::{Config, USER_AGENT};
+use crate::dams::DamsClient;
+use crate::db;
+use crate::forecasting::{self, Forecaster, RunInfo};
+use crate::hii::HiiClient;
+use crate::notify;
+use crate::rain::OpenMeteoClient;
+use crate::rid::{self, CycleOutcome, RidClient};
+use crate::stations::Stations;
+use crate::util::{bangkok_now, floor_hour};
+use anyhow::{Result, bail};
+use chrono::{Duration, NaiveDateTime};
+use ping_forecast::ModelStore;
+use ping_forecast::rain::RainSeries;
+use serde_json::json;
+use sqlx::PgPool;
+use std::sync::{Arc, RwLock};
+
+pub struct Collector {
+    pub cfg: Config,
+    pub pool: PgPool,
+    /// Shared with the website (the station CRUD API edits it).
+    pub stations: Arc<RwLock<Stations>>,
+    /// The last status written, shared with /scraping/status.
+    pub status: Arc<RwLock<serde_json::Value>>,
+    /// POST /scrape/trigger: poll now.
+    pub trigger: Arc<tokio::sync::Notify>,
+    pub rid: RidClient,
+    hii: HiiClient,
+    dams: DamsClient,
+    rain: OpenMeteoClient,
+    http: reqwest::Client,
+}
+
+#[derive(Default)]
+struct Stats {
+    total_runs: u64,
+    successful_runs: u64,
+    failed_runs: u64,
+    last_run: Option<NaiveDateTime>,
+    next_run: Option<NaiveDateTime>,
+    feed_current: Option<bool>,
+    latest_reading: Option<NaiveDateTime>,
+    readings_saved: usize,
+    forecast: Option<RunInfo>,
+}
+
+fn iso(t: Option<NaiveDateTime>) -> serde_json::Value {
+    t.map_or(serde_json::Value::Null, |t| {
+        json!(t.format("%Y-%m-%dT%H:%M:%S").to_string())
+    })
+}
+
+impl Collector {
+    pub fn new(cfg: Config, pool: PgPool, stations: Stations) -> Result<Self> {
+        let http = reqwest::Client::builder()
+            .user_agent(USER_AGENT)
+            .connect_timeout(std::time::Duration::from_secs(20))
+            .build()?;
+        Ok(Self {
+            rid: RidClient::new(http.clone(), cfg.rid_api_url.clone()),
+            hii: HiiClient::new(http.clone(), cfg.hii_api_base.clone()),
+            dams: DamsClient::new(http.clone(), cfg.dams_url.clone()),
+            rain: OpenMeteoClient::new(http.clone(), cfg.openmeteo_url.clone()),
+            http,
+            cfg,
+            pool,
+            stations: Arc::new(RwLock::new(stations)),
+            status: Arc::new(RwLock::new(json!({"is_running": false}))),
+            trigger: Arc::new(tokio::sync::Notify::new()),
+        })
+    }
+
+    /// The station mapping as of now (the API may edit it between cycles).
+    pub fn stations(&self) -> Stations {
+        self.stations.read().unwrap().clone()
+    }
+
+    /// The website/API state over this collector's pool, stations and status.
+    pub fn web_state(&self, in_process: bool) -> crate::web::Shared {
+        crate::web::AppState::new(
+            self.cfg.clone(),
+            self.pool.clone(),
+            self.http.clone(),
+            self.stations.clone(),
+            in_process.then(|| self.status.clone()),
+            in_process.then(|| self.trigger.clone()),
+        )
+    }
+
+    /// `ping-collector serve`: the website and API only; /scraping/status
+    /// reads the status file the collector service writes.
+    pub async fn serve(&self, listen: String) -> Result<()> {
+        db::ensure_schema(&self.pool).await?;
+        crate::web::serve(self.web_state(false), listen, shutdown_signal()).await
+    }
+
+    /// HII rain + water level, RID dams (today and yesterday), Open-Meteo.
+    /// Each source is failure-isolated.
+    pub async fn collect_hourly(&self, now: NaiveDateTime) {
+        if self.cfg.hii_enabled {
+            match self.hii.fetch_rain(self.cfg.hii_basin_code).await {
+                Ok(r) => match db::save_hii_rain(&self.pool, &r, now).await {
+                    Ok(n) => tracing::info!("HII rainfall: {n} rows saved"),
+                    Err(e) => tracing::error!("HII rainfall save failed: {e:#}"),
+                },
+                Err(e) => tracing::error!("HII rainfall fetch failed: {e:#}"),
+            }
+            match self.hii.fetch_waterlevel(self.cfg.hii_basin_code).await {
+                Ok(r) => match db::save_hii_waterlevel(&self.pool, &r, now).await {
+                    Ok(n) => tracing::info!("HII water level: {n} rows saved"),
+                    Err(e) => tracing::error!("HII water level save failed: {e:#}"),
+                },
+                Err(e) => tracing::error!("HII water level fetch failed: {e:#}"),
+            }
+        }
+        if self.cfg.dams_enabled {
+            for date in [None, Some(now.date() - Duration::days(1))] {
+                match self.dams.fetch_day(date).await {
+                    Ok(r) => match db::save_dams(&self.pool, &r, now).await {
+                        Ok(n) => tracing::info!("RID dams {date:?}: {n} dam-day rows saved"),
+                        Err(e) => tracing::error!("RID dams save failed: {e:#}"),
+                    },
+                    Err(e) => tracing::error!("RID dams fetch {date:?} failed: {e:#}"),
+                }
+            }
+        }
+        if self.cfg.rain_enabled {
+            match self.rain.fetch_forecast(5, 2).await {
+                Ok(r) => match db::save_rain(&self.pool, &r).await {
+                    Ok(n) => tracing::info!("Open-Meteo rain: {n} hourly rows upserted"),
+                    Err(e) => tracing::error!("Open-Meteo save failed: {e:#}"),
+                },
+                Err(e) => tracing::error!("Open-Meteo fetch failed: {e:#}"),
+            }
+        }
+    }
+
+    fn write_status(&self, stats: &Stats, running: bool) {
+        let body = json!({
+            "is_running": running,
+            "total_runs": stats.total_runs,
+            "successful_runs": stats.successful_runs,
+            "failed_runs": stats.failed_runs,
+            "last_run": iso(stats.last_run),
+            "next_run": iso(stats.next_run),
+            "feed_current": stats.feed_current,
+            "latest_reading": iso(stats.latest_reading),
+            "readings_saved": stats.readings_saved,
+            "collector": concat!("ping-collector ", env!("CARGO_PKG_VERSION")),
+            "forecast": stats.forecast.as_ref().map(|f| json!({
+                "last_run": iso(Some(f.at)),
+                "rows": f.rows,
+                "model_rows": f.model_rows,
+                "models": f.models,
+                "notifications_sent": f.notified,
+            })),
+        });
+        *self.status.write().unwrap() = body.clone();
+        let Some(path) = &self.cfg.status_file else {
+            return;
+        };
+        let tmp = path.with_extension("json.tmp");
+        let result =
+            std::fs::write(&tmp, body.to_string()).and_then(|_| std::fs::rename(&tmp, path));
+        if let Err(e) = result {
+            tracing::warn!("could not write status file {}: {e}", path.display());
+        }
+    }
+
+    /// The service: startup gap fill, then RID polls on the hour (+offset)
+    /// with retries while RID is behind, and the hourly sources once per
+    /// clock hour. Returns when a shutdown signal arrives.
+    pub async fn run(&self) -> Result<()> {
+        db::ensure_schema(&self.pool).await?;
+        let shutdown = shutdown_signal();
+        tokio::pin!(shutdown);
+        // the website, when WEB_LISTEN is set: stopped with the collector
+        let (stop_web, web_stopped) = tokio::sync::watch::channel(false);
+        let web = self.cfg.web.listen.clone().map(|listen| {
+            let state = self.web_state(true);
+            let mut rx = web_stopped.clone();
+            tokio::spawn(async move {
+                let stop = async move {
+                    let _ = rx.wait_for(|stop| *stop).await;
+                };
+                if let Err(e) = crate::web::serve(state, listen, stop).await {
+                    tracing::error!("website stopped: {e:#}");
+                }
+            })
+        });
+        let result = self.collect_loop(&mut shutdown).await;
+        let _ = stop_web.send(true);
+        if let Some(handle) = web {
+            let _ = tokio::time::timeout(std::time::Duration::from_secs(10), handle).await;
+        }
+        result
+    }
+
+    async fn collect_loop(
+        &self,
+        shutdown: &mut std::pin::Pin<&mut impl std::future::Future<Output = ()>>,
+    ) -> Result<()> {
+        if self.cfg.gap_fill_days_on_start > 0 {
+            let days = Some(self.cfg.gap_fill_days_on_start);
+            let stations = self.stations();
+            tokio::select! {
+                r = rid::fill_gaps(&self.rid, &self.pool, &stations, days, bangkok_now()) => {
+                    if let Err(e) = r { tracing::error!("startup gap fill failed: {e:#}") }
+                }
+                _ = &mut *shutdown => return Ok(()),
+            }
+        }
+
+        let mut stats = Stats::default();
+        let mut last_hourly_slot: Option<NaiveDateTime> = None;
+        let mut forecaster = self.cfg.forecast_enabled.then(|| {
+            tracing::info!(
+                "FORECAST_IN_COLLECTOR: forecasting from {} and notifying here",
+                self.cfg.models_dir.display()
+            );
+            Forecaster::new(&self.cfg, self.http.clone())
+        });
+        loop {
+            let start = bangkok_now();
+            self.write_status(&stats, true);
+            let outcome: CycleOutcome =
+                rid::run_cycle(&self.rid, &self.pool, &self.stations(), start).await;
+            stats.total_runs += 1;
+            stats.last_run = Some(start);
+            if outcome.current {
+                stats.successful_runs += 1;
+            } else {
+                stats.failed_runs += 1;
+            }
+            stats.feed_current = Some(outcome.current);
+            stats.readings_saved = outcome.saved;
+            if outcome.latest.is_some() {
+                stats.latest_reading = outcome.latest.max(stats.latest_reading);
+            }
+
+            let slot = floor_hour(bangkok_now());
+            if last_hourly_slot != Some(slot) {
+                last_hourly_slot = Some(slot);
+                self.collect_hourly(bangkok_now()).await;
+            }
+            if let Some(f) = forecaster.as_mut()
+                && let Some(info) = f.maybe_run(&self.pool, &self.rain, bangkok_now()).await
+            {
+                stats.forecast = Some(info);
+            }
+
+            let now = bangkok_now();
+            let next = rid::next_poll_time(
+                now,
+                outcome.current,
+                self.cfg.poll_offset_minutes,
+                self.cfg.retry_minutes,
+            );
+            stats.next_run = Some(next);
+            self.write_status(&stats, false);
+            tracing::info!(
+                "next RID poll at {}{}",
+                next.format("%H:%M"),
+                if outcome.current {
+                    ""
+                } else {
+                    " (current hour not published yet)"
+                }
+            );
+            let wait = (next - now)
+                .to_std()
+                .unwrap_or_default()
+                .max(std::time::Duration::from_secs(30));
+            tokio::select! {
+                _ = tokio::time::sleep(wait) => {}
+                _ = self.trigger.notified() => tracing::info!("collection triggered via the API"),
+                _ = &mut *shutdown => {
+                    tracing::info!("shutdown signal received");
+                    return Ok(());
+                }
+            }
+        }
+    }
+
+    /// One cycle of every source; with `dry_run` nothing is written and the
+    /// RID fetch is compared against what the database already holds.
+    pub async fn once(&self, dry_run: bool) -> Result<()> {
+        let now = bangkok_now();
+        if !dry_run {
+            db::ensure_schema(&self.pool).await?;
+            let o = rid::run_cycle(&self.rid, &self.pool, &self.stations(), now).await;
+            println!(
+                "RID: reports {:?}, {} fetched, {} saved, latest {:?}, current hour in: {}{}",
+                o.dates,
+                o.fetched,
+                o.saved,
+                o.latest,
+                o.current,
+                o.error.map(|e| format!(", error: {e}")).unwrap_or_default()
+            );
+            self.collect_hourly(now).await;
+            return Ok(());
+        }
+
+        let dates = rid::cycle_dates(&self.pool, now).await;
+        let (readings, error) = rid::fetch_dates(&self.rid, &dates).await;
+        let stored = db::count_stored_rid(&self.pool, &readings).await?;
+        let latest = readings.iter().map(|r| r.timestamp).max();
+        let db_latest = db::latest_rid_timestamp(&self.pool).await?;
+        println!(
+            "RID (dry run): reports {dates:?}: {} readings fetched, {stored} already stored, {} would be added; latest fetched {latest:?}, latest stored {db_latest:?}{}",
+            readings.len(),
+            readings.len() as i64 - stored,
+            error.map(|e| format!(", error: {e}")).unwrap_or_default()
+        );
+        if self.cfg.hii_enabled {
+            match self.hii.fetch_rain(self.cfg.hii_basin_code).await {
+                Ok(r) => println!("HII rainfall (dry run): {} Ping-basin stations", r.len()),
+                Err(e) => println!("HII rainfall (dry run): {e:#}"),
+            }
+            match self.hii.fetch_waterlevel(self.cfg.hii_basin_code).await {
+                Ok(r) => println!("HII water level (dry run): {} Ping-basin stations", r.len()),
+                Err(e) => println!("HII water level (dry run): {e:#}"),
+            }
+        }
+        if self.cfg.dams_enabled {
+            match self.dams.fetch_day(None).await {
+                Ok(r) => println!("RID dams (dry run): {} dams today", r.len()),
+                Err(e) => println!("RID dams (dry run): {e:#}"),
+            }
+        }
+        if self.cfg.rain_enabled {
+            match self.rain.fetch_forecast(5, 2).await {
+                Ok(r) => println!("Open-Meteo (dry run): {} hourly rows", r.len()),
+                Err(e) => println!("Open-Meteo (dry run): {e:#}"),
+            }
+        }
+        Ok(())
+    }
+}
+
+impl Collector {
+    /// `ping-collector forecast`: compute from the database now (or at
+    /// `now`), print a table or the /forecast JSON, store only with `save`.
+    pub async fn forecast(
+        &self,
+        json_out: bool,
+        save: bool,
+        now: Option<NaiveDateTime>,
+        rain: Option<RainSeries>,
+    ) -> Result<()> {
+        let mut store = ModelStore::new(&self.cfg.models_dir);
+        let models = store.refresh()?;
+        if models.is_empty() {
+            bail!(
+                "no exported models in {} (run `python -m src.ml.export`)",
+                self.cfg.models_dir.display()
+            );
+        }
+        let now = now.unwrap_or_else(bangkok_now);
+        let snap = forecasting::compute(&self.pool, &self.rain, &models, now, rain).await?;
+        if json_out {
+            println!("{}", serde_json::to_string(&snap.forecasts)?);
+        } else {
+            println!(
+                "{:<6} {:>3}  {:<9} {:<16} {:>7} {:>7} {:>9} {:>8}",
+                "code", "h", "source", "as_of", "level", "peak", "p_warning", "p_danger"
+            );
+            for f in &snap.forecasts {
+                println!(
+                    "{:<6} {:>3}  {:<9} {:<16} {:>7.2} {:>7.2} {:>9.3} {:>8.3}",
+                    f.station_code,
+                    f.horizon_hours,
+                    f.source,
+                    f.as_of.format("%Y-%m-%d %H:%M"),
+                    f.current_level,
+                    f.predicted_max_level,
+                    f.p_warning,
+                    f.p_danger
+                );
+            }
+            eprintln!(
+                "{} models from {}; {}",
+                models.len(),
+                self.cfg.models_dir.display(),
+                forecasting::summarize(&snap)
+            );
+        }
+        if save {
+            db::ensure_schema(&self.pool).await?;
+            let n = db::save_forecast_history(&self.pool, &snap.forecasts).await?;
+            db::save_forecast_current(&self.pool, now, &snap.forecasts).await?;
+            eprintln!("stored: {n} forecast_history rows, forecast_current");
+        }
+        Ok(())
+    }
+
+    /// `ping-collector notify`: what would be (or, without `dry_run`, is)
+    /// published right now against the stored notification state.
+    pub async fn notify(&self, dry_run: bool) -> Result<()> {
+        if !dry_run && !self.cfg.forecast_enabled {
+            bail!(
+                "notifications are off while FORECAST_IN_COLLECTOR is not true; \
+                 use --dry-run"
+            );
+        }
+        let Some(notifier) = forecasting::notifier(&self.cfg, self.http.clone()) else {
+            bail!(
+                "notifications are off: neither NTFY_SERVER nor MATRIX_ACCESS_TOKEN/MATRIX_ROOM_ID is set"
+            );
+        };
+        let mut store = ModelStore::new(&self.cfg.models_dir);
+        let models = store.refresh()?;
+        let now = bangkok_now();
+        let snap = forecasting::compute(&self.pool, &self.rain, &models, now, None).await?;
+        let readings =
+            notify::readings_with_hii(&db::latest_rid_readings(&self.pool).await?, &snap.fills);
+        for r in &readings {
+            println!(
+                "{:<6} {} {:>7} {:?}{}",
+                r.station_code,
+                r.timestamp.format("%Y-%m-%d %H:%M"),
+                r.water_level.map_or("-".into(), |l| format!("{l:.2}")),
+                r.source,
+                r.discharge_percent
+                    .map_or(String::new(), |p| format!(" {p:.0}%"))
+            );
+        }
+        db::ensure_schema(&self.pool).await?;
+        if dry_run {
+            let states = db::notification_states(&self.pool).await?;
+            let mut plan = vec![];
+            if notifier.ntfy.is_some() {
+                plan.extend(
+                    notify::evaluate(&readings, &snap.forecasts, &states, &notifier.settings, now)
+                        .into_iter()
+                        .map(|p| (p.notification.topic.clone(), p)),
+                );
+            }
+            if notifier.matrix.is_some() {
+                let mstates = notifier
+                    .matrix_states(&self.pool, &states, now, false)
+                    .await?;
+                plan.extend(
+                    notifier
+                        .matrix_plan(&readings, &snap.forecasts, &mstates, now)
+                        .into_iter()
+                        .map(|p| ("matrix".to_string(), p)),
+                );
+            }
+            if plan.is_empty() {
+                println!("no transitions: nothing would be published");
+            }
+            for (to, p) in plan {
+                println!(
+                    "would publish {to}: {} -- {}",
+                    p.notification.title, p.notification.message
+                );
+            }
+        } else {
+            for n in notifier
+                .run(&self.pool, &readings, &snap.forecasts, now)
+                .await?
+            {
+                println!("published {}: {}", n.topic, n.title);
+            }
+        }
+        Ok(())
+    }
+}
+
+async fn shutdown_signal() {
+    #[cfg(unix)]
+    {
+        use tokio::signal::unix::{SignalKind, signal};
+        let mut term = signal(SignalKind::terminate()).expect("SIGTERM handler");
+        tokio::select! {
+            _ = tokio::signal::ctrl_c() => {}
+            _ = term.recv() => {}
+        }
+    }
+    #[cfg(not(unix))]
+    {
+        let _ = tokio::signal::ctrl_c().await;
+    }
+}

rust/collector/src/stations.rs

@@ -0,0 +1,148 @@
+//! RID station mapping: report column number (1..16) -> station metadata.
+//! Same precedence as the Python scraper: the runtime file
+//! (STATION_CONFIG_PATH, edited through the station CRUD API) if it exists,
+//! else the defaults bundled in src/data/stations.json.
+
+use anyhow::{Context, Result};
+use serde::{Deserialize, Serialize};
+use std::collections::BTreeMap;
+use std::path::Path;
+
+const BUNDLED: &str = include_str!("../../../src/data/stations.json");
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct StationInfo {
+    pub code: String,
+    pub thai_name: String,
+    pub english_name: String,
+    pub latitude: Option<f64>,
+    pub longitude: Option<f64>,
+    pub geohash: Option<String>,
+}
+
+#[derive(Clone, Debug)]
+pub struct Stations(BTreeMap<u32, StationInfo>);
+
+impl Stations {
+    pub fn load(runtime_path: &Path) -> Result<Self> {
+        if runtime_path.exists() {
+            let text = std::fs::read_to_string(runtime_path)
+                .with_context(|| format!("reading {}", runtime_path.display()))?;
+            let stations = Self::parse(&text)
+                .with_context(|| format!("parsing {}", runtime_path.display()))?;
+            tracing::info!(
+                "loaded {} stations from {}",
+                stations.0.len(),
+                runtime_path.display()
+            );
+            return Ok(stations);
+        }
+        let stations = Self::bundled();
+        tracing::info!("loaded {} bundled default stations", stations.0.len());
+        Ok(stations)
+    }
+
+    pub fn bundled() -> Self {
+        Self::parse(BUNDLED).expect("bundled stations.json is valid")
+    }
+
+    fn parse(text: &str) -> Result<Self> {
+        let raw: BTreeMap<String, StationInfo> = serde_json::from_str(text)?;
+        let map = raw
+            .into_iter()
+            .map(|(k, v)| Ok((k.trim().parse::<u32>()?, v)))
+            .collect::<Result<BTreeMap<_, _>>>()?;
+        Ok(Stations(map))
+    }
+
+    pub fn iter(&self) -> impl Iterator<Item = (&u32, &StationInfo)> {
+        self.0.iter()
+    }
+
+    pub fn info(&self, n: u32) -> Option<&StationInfo> {
+        self.0.get(&n)
+    }
+
+    pub fn info_mut(&mut self, n: u32) -> Option<&mut StationInfo> {
+        self.0.get_mut(&n)
+    }
+
+    /// The id a new station gets (the station CRUD API): one past the highest.
+    pub fn next_id(&self) -> u32 {
+        self.0.keys().max().map_or(1, |m| m + 1)
+    }
+
+    pub fn insert(&mut self, n: u32, info: StationInfo) {
+        self.0.insert(n, info);
+    }
+
+    pub fn remove(&mut self, n: u32) -> Option<StationInfo> {
+        self.0.remove(&n)
+    }
+
+    /// Persist to the runtime file as the Python scraper's `save_stations`
+    /// (indent 2, UTF-8, trailing newline), atomically.
+    pub fn save(&self, path: &Path) -> Result<()> {
+        // numeric key order ("10" after "9"), as the Python mapping kept it
+        let mut obj = serde_json::Map::new();
+        for (k, v) in &self.0 {
+            obj.insert(k.to_string(), serde_json::to_value(v)?);
+        }
+        let text = serde_json::to_string_pretty(&serde_json::Value::Object(obj))? + "\n";
+        let tmp = path.with_extension("json.tmp");
+        std::fs::write(&tmp, text).with_context(|| format!("writing {}", tmp.display()))?;
+        std::fs::rename(&tmp, path).with_context(|| format!("replacing {}", path.display()))?;
+        Ok(())
+    }
+
+    /// Metadata for report column `n`, with the Python fallback for unknown
+    /// columns (`P.{19+n}` / `Station n`).
+    pub fn get(&self, n: u32) -> StationInfo {
+        self.0.get(&n).cloned().unwrap_or_else(|| StationInfo {
+            code: format!("P.{}", 19 + n),
+            thai_name: format!("Station {n}"),
+            english_name: format!("Station {n}"),
+            latitude: None,
+            longitude: None,
+            geohash: None,
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn bundled_mapping_has_the_16_rid_gauges() {
+        let s = Stations::bundled();
+        assert_eq!(s.0.len(), 16);
+        assert_eq!(s.get(1).code, "P.20");
+        assert_eq!(s.get(8).code, "P.1");
+        assert_eq!(s.get(99).code, "P.118"); // Python fallback
+    }
+
+    #[test]
+    fn save_round_trips() {
+        let mut s = Stations::bundled();
+        let id = s.next_id();
+        assert_eq!(id, 17);
+        let mut extra = s.get(8);
+        extra.code = "P.999".into();
+        s.insert(id, extra);
+        let dir = std::env::temp_dir().join(format!("ping-stations-{}", std::process::id()));
+        std::fs::create_dir_all(&dir).unwrap();
+        let path = dir.join("stations.json");
+        s.save(&path).unwrap();
+        let text = std::fs::read_to_string(&path).unwrap();
+        assert!(
+            text.starts_with("{\n  \"1\": {\n    \"code\": \"P.20\","),
+            "{text}"
+        );
+        assert!(text.contains("\"thai_name\": \"บ้านเชียงดาว\""));
+        let back = Stations::load(&path).unwrap();
+        assert_eq!(back.info(17).unwrap().code, "P.999");
+        assert_eq!(back.iter().count(), 17);
+        std::fs::remove_dir_all(&dir).unwrap();
+    }
+}

rust/collector/src/util.rs

@@ -0,0 +1,86 @@
+//! Small shared helpers: Bangkok wall-clock time and lenient JSON numerics.
+
+use chrono::{Duration, NaiveDateTime, Timelike, Utc};
+use serde_json::Value;
+
+/// Asia/Bangkok wall-clock time (UTC+7, no DST), without an offset. Every
+/// timestamp in the database is stored this way, independent of the
+/// server's own time zone.
+pub fn bangkok_now() -> NaiveDateTime {
+    (Utc::now() + Duration::hours(7)).naive_utc()
+}
+
+pub fn floor_hour(t: NaiveDateTime) -> NaiveDateTime {
+    t.with_minute(0)
+        .and_then(|t| t.with_second(0))
+        .and_then(|t| t.with_nanosecond(0))
+        .expect("valid time")
+}
+
+/// Upstream APIs send numbers as JSON numbers, numeric strings ("335.00",
+/// "1,234.5"), placeholders ("", "-", "***") or null. Anything that is not a
+/// finite number becomes None.
+pub fn lenient_f64(v: Option<&Value>) -> Option<f64> {
+    let x = match v? {
+        Value::Number(n) => n.as_f64()?,
+        Value::String(s) => s.trim().replace(',', "").parse::<f64>().ok()?,
+        _ => return None,
+    };
+    x.is_finite().then_some(x)
+}
+
+pub fn lenient_i64(v: Option<&Value>) -> Option<i64> {
+    match v? {
+        Value::Number(n) => n.as_i64().or_else(|| n.as_f64().map(|f| f as i64)),
+        Value::String(s) => s.trim().parse::<f64>().ok().map(|f| f as i64),
+        _ => None,
+    }
+}
+
+/// A JSON scalar as a string (ids arrive as numbers or strings).
+pub fn scalar_string(v: Option<&Value>) -> Option<String> {
+    match v? {
+        Value::String(s) if !s.is_empty() => Some(s.clone()),
+        Value::Number(n) => Some(n.to_string()),
+        _ => None,
+    }
+}
+
+/// Python truthiness of a JSON value (`bool(x)`).
+pub fn truthy(v: Option<&Value>) -> bool {
+    match v {
+        None | Some(Value::Null) => false,
+        Some(Value::Bool(b)) => *b,
+        Some(Value::Number(n)) => n.as_f64() != Some(0.0),
+        Some(Value::String(s)) => !s.is_empty(),
+        Some(Value::Array(a)) => !a.is_empty(),
+        Some(Value::Object(o)) => !o.is_empty(),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use serde_json::json;
+
+    #[test]
+    fn lenient_numbers() {
+        assert_eq!(lenient_f64(Some(&json!("335.00"))), Some(335.0));
+        assert_eq!(lenient_f64(Some(&json!(" 1,234.5 "))), Some(1234.5));
+        assert_eq!(lenient_f64(Some(&json!(2.5))), Some(2.5));
+        for junk in [json!(""), json!("-"), json!("***"), json!(null), json!([1])] {
+            assert_eq!(lenient_f64(Some(&junk)), None, "{junk}");
+        }
+        assert_eq!(lenient_i64(Some(&json!("2"))), Some(2));
+        assert_eq!(lenient_i64(Some(&json!(3))), Some(3));
+    }
+
+    #[test]
+    fn python_truthiness() {
+        assert!(truthy(Some(&json!(true))));
+        assert!(truthy(Some(&json!(1))));
+        assert!(!truthy(Some(&json!(0))));
+        assert!(!truthy(Some(&json!(""))));
+        assert!(!truthy(None));
+    }
+}

rust/collector/src/web/cache.rs

@@ -0,0 +1,245 @@
+//! Stale-while-revalidate response cache, as the retired FastAPI app's
+//! `_cached_swr`:
+//!
+//! - fresh entry: served;
+//! - expired entry (no older than `max_stale`): served at once, marked stale,
+//!   and ONE background refresh is started;
+//! - no entry: computed inline, single-flight per key (concurrent misses
+//!   wait for the one computation instead of piling queries onto Postgres);
+//! - a failed computation serves the last good value, whatever its age;
+//! - empty results are not cached, so recovery is immediate;
+//! - a byte budget per cache: the oldest entries go first (query parameters
+//!   are client-controlled, so keys are unbounded, and one history response
+//!   can be 10 MB).
+
+use super::error::ApiError;
+use std::collections::{HashMap, HashSet};
+use std::future::Future;
+use std::sync::{Arc, Mutex};
+use std::time::{Duration, Instant};
+
+/// The size of a cached value, for the budget.
+pub trait Weigh {
+    fn weight(&self) -> usize;
+}
+
+impl Weigh for axum::body::Bytes {
+    fn weight(&self) -> usize {
+        self.len()
+    }
+}
+
+impl Weigh for (axum::body::Bytes, bool) {
+    fn weight(&self) -> usize {
+        self.0.len()
+    }
+}
+
+#[cfg(test)]
+impl Weigh for u32 {
+    fn weight(&self) -> usize {
+        4
+    }
+}
+
+pub struct Swr<T> {
+    budget: usize,
+    entries: Mutex<HashMap<String, (Instant, T)>>,
+    flights: Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>,
+    refreshing: Mutex<HashSet<String>>,
+}
+
+/// What a computation returns: the value and whether to keep it.
+pub type Computed<T> = Result<(T, bool), ApiError>;
+
+impl<T: Clone + Send + Sync + Weigh + 'static> Swr<T> {
+    /// A cache holding at most `budget` bytes.
+    pub fn new(budget: usize) -> Arc<Self> {
+        Arc::new(Self {
+            budget,
+            entries: Mutex::new(HashMap::new()),
+            flights: Mutex::new(HashMap::new()),
+            refreshing: Mutex::new(HashSet::new()),
+        })
+    }
+
+    fn entry(&self, key: &str) -> Option<(Instant, T)> {
+        self.entries.lock().unwrap().get(key).cloned()
+    }
+
+    /// `(value, served_stale)`
+    pub async fn get<F, Fut>(
+        self: &Arc<Self>,
+        key: String,
+        ttl: Duration,
+        max_stale: Option<Duration>,
+        compute: F,
+    ) -> Result<(T, bool), ApiError>
+    where
+        F: Fn() -> Fut + Send + Sync + 'static,
+        Fut: Future<Output = Computed<T>> + Send + 'static,
+    {
+        if let Some((at, v)) = self.entry(&key) {
+            let age = at.elapsed();
+            if age < ttl {
+                return Ok((v, false));
+            }
+            if max_stale.is_none_or(|m| age <= m) {
+                let start = self.refreshing.lock().unwrap().insert(key.clone());
+                if start {
+                    let me = self.clone();
+                    tokio::spawn(async move {
+                        if let Err(e) = me.compute(&key, ttl, &compute).await {
+                            tracing::warn!("{key}: background refresh failed: {}", e.detail);
+                        }
+                        me.refreshing.lock().unwrap().remove(&key);
+                    });
+                }
+                return Ok((v, true));
+            }
+        }
+        self.compute(&key, ttl, &compute).await
+    }
+
+    async fn compute<F, Fut>(
+        &self,
+        key: &str,
+        ttl: Duration,
+        compute: &F,
+    ) -> Result<(T, bool), ApiError>
+    where
+        F: Fn() -> Fut,
+        Fut: Future<Output = Computed<T>>,
+    {
+        let flight = self
+            .flights
+            .lock()
+            .unwrap()
+            .entry(key.to_string())
+            .or_default()
+            .clone();
+        let _guard = flight.lock().await;
+        let previous = self.entry(key);
+        if let Some((at, v)) = &previous
+            && at.elapsed() < ttl
+        {
+            return Ok((v.clone(), false));
+        }
+        match compute().await {
+            Ok((v, keep)) => {
+                if keep && v.weight() <= self.budget {
+                    let mut entries = self.entries.lock().unwrap();
+                    entries.insert(key.to_string(), (Instant::now(), v.clone()));
+                    let mut total: usize = entries.values().map(|(_, v)| v.weight()).sum();
+                    if total > self.budget {
+                        let mut ages: Vec<(Instant, String)> = entries
+                            .iter()
+                            .map(|(k, (at, _))| (*at, k.clone()))
+                            .collect();
+                        ages.sort();
+                        for (_, k) in ages {
+                            if total <= self.budget {
+                                break;
+                            }
+                            if let Some((_, old)) = entries.remove(&k) {
+                                total -= old.weight();
+                            }
+                        }
+                    }
+                }
+                Ok((v, false))
+            }
+            Err(e) => match previous {
+                Some((at, v)) => {
+                    tracing::warn!(
+                        "{key}: recompute failed ({}); serving {}s-stale copy",
+                        e.detail,
+                        at.elapsed().as_secs()
+                    );
+                    Ok((v, true))
+                }
+                None => Err(e),
+            },
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::sync::atomic::{AtomicU32, Ordering};
+
+    #[tokio::test]
+    async fn fresh_stale_and_error_fallback() {
+        let cache: Arc<Swr<u32>> = Swr::new(1024);
+        let calls = Arc::new(AtomicU32::new(0));
+        let fail = Arc::new(std::sync::atomic::AtomicBool::new(false));
+        let compute = {
+            let (calls, fail) = (calls.clone(), fail.clone());
+            move || {
+                let (calls, fail) = (calls.clone(), fail.clone());
+                async move {
+                    let n = calls.fetch_add(1, Ordering::SeqCst) + 1;
+                    if fail.load(Ordering::SeqCst) {
+                        Err(ApiError::new(500, "down"))
+                    } else {
+                        Ok((n, true))
+                    }
+                }
+            }
+        };
+        let ttl = Duration::from_millis(40);
+        let key = || "k".to_string();
+        assert_eq!(
+            cache.get(key(), ttl, None, compute.clone()).await.unwrap(),
+            (1, false)
+        );
+        assert_eq!(
+            cache.get(key(), ttl, None, compute.clone()).await.unwrap(),
+            (1, false)
+        );
+        tokio::time::sleep(Duration::from_millis(60)).await;
+        // expired: the old value at once, refreshed in the background
+        assert_eq!(
+            cache.get(key(), ttl, None, compute.clone()).await.unwrap(),
+            (1, true)
+        );
+        tokio::time::sleep(Duration::from_millis(10)).await;
+        assert_eq!(
+            cache.get(key(), ttl, None, compute.clone()).await.unwrap(),
+            (2, false)
+        );
+        // older than max_stale and the database is down: last good value
+        tokio::time::sleep(Duration::from_millis(60)).await;
+        fail.store(true, Ordering::SeqCst);
+        let max = Some(Duration::from_millis(1));
+        assert_eq!(
+            cache.get(key(), ttl, max, compute.clone()).await.unwrap(),
+            (2, true)
+        );
+        // nothing cached yet: the error
+        assert!(cache.get("other".into(), ttl, max, compute).await.is_err());
+    }
+
+    #[tokio::test]
+    async fn budget_evicts_the_oldest() {
+        let cache: Arc<Swr<u32>> = Swr::new(8); // two u32 entries
+        let ttl = Duration::from_secs(60);
+        for (i, k) in ["a", "b", "c"].iter().enumerate() {
+            let v = i as u32;
+            cache
+                .get(
+                    k.to_string(),
+                    ttl,
+                    None,
+                    move || async move { Ok((v, true)) },
+                )
+                .await
+                .unwrap();
+            tokio::time::sleep(Duration::from_millis(2)).await;
+        }
+        let entries = cache.entries.lock().unwrap();
+        assert_eq!(entries.len(), 2);
+        assert!(!entries.contains_key("a"));
+    }
+}

rust/collector/src/web/data.rs

@@ -0,0 +1,357 @@
+//! RID gauge readings: /measurements/*, /api/stats.
+
+use super::error::{ApiError, ApiResult, ok, to_body};
+use super::params::{self, Param};
+use super::pyfmt::{end_of_day, isoformat, py_round, start_of_day};
+use super::{Shared, secs};
+use crate::util::bangkok_now;
+use axum::extract::{Path, RawQuery, State};
+use axum::http::HeaderMap;
+use chrono::{NaiveDateTime, TimeDelta};
+use serde_json::{Map, Value, json};
+use sqlx::postgres::PgRow;
+use sqlx::{Column, Row, TypeInfo};
+use std::time::Duration;
+
+/// A result row as a JSON object: column names as keys (SQL aliases), in
+/// select order. NUMERIC columns must be cast to float8 in the query (as
+/// Python's `float(Decimal)`), timestamps become `isoformat()` strings.
+pub fn row_json(row: &PgRow) -> Map<String, Value> {
+    let mut m = Map::new();
+    for (i, col) in row.columns().iter().enumerate() {
+        let v = match col.type_info().name() {
+            "INT2" => json!(row.try_get::<Option<i16>, _>(i).ok().flatten()),
+            "INT4" => json!(row.try_get::<Option<i32>, _>(i).ok().flatten()),
+            "INT8" => json!(row.try_get::<Option<i64>, _>(i).ok().flatten()),
+            "FLOAT4" => super::pyfmt::num(
+                row.try_get::<Option<f32>, _>(i)
+                    .ok()
+                    .flatten()
+                    .map(f64::from),
+            ),
+            "FLOAT8" => super::pyfmt::num(row.try_get::<Option<f64>, _>(i).ok().flatten()),
+            "BOOL" => json!(row.try_get::<Option<bool>, _>(i).ok().flatten()),
+            "TIMESTAMP" => json!(
+                row.try_get::<Option<NaiveDateTime>, _>(i)
+                    .ok()
+                    .flatten()
+                    .map(isoformat)
+            ),
+            "DATE" => json!(
+                row.try_get::<Option<chrono::NaiveDate>, _>(i)
+                    .ok()
+                    .flatten()
+                    .map(|d| d.format("%Y-%m-%d").to_string())
+            ),
+            _ => json!(row.try_get::<Option<String>, _>(i).ok().flatten()),
+        };
+        m.insert(col.name().to_string(), v);
+    }
+    m
+}
+
+pub fn rows_json(rows: &[PgRow]) -> Vec<Value> {
+    rows.iter().map(|r| Value::Object(row_json(r))).collect()
+}
+
+fn query(q: &Option<String>) -> std::collections::HashMap<String, String> {
+    params::query_map(q.as_deref())
+}
+
+/// `datetime.now() - timedelta(hours=h)`; `None` where Python overflows.
+pub fn hours_before(t: NaiveDateTime, hours: i64) -> Option<NaiveDateTime> {
+    t.checked_sub_signed(TimeDelta::try_hours(hours)?)
+}
+
+const MEASUREMENT_COLUMNS: &str = "m.timestamp, s.station_code, s.english_name AS station_name_en,
+       s.thai_name AS station_name_th, m.water_level::float8 AS water_level,
+       m.discharge::float8 AS discharge, m.discharge_percent::float8 AS discharge_percent, m.status";
+
+pub const LATEST: &[Param] = &[Param::int("limit", 100)];
+
+/// `SQLAdapter.get_latest_measurements`: the newest reading per station.
+pub async fn latest_rows(pool: &sqlx::PgPool, limit: i64) -> Result<Vec<PgRow>, sqlx::Error> {
+    if limit < 0 {
+        // Postgres refuses a negative LIMIT; the Python adapter answered []
+        return Ok(vec![]);
+    }
+    sqlx::query(&format!(
+        "SELECT {MEASUREMENT_COLUMNS}
+         FROM water_measurements m
+         JOIN stations s ON m.station_id = s.id
+         INNER JOIN (
+             SELECT station_id, MAX(timestamp) as max_timestamp
+             FROM water_measurements
+             GROUP BY station_id
+         ) latest ON m.station_id = latest.station_id AND m.timestamp = latest.max_timestamp
+         ORDER BY s.station_code
+         LIMIT $1"
+    ))
+    .bind(limit)
+    .fetch_all(pool)
+    .await
+}
+
+/// GET /measurements/latest: the most frequently hit endpoint (every
+/// dashboard poll), stale-while-revalidate cached.
+pub async fn latest(State(st): State<Shared>, RawQuery(q): RawQuery, h: HeaderMap) -> ApiResult {
+    st.metrics.count("measurements_latest");
+    let a = params::parse(LATEST, &query(&q), &[], &h)?;
+    let limit = a.int("limit");
+    let pool = st.pool.clone();
+    let (body, stale) = st
+        .caches
+        .latest
+        .get(
+            format!("latest:{limit}"),
+            secs(st.cfg.web.latest_cache_ttl_s),
+            None,
+            move || {
+                let pool = pool.clone();
+                async move {
+                    let rows = rows_json(&latest_rows(&pool, limit).await?);
+                    let keep = !rows.is_empty();
+                    Ok((to_body(&rows), keep))
+                }
+            },
+        )
+        .await?;
+    ok(body, stale)
+}
+
+type HistoryRow = (NaiveDateTime, String, Option<f64>, Option<f64>, Option<f64>);
+
+pub const HISTORY: &[Param] = &[
+    Param::path_str("station_code"),
+    Param::int("hours", 168).ge(1),
+    Param::int("limit", 50000).ge(1).le(100000),
+    Param::opt_date("start", "First day (overrides hours)"),
+    Param::opt_date("end", "Last day, inclusive"),
+];
+
+/// GET /measurements/history/{station_code} (`PostgresHistory.station_history`)
+pub async fn history(
+    State(st): State<Shared>,
+    Path(code): Path<String>,
+    RawQuery(q): RawQuery,
+    h: HeaderMap,
+) -> ApiResult {
+    let a = params::parse(HISTORY, &query(&q), &[("station_code", &code)], &h)?;
+    let key = format!(
+        "{code}:{}:{}:{}:{}",
+        a.py_str("hours"),
+        a.py_str("limit"),
+        a.py_str("start"),
+        a.py_str("end")
+    );
+    let (hours, limit, start, end) = (
+        a.int("hours"),
+        a.int("limit"),
+        a.date("start"),
+        a.date("end"),
+    );
+    let pool = st.pool.clone();
+    let unavailable = || ApiError::new(502, "Measurement history unavailable");
+    let (body, stale) = st
+        .caches
+        .history
+        .get(
+            key,
+            Duration::from_secs(300),
+            Some(Duration::ZERO),
+            move || {
+                let (pool, code) = (pool.clone(), code.clone());
+                async move {
+                    let end_time = end.map_or_else(bangkok_now, end_of_day);
+                    let start_time = match start {
+                        Some(d) => start_of_day(d),
+                        None => hours_before(end_time, hours).ok_or_else(unavailable)?,
+                    };
+                    if start_time >= end_time {
+                        return Err(unavailable());
+                    }
+                    let rows: Vec<HistoryRow> = sqlx::query_as(
+                        "SELECT m.timestamp, s.station_code, m.water_level::float8,
+                                m.discharge::float8, m.discharge_percent::float8
+                         FROM water_measurements m
+                         JOIN stations s ON m.station_id = s.id
+                         WHERE s.station_code = $1
+                           AND m.timestamp >= $2
+                           AND m.timestamp <= $3
+                         ORDER BY m.timestamp ASC
+                         LIMIT $4",
+                    )
+                    .bind(&code)
+                    .bind(start_time)
+                    .bind(end_time)
+                    .bind(limit)
+                    .fetch_all(&pool)
+                    .await
+                    .map_err(|e| {
+                        tracing::error!("Error fetching measurement history: {e}");
+                        unavailable()
+                    })?;
+                    let out: Vec<Value> = rows
+                        .into_iter()
+                        .map(|(t, code, level, discharge, pct)| {
+                            // no rating curves are defined: Q = 20 * H, as postgres_history.py
+                            let discharge = discharge.or(level.map(|h| py_round(20.0 * h, 2)));
+                            json!({
+                                "timestamp": isoformat(t),
+                                "station_code": code,
+                                "water_level": level,
+                                "discharge": discharge,
+                                "discharge_percent": pct,
+                            })
+                        })
+                        .collect();
+                    Ok((to_body(&out), true))
+                }
+            },
+        )
+        .await?;
+    ok(body, stale)
+}
+
+pub const STATION: &[Param] = &[
+    Param::path_str("station_code"),
+    Param::int("hours", 24),
+    Param::int("limit", 1000),
+];
+
+/// GET /measurements/station/{station_code}: newest first.
+pub async fn station(
+    State(st): State<Shared>,
+    Path(code): Path<String>,
+    RawQuery(q): RawQuery,
+    h: HeaderMap,
+) -> ApiResult {
+    st.metrics.count("measurements_station");
+    let a = params::parse(STATION, &query(&q), &[("station_code", &code)], &h)?;
+    let end = bangkok_now();
+    let start = hours_before(end, a.int("hours"))
+        .ok_or_else(|| ApiError::new(500, "date value out of range"))?;
+    let rows = sqlx::query(&format!(
+        "SELECT {MEASUREMENT_COLUMNS}
+         FROM water_measurements m
+         JOIN stations s ON m.station_id = s.id
+         WHERE m.timestamp BETWEEN $1 AND $2 AND s.station_code IN ($3)
+         ORDER BY m.timestamp DESC, s.station_code"
+    ))
+    .bind(start)
+    .bind(end)
+    .bind(&code)
+    .fetch_all(&st.pool)
+    .await?;
+    let mut out = rows_json(&rows);
+    // measurements[:limit], Python slice semantics
+    let limit = a.int("limit");
+    let keep = if limit >= 0 {
+        (limit as usize).min(out.len())
+    } else {
+        out.len().saturating_sub(limit.unsigned_abs() as usize)
+    };
+    out.truncate(keep);
+    ok(to_body(&out), false)
+}
+
+async fn count(pool: &sqlx::PgPool, table: &str) -> i64 {
+    sqlx::query_scalar::<_, i64>(&format!("SELECT COUNT(*) FROM {table}"))
+        .fetch_one(pool)
+        .await
+        .unwrap_or(0)
+}
+
+type HiiTotals = (
+    i64,
+    i64,
+    i64,
+    i64,
+    Option<NaiveDateTime>,
+    Option<NaiveDateTime>,
+    Option<NaiveDateTime>,
+    Option<NaiveDateTime>,
+);
+
+async fn compute_stats(pool: sqlx::PgPool) -> super::cache::Computed<axum::body::Bytes> {
+    // Heavy: full-table counts and coverage; runs at most once per TTL
+    type Stats = (i64, i64, Option<NaiveDateTime>, Option<NaiveDateTime>, i64);
+    let (total, stations, first, last, recorded): Stats = sqlx::query_as(
+        "SELECT COUNT(*), COUNT(DISTINCT station_id), MIN(timestamp), MAX(timestamp),
+                COUNT(DISTINCT TO_CHAR(timestamp, 'YYYY-MM-DD HH24'))
+         FROM water_measurements",
+    )
+    .fetch_one(&pool)
+    .await?;
+    let (Some(mut first_ts), Some(mut last_ts)) = (first, last) else {
+        return Err(ApiError::new(503, "Database statistics unavailable"));
+    };
+    if total == 0 {
+        return Err(ApiError::new(503, "Database statistics unavailable"));
+    }
+    let slot = crate::util::floor_hour;
+    let expected = (slot(last_ts) - slot(first_ts)).num_hours() + 1;
+    let coverage = py_round(100.0 * recorded as f64 / expected as f64, 1);
+
+    let hii: Option<HiiTotals> = sqlx::query_as(
+        "SELECT (SELECT COUNT(*) FROM hii_rainfall),
+                (SELECT COUNT(*) FROM hii_waterlevel),
+                (SELECT COUNT(*) FROM hii_rain_stations),
+                (SELECT COUNT(*) FROM hii_wl_stations),
+                (SELECT MIN(timestamp) FROM hii_rainfall),
+                (SELECT MAX(timestamp) FROM hii_rainfall),
+                (SELECT MIN(timestamp) FROM hii_waterlevel),
+                (SELECT MAX(timestamp) FROM hii_waterlevel)",
+    )
+    .fetch_one(&pool)
+    .await
+    .map_err(|e| tracing::warn!("HII stats unavailable: {e}"))
+    .ok();
+    let openmeteo_n = count(&pool, "openmeteo_rain").await;
+    let reservoir_n = count(&pool, "rid_reservoir_daily").await;
+    let (mut rain_n, mut wl_n, mut hii_stations) = (0, 0, 0);
+    if let Some((rn, wn, rs, ws, rain_lo, rain_hi, wl_lo, wl_hi)) = hii {
+        (rain_n, wl_n, hii_stations) = (rn, wn, rs + ws);
+        for lo in [rain_lo, wl_lo].into_iter().flatten() {
+            if lo < first_ts {
+                first_ts = lo;
+            }
+        }
+        for hi in [rain_hi, wl_hi].into_iter().flatten() {
+            if hi > last_ts {
+                last_ts = hi;
+            }
+        }
+    }
+    let body = json!({
+        // whole-DB totals (RID + HII + Open-Meteo + dams), breakdown alongside
+        "total_measurements": total + rain_n + wl_n + openmeteo_n + reservoir_n,
+        "rid_measurements": total,
+        "hii_rainfall_measurements": rain_n,
+        "hii_waterlevel_measurements": wl_n,
+        "openmeteo_rain_measurements": openmeteo_n,
+        "reservoir_measurements": reservoir_n,
+        "station_count": stations + hii_stations,
+        "rid_station_count": stations,
+        "hii_station_count": hii_stations,
+        "first_timestamp": isoformat(first_ts),
+        "last_timestamp": isoformat(last_ts),
+        "days_spanned": (last_ts.date() - first_ts.date()).num_days() + 1,
+        "coverage_percent": coverage,
+    });
+    Ok((to_body(&body), true))
+}
+
+/// GET /api/stats: database coverage (totals, date range, hourly coverage).
+pub async fn stats(State(st): State<Shared>) -> ApiResult {
+    st.metrics.count("api_stats");
+    let pool = st.pool.clone();
+    let (body, _) = st
+        .caches
+        .stats
+        .get("all".into(), Duration::from_secs(300), None, move || {
+            compute_stats(pool.clone())
+        })
+        .await?;
+    ok(body, false)
+}

rust/collector/src/web/error.rs

@@ -0,0 +1,88 @@
+//! FastAPI-shaped errors (`{"detail": ...}`) and JSON responses.
+
+use axum::body::Bytes;
+use axum::http::{HeaderValue, StatusCode, header};
+use axum::response::{IntoResponse, Response};
+use serde::Serialize;
+use serde_json::{Value, json};
+
+#[derive(Debug, Clone)]
+pub struct ApiError {
+    pub status: StatusCode,
+    pub detail: Value,
+}
+
+impl ApiError {
+    pub fn new(status: u16, detail: impl Into<String>) -> Self {
+        Self {
+            status: StatusCode::from_u16(status).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR),
+            detail: Value::String(detail.into()),
+        }
+    }
+
+    /// 422 with pydantic's error list.
+    pub fn validation(errors: Vec<Value>) -> Self {
+        Self {
+            status: StatusCode::UNPROCESSABLE_ENTITY,
+            detail: Value::Array(errors),
+        }
+    }
+
+    /// An unexpected failure (FastAPI handlers answer `500 str(e)`).
+    pub fn internal(e: impl std::fmt::Display) -> Self {
+        Self::new(500, e.to_string())
+    }
+}
+
+impl From<sqlx::Error> for ApiError {
+    fn from(e: sqlx::Error) -> Self {
+        tracing::error!("database: {e}");
+        Self::internal(e)
+    }
+}
+
+impl From<anyhow::Error> for ApiError {
+    fn from(e: anyhow::Error) -> Self {
+        tracing::error!("{e:#}");
+        Self::new(500, format!("{e:#}"))
+    }
+}
+
+impl IntoResponse for ApiError {
+    fn into_response(self) -> Response {
+        json_response(
+            self.status,
+            to_body(&json!({ "detail": self.detail })),
+            false,
+        )
+    }
+}
+
+pub type ApiResult = Result<Response, ApiError>;
+
+pub fn to_body(v: &impl Serialize) -> Bytes {
+    Bytes::from(serde_json::to_vec(v).expect("JSON-serializable value"))
+}
+
+/// A JSON body; `stale` adds `X-Data-Stale: true` (served from the cache
+/// after a failed refresh, or an old stored forecast).
+pub fn json_response(status: StatusCode, body: Bytes, stale: bool) -> Response {
+    let mut r = (status, body).into_response();
+    let h = r.headers_mut();
+    h.insert(
+        header::CONTENT_TYPE,
+        HeaderValue::from_static("application/json"),
+    );
+    if stale {
+        h.insert("x-data-stale", HeaderValue::from_static("true"));
+    }
+    r
+}
+
+pub fn ok(body: Bytes, stale: bool) -> ApiResult {
+    Ok(json_response(StatusCode::OK, body, stale))
+}
+
+pub fn ok_json(v: &impl Serialize) -> ApiResult {
+    ok(to_body(v), false)
+}

rust/collector/src/web/forecasts.rs

@@ -0,0 +1,333 @@
+//! Flood forecasts: /forecast, /api/forecast/*, /api/notifications.
+
+use super::data::{hours_before, rows_json};
+use super::error::{ApiError, ApiResult, ok, ok_json, to_body};
+use super::params::{self, Param};
+use super::pyfmt::{end_of_day, isoformat, py_round, start_of_day};
+use super::{Shared, secs};
+use crate::util::bangkok_now;
+use axum::extract::{Path, RawQuery, State};
+use axum::http::HeaderMap;
+use chrono::NaiveDateTime;
+use serde_json::{Map, Value, json};
+use std::time::Duration;
+
+/// GET /forecast: flood-risk forecasts per station for the 6/12/24 h
+/// horizons, as the forecaster last stored them (forecast_current).
+///
+/// The Python app recomputed in-process when the stored forecast was older
+/// than FORECAST_MAX_AGE_SECONDS; here the forecaster runs in the same
+/// process, so an old forecast is served marked `X-Data-Stale: true`.
+pub async fn forecast(State(st): State<Shared>) -> ApiResult {
+    st.metrics.count("forecast");
+    let pool = st.pool.clone();
+    let max_age = st.cfg.web.forecast_max_age_s as i64;
+    let ((body, old), _) = st
+        .caches
+        .forecast
+        .get(
+            "stored".into(),
+            Duration::from_secs(60),
+            Some(Duration::ZERO),
+            move || {
+                let pool = pool.clone();
+                async move {
+                    let row: Option<(NaiveDateTime, String)> = sqlx::query_as(
+                        "SELECT computed_at, payload FROM forecast_current WHERE id = 'all'",
+                    )
+                    .fetch_optional(&pool)
+                    .await?;
+                    let Some((computed_at, payload)) = row else {
+                        return Err(ApiError::new(
+                            503,
+                            "No forecast stored yet (the forecaster has not run)",
+                        ));
+                    };
+                    let age = (bangkok_now() - computed_at).num_seconds();
+                    if age > max_age {
+                        tracing::warn!("stored forecast is {:.1} h old", age as f64 / 3600.0);
+                    }
+                    Ok(((payload.into(), age > max_age), true))
+                }
+            },
+        )
+        .await?;
+    ok(body, old)
+}
+
+pub const HISTORY: &[Param] = &[
+    Param::path_str("station_code"),
+    Param::int("hours", 168).ge(1),
+    Param::opt_date("start", "First day (overrides hours)"),
+    Param::opt_date("end", "Last day, inclusive"),
+    Param::opt_int("horizon").ge(1).le(48),
+];
+
+/// GET /api/forecast/history/{station_code}: issued model forecasts, what
+/// the model predicted when (`ForecastHistoryStore.fetch`).
+pub async fn history(
+    State(st): State<Shared>,
+    Path(code): Path<String>,
+    RawQuery(q): RawQuery,
+    h: HeaderMap,
+) -> ApiResult {
+    st.metrics.count("forecast_history");
+    let a = params::parse(
+        HISTORY,
+        &params::query_map(q.as_deref()),
+        &[("station_code", &code)],
+        &h,
+    )?;
+    let end = a.date("end").map_or_else(bangkok_now, end_of_day);
+    let start = match a.date("start") {
+        Some(d) => start_of_day(d),
+        None => hours_before(end, a.int("hours"))
+            .ok_or_else(|| ApiError::new(500, "date value out of range"))?,
+    };
+    let horizon = a.opt_int("horizon");
+    let rows = sqlx::query(
+        "SELECT as_of, station_code, horizon_hours,
+                predicted_max_level::float8 AS predicted_max_level,
+                p_warning::float8 AS p_warning, p_danger::float8 AS p_danger,
+                current_level::float8 AS current_level, model_version, source
+         FROM forecast_history
+         WHERE station_code = $1 AND as_of >= $2 AND as_of <= $3
+           AND ($4::int IS NULL OR horizon_hours = $4)
+         ORDER BY as_of ASC, horizon_hours ASC LIMIT 5000",
+    )
+    .bind(&code)
+    .bind(start)
+    .bind(end)
+    .bind(horizon.map(|v| v as i32))
+    .fetch_all(&st.pool)
+    .await;
+    match rows {
+        Ok(rows) => ok_json(&rows_json(&rows)),
+        Err(e) => {
+            // as the Python store: an empty history rather than an error
+            tracing::error!("forecast history fetch failed: {e}");
+            ok_json(&Vec::<Value>::new())
+        }
+    }
+}
+
+/// GET /api/notifications: public ntfy settings for the dashboard's
+/// subscribe links.
+pub async fn notifications(State(st): State<Shared>) -> ApiResult {
+    let Some(server) = &st.cfg.ntfy_server else {
+        return ok_json(&json!({"enabled": false}));
+    };
+    let p = &st.cfg.ntfy_topic_prefix;
+    ok_json(&json!({
+        "enabled": true,
+        "server": server,
+        "prefix": p,
+        "topics": {
+            "warning": format!("{p}-warning"),
+            "danger": format!("{p}-danger"),
+            "p1_outlook": format!("{p}-p1-outlook"),
+            "status": format!("{p}-status"),
+            "station_pattern": format!("{p}-<station>-warning | {p}-<station>-danger (station code lowercase, no dot: p1, p103)"),
+        },
+        "semantics": "transitions only: one message on crossing up, one all-clear on the way down (0.10 m hysteresis)",
+    }))
+}
+
+pub const SKILL: &[Param] = &[
+    Param::str("station_code", "P.1"),
+    Param::int("horizon", 24).ge(1).le(48),
+];
+
+const MIN_VERIFIED: usize = 24; // fewer than a day of verified hours is not a number
+
+struct Verified {
+    as_of: NaiveDateTime,
+    predicted: f64,
+    current: Option<f64>,
+    observed: f64,
+}
+
+/// Python's `sum()` (3.11: left to right) / len
+fn mean(v: impl Iterator<Item = f64>) -> f64 {
+    let (mut s, mut n) = (0.0, 0usize);
+    for x in v {
+        s += x;
+        n += 1;
+    }
+    s / n as f64
+}
+
+/// `src/ml/skill.compute_skill`: issued forecasts verified against what the
+/// river did, per model version, with a persistence baseline.
+pub async fn compute_skill(
+    pool: &sqlx::PgPool,
+    code: &str,
+    horizon: i64,
+    now: NaiveDateTime,
+) -> Result<Value, sqlx::Error> {
+    let verifiable_before = now - chrono::TimeDelta::hours(horizon);
+    type Row = (
+        NaiveDateTime,
+        Option<String>,
+        f64,
+        Option<f64>,
+        Option<f64>,
+        Option<i64>,
+    );
+    let rows: Vec<Row> = sqlx::query_as(
+        "SELECT f.as_of, f.model_version, f.predicted_max_level::float8, f.current_level::float8,
+               (SELECT MAX(m.water_level) FROM water_measurements m
+                 JOIN stations s ON s.id = m.station_id
+                WHERE s.station_code = f.station_code
+                  AND m.timestamp > f.as_of
+                  AND m.timestamp <= f.as_of + (f.horizon_hours || ' hours')::interval)::float8,
+               (SELECT COUNT(m.water_level) FROM water_measurements m
+                 JOIN stations s ON s.id = m.station_id
+                WHERE s.station_code = f.station_code
+                  AND m.timestamp > f.as_of
+                  AND m.timestamp <= f.as_of + (f.horizon_hours || ' hours')::interval)
+         FROM forecast_history f
+         WHERE f.station_code = $1 AND f.horizon_hours = $2
+           AND f.source = 'model' AND f.predicted_max_level IS NOT NULL
+           AND f.as_of <= $3
+         ORDER BY f.as_of",
+    )
+    .bind(code)
+    .bind(horizon as i32)
+    .bind(verifiable_before)
+    .fetch_all(pool)
+    .await?;
+
+    // versions in order of first appearance, as Python's dict
+    let mut by_version: Vec<(String, Vec<Verified>)> = vec![];
+    for (as_of, version, predicted, current, observed, n) in rows {
+        // most of the window must be observed, or the "max" is not the peak
+        let Some(observed) = observed else { continue };
+        if (n.unwrap_or(0) as f64) < horizon as f64 * 0.75 {
+            continue;
+        }
+        let version = version.unwrap_or_else(|| "unknown".into());
+        let row = Verified {
+            as_of,
+            predicted,
+            current,
+            observed,
+        };
+        match by_version.iter_mut().find(|(v, _)| *v == version) {
+            Some((_, list)) => list.push(row),
+            None => by_version.push((version, vec![row])),
+        }
+    }
+
+    let mut versions: Vec<Map<String, Value>> = vec![];
+    for (version, rows) in &by_version {
+        let err: Vec<f64> = rows.iter().map(|r| r.predicted - r.observed).collect();
+        let mae = mean(err.iter().map(|e| e.abs()));
+        let bias = mean(err.iter().copied());
+        let pers: Vec<(f64, f64)> = rows
+            .iter()
+            .filter_map(|r| r.current.map(|c| (c, r.observed)))
+            .collect();
+        let persistence = (!pers.is_empty()).then(|| mean(pers.iter().map(|(c, o)| (c - o).abs())));
+        let high: Vec<(f64, f64)> = rows
+            .iter()
+            .filter(|r| r.observed >= 2.0)
+            .map(|r| (r.predicted, r.observed))
+            .collect();
+        let first = rows.iter().map(|r| r.as_of).min().expect("non-empty");
+        let last = rows.iter().map(|r| r.as_of).max().expect("non-empty");
+        let mut v = Map::new();
+        v.insert("model_version".into(), json!(version));
+        v.insert("first_issued".into(), json!(isoformat(first)));
+        v.insert("last_issued".into(), json!(isoformat(last)));
+        v.insert("n".into(), json!(rows.len()));
+        v.insert("mae_m".into(), json!(py_round(mae, 3)));
+        v.insert("bias_m".into(), json!(py_round(bias, 3)));
+        v.insert(
+            "persistence_mae_m".into(),
+            json!(persistence.map(|p| py_round(p, 3))),
+        );
+        v.insert(
+            "skill".into(),
+            json!(
+                persistence
+                    .filter(|p| *p != 0.0)
+                    .map(|p| py_round(1.0 - mae / p, 3))
+            ),
+        );
+        v.insert("above_2m_n".into(), json!(high.len()));
+        v.insert(
+            "above_2m_mae_m".into(),
+            json!(
+                (!high.is_empty())
+                    .then(|| py_round(mean(high.iter().map(|(p, o)| (p - o).abs())), 3))
+            ),
+        );
+        v.insert("enough_data".into(), json!(rows.len() >= MIN_VERIFIED));
+        versions.push(v);
+    }
+    // stable sort on the ISO string, as Python
+    versions.sort_by(|a, b| a["first_issued"].as_str().cmp(&b["first_issued"].as_str()));
+
+    let current = versions.last().cloned();
+    let previous = versions
+        .len()
+        .checked_sub(1)
+        .and_then(|n| {
+            versions[..n]
+                .iter()
+                .rev()
+                .find(|v| v["enough_data"] == true)
+        })
+        .cloned();
+    let trend = match (&current, &previous) {
+        (Some(c), Some(p)) if c["enough_data"] == true => {
+            let (cm, pm) = (c["mae_m"].as_f64().unwrap(), p["mae_m"].as_f64().unwrap());
+            let skill_delta = match (c["skill"].as_f64(), p["skill"].as_f64()) {
+                (Some(cs), Some(ps)) => json!(py_round(cs - ps, 3)),
+                _ => Value::Null,
+            };
+            json!({
+                "previous_version": p["model_version"],
+                "mae_delta_m": py_round(cm - pm, 3),
+                "skill_delta": skill_delta,
+                "better": cm < pm,
+            })
+        }
+        _ => Value::Null,
+    };
+    Ok(json!({
+        "station_code": code,
+        "horizon_hours": horizon,
+        "verified_until": isoformat(verifiable_before),
+        "min_verified": MIN_VERIFIED,
+        "versions": versions,
+        "current": current,
+        "trend": trend,
+    }))
+}
+
+/// GET /api/forecast/skill: is the model getting better?
+pub async fn skill(State(st): State<Shared>, RawQuery(q): RawQuery, h: HeaderMap) -> ApiResult {
+    st.metrics.count("forecast_skill");
+    let a = params::parse(SKILL, &params::query_map(q.as_deref()), &[], &h)?;
+    let (code, horizon) = (a.str("station_code").to_string(), a.int("horizon"));
+    let pool = st.pool.clone();
+    let (body, stale) = st
+        .caches
+        .hii
+        .get(
+            format!("skill:{code}:{horizon}"),
+            secs(st.cfg.web.hii_cache_ttl_s.max(900)),
+            None,
+            move || {
+                let (pool, code) = (pool.clone(), code.clone());
+                async move {
+                    let v = compute_skill(&pool, &code, horizon, bangkok_now()).await?;
+                    Ok((to_body(&v), true))
+                }
+            },
+        )
+        .await?;
+    ok(body, stale)
+}

rust/collector/src/web/hii.rs

@@ -0,0 +1,427 @@
+//! HII/ThaiWater feeds: /api/hii/*, /sensors/thaiwater.
+
+use super::data::{hours_before, rows_json};
+use super::error::{ApiError, ApiResult, ok, to_body};
+use super::params::{self, Param};
+use super::pyfmt::{isoformat, num, numpy_mean, py_round, py_str};
+use super::{Shared, secs};
+use crate::util::{bangkok_now, floor_hour};
+use axum::extract::{RawQuery, State};
+use axum::http::HeaderMap;
+use chrono::{NaiveDateTime, TimeDelta};
+use serde_json::{Map, Value, json};
+use std::collections::BTreeMap;
+use std::time::Duration;
+
+pub const LATEST: &[Param] = &[Param::int("hours", 26).ge(1).le(168)];
+
+const RAIN_SQL: &str = "
+    SELECT s.id AS station_id, s.oldcode, s.name_en, s.name_th,
+           s.latitude::float8 AS latitude, s.longitude::float8 AS longitude, s.agency,
+           m.timestamp, m.rain_1h::float8 AS rain_1h, m.rain_24h::float8 AS rain_24h
+    FROM hii_rainfall m
+    JOIN hii_rain_stations s ON s.id = m.station_id
+    JOIN (SELECT station_id, MAX(timestamp) AS latest_ts
+          FROM hii_rainfall WHERE timestamp >= $1
+          GROUP BY station_id) latest
+      ON latest.station_id = m.station_id AND latest.latest_ts = m.timestamp
+    ORDER BY s.id";
+
+const WATERLEVEL_SQL: &str = "
+    SELECT s.id AS station_id, s.oldcode, s.rid_code, s.name_en, s.name_th,
+           s.latitude::float8 AS latitude, s.longitude::float8 AS longitude, s.agency,
+           s.river_name, s.offset_msl::float8 AS offset_msl,
+           s.min_bank_msl::float8 AS min_bank_msl, s.is_key_station,
+           m.timestamp, m.wl_msl::float8 AS wl_msl, m.discharge::float8 AS discharge,
+           m.storage_percent::float8 AS storage_percent, m.situation_level,
+           m.diff_wl_bank::float8 AS diff_wl_bank
+    FROM hii_waterlevel m
+    JOIN hii_wl_stations s ON s.id = m.station_id
+    JOIN (SELECT station_id, MAX(timestamp) AS latest_ts
+          FROM hii_waterlevel WHERE timestamp >= $1
+          GROUP BY station_id) latest
+      ON latest.station_id = m.station_id AND latest.latest_ts = m.timestamp
+    ORDER BY s.id";
+
+async fn latest_feed(
+    st: Shared,
+    feed: &'static str,
+    sql: &'static str,
+    q: Option<String>,
+    h: HeaderMap,
+) -> ApiResult {
+    let a = params::parse(LATEST, &params::query_map(q.as_deref()), &[], &h)?;
+    let hours = a.int("hours");
+    let pool = st.pool.clone();
+    let (body, stale) = st
+        .caches
+        .hii
+        .get(
+            format!("{feed}:{hours}"),
+            secs(st.cfg.web.hii_cache_ttl_s),
+            None,
+            move || {
+                let pool = pool.clone();
+                async move {
+                    let cutoff = hours_before(bangkok_now(), hours)
+                        .ok_or_else(|| ApiError::new(500, "date value out of range"))?;
+                    let rows = rows_json(&sqlx::query(sql).bind(cutoff).fetch_all(&pool).await?);
+                    let keep = !rows.is_empty();
+                    Ok((to_body(&rows), keep))
+                }
+            },
+        )
+        .await?;
+    ok(body, stale)
+}
+
+/// GET /api/hii/rainfall/latest: latest rainfall per HII station (Ping basin).
+pub async fn rainfall_latest(
+    State(st): State<Shared>,
+    RawQuery(q): RawQuery,
+    h: HeaderMap,
+) -> ApiResult {
+    st.metrics.count("hii_rainfall");
+    latest_feed(st, "rain", RAIN_SQL, q, h).await
+}
+
+/// GET /api/hii/waterlevel/latest: latest water level per HII station (m MSL).
+pub async fn waterlevel_latest(
+    State(st): State<Shared>,
+    RawQuery(q): RawQuery,
+    h: HeaderMap,
+) -> ApiResult {
+    st.metrics.count("hii_waterlevel");
+    latest_feed(st, "waterlevel", WATERLEVEL_SQL, q, h).await
+}
+
+/// The upper Ping above P.1 (src/ml/hii_rain.py CATCHMENT_BOX):
+/// lat_lo, lat_hi, lon_lo, lon_hi.
+const CATCHMENT_BOX: [f64; 4] = [18.75, 19.60, 98.60, 99.30];
+const MIN_GAUGES_PER_HOUR: i64 = 5;
+const WINDOW_H: usize = 24;
+
+pub const CATCHMENT: &[Param] = &[Param::int("days", 14).ge(1).le(60)];
+
+/// `hii_rain.load_gauge_mean`: hourly catchment-mean rain_1h; hours with
+/// fewer than MIN_GAUGES_PER_HOUR gauges are NaN.
+async fn gauge_mean(
+    pool: &sqlx::PgPool,
+    start: NaiveDateTime,
+) -> Option<Vec<(NaiveDateTime, f64)>> {
+    let [lat_lo, lat_hi, lon_lo, lon_hi] = CATCHMENT_BOX.map(|x| x.to_string());
+    let rows: Vec<(NaiveDateTime, i64, Option<f64>)> = sqlx::query_as(
+        "SELECT m.timestamp, COUNT(m.rain_1h) AS n, AVG(m.rain_1h)::float8 AS rain_1h
+         FROM hii_rainfall m JOIN hii_rain_stations s ON s.id = m.station_id
+         WHERE s.latitude BETWEEN $1::numeric AND $2::numeric
+           AND s.longitude BETWEEN $3::numeric AND $4::numeric
+           AND m.rain_1h IS NOT NULL AND m.timestamp >= $5
+         GROUP BY m.timestamp ORDER BY m.timestamp",
+    )
+    .bind(lat_lo)
+    .bind(lat_hi)
+    .bind(lon_lo)
+    .bind(lon_hi)
+    .bind(start)
+    .fetch_all(pool)
+    .await
+    .map_err(|e| tracing::warn!("HII gauge rain load failed: {e}"))
+    .ok()?;
+    if rows.is_empty() {
+        return None;
+    }
+    // groupby(floor('h')).agg(n=sum, rain_1h=mean): pandas' group_mean
+    // (Kahan-compensated sum / count)
+    struct Acc {
+        n: i64,
+        sum: f64,
+        comp: f64,
+        nobs: u32,
+    }
+    let mut groups: BTreeMap<NaiveDateTime, Acc> = BTreeMap::new();
+    for (t, n, rain) in rows {
+        let g = groups.entry(floor_hour(t)).or_insert(Acc {
+            n: 0,
+            sum: 0.0,
+            comp: 0.0,
+            nobs: 0,
+        });
+        g.n += n;
+        if let Some(val) = rain.filter(|v| !v.is_nan()) {
+            g.nobs += 1;
+            let y = val - g.comp;
+            let t = g.sum + y;
+            g.comp = t - g.sum - y;
+            if g.comp.is_nan() {
+                g.comp = 0.0;
+            }
+            g.sum = t;
+        }
+    }
+    Some(
+        groups
+            .into_iter()
+            .map(|(t, g)| {
+                let mean = if g.nobs == 0 {
+                    f64::NAN
+                } else {
+                    g.sum / g.nobs as f64
+                };
+                (
+                    t,
+                    if g.n < MIN_GAUGES_PER_HOUR {
+                        f64::NAN
+                    } else {
+                        mean
+                    },
+                )
+            })
+            .collect(),
+    )
+}
+
+/// `hii_rain.compare_with_openmeteo`: agreement of 24 h sums over the hours
+/// both series have.
+fn compare(gauge: &[(NaiveDateTime, f64)], openmeteo: &[(NaiveDateTime, f64)]) -> Value {
+    let om: std::collections::HashMap<NaiveDateTime, f64> = openmeteo.iter().copied().collect();
+    let joined: Vec<(f64, f64)> = gauge
+        .iter()
+        .filter(|(_, g)| !g.is_nan())
+        .filter_map(|(t, g)| om.get(t).filter(|o| !o.is_nan()).map(|o| (*g, *o)))
+        .collect();
+    if joined.is_empty() {
+        return json!({"overlap_hours": 0});
+    }
+    if joined.len() < WINDOW_H {
+        return json!({"overlap_hours": joined.len()});
+    }
+    let gs: Vec<f64> = joined.iter().map(|p| p.0).collect();
+    let os: Vec<f64> = joined.iter().map(|p| p.1).collect();
+    let g = ping_forecast::rolling::rolling_sum(&gs, WINDOW_H)[WINDOW_H - 1..].to_vec();
+    let o = ping_forecast::rolling::rolling_sum(&os, WINDOW_H)[WINDOW_H - 1..].to_vec();
+    let diff: Vec<f64> = o.iter().zip(&g).map(|(o, g)| o - g).collect();
+    let abs: Vec<f64> = diff.iter().map(|d| d.abs()).collect();
+    json!({
+        "overlap_hours": joined.len(),
+        "window_h": WINDOW_H,
+        "gauge_mean_mm": num(Some(numpy_mean(&g))),
+        "openmeteo_mean_mm": num(Some(numpy_mean(&o))),
+        "bias_mm": num(Some(numpy_mean(&diff))),
+        "mae_mm": num(Some(numpy_mean(&abs))),
+        "corr": num(Some(pearson(&g, &o))),
+    })
+}
+
+/// `np.corrcoef(a, b)[0, 1]` (same formula; numpy's BLAS dot may differ in
+/// the last bits). NaN for a constant series, which JSON shows as null.
+fn pearson(a: &[f64], b: &[f64]) -> f64 {
+    let (ma, mb) = (numpy_mean(a), numpy_mean(b));
+    let da: Vec<f64> = a.iter().map(|x| x - ma).collect();
+    let db: Vec<f64> = b.iter().map(|x| x - mb).collect();
+    let dot = |x: &[f64], y: &[f64]| x.iter().zip(y).map(|(p, q)| p * q).sum::<f64>();
+    let fact = (a.len() - 1) as f64;
+    let (cab, caa, cbb) = (
+        dot(&da, &db) / fact,
+        dot(&da, &da) / fact,
+        dot(&db, &db) / fact,
+    );
+    (cab / caa.sqrt() / cbb.sqrt()).clamp(-1.0, 1.0)
+}
+
+fn series_rows(s: &[(NaiveDateTime, f64)]) -> Vec<Value> {
+    s.iter()
+        .map(|(t, v)| {
+            json!({
+                "timestamp": isoformat(*t),
+                "rain_mm": if v.is_nan() { Value::Null } else { json!(py_round(*v, 2)) },
+            })
+        })
+        .collect()
+}
+
+/// GET /api/hii/rainfall/catchment: HII gauges vs the Open-Meteo series the
+/// flood model uses, plus their agreement over the window.
+pub async fn rainfall_catchment(
+    State(st): State<Shared>,
+    RawQuery(q): RawQuery,
+    h: HeaderMap,
+) -> ApiResult {
+    st.metrics.count("hii_rain_catchment");
+    let a = params::parse(CATCHMENT, &params::query_map(q.as_deref()), &[], &h)?;
+    let days = a.int("days");
+    let pool = st.pool.clone();
+    let (body, stale) = st
+        .caches
+        .hii
+        .get(
+            format!("rain_catchment:{days}"),
+            secs(st.cfg.web.hii_cache_ttl_s),
+            None,
+            move || {
+                let pool = pool.clone();
+                async move {
+                    let start = bangkok_now() - TimeDelta::days(days);
+                    let gauge = gauge_mean(&pool, start).await;
+                    let openmeteo: Option<Vec<(NaiveDateTime, f64)>> =
+                        match sqlx::query_as::<_, (NaiveDateTime, Option<f64>)>(
+                            "SELECT timestamp, catchment_mean::float8 FROM openmeteo_rain
+                         WHERE timestamp >= $1 ORDER BY timestamp",
+                        )
+                        .bind(start)
+                        .fetch_all(&pool)
+                        .await
+                        {
+                            Ok(rows) if !rows.is_empty() => Some(
+                                rows.into_iter()
+                                    .map(|(t, v)| (t, v.unwrap_or(f64::NAN)))
+                                    .collect(),
+                            ),
+                            Ok(_) => None,
+                            Err(e) => {
+                                tracing::warn!("openmeteo_rain read failed: {e}");
+                                None
+                            }
+                        };
+                    let comparison = match (&gauge, &openmeteo) {
+                        (Some(g), Some(o)) => compare(g, o),
+                        _ => json!({"overlap_hours": 0}),
+                    };
+                    let body = json!({
+                        "box": CATCHMENT_BOX,
+                        "gauge": gauge.as_deref().map(series_rows).unwrap_or_default(),
+                        "openmeteo": openmeteo.as_deref().map(series_rows).unwrap_or_default(),
+                        "comparison_24h_sums": comparison,
+                    });
+                    Ok((to_body(&body), true))
+                }
+            },
+        )
+        .await?;
+    ok(body, stale)
+}
+
+pub const THAIWATER_URL: &str = "https://twa-api-public.thaiwater.net/v2/waterlevel";
+
+/// `ThaiWaterClient._parse_ping_features`: the Ping-basin sensors.
+pub fn parse_ping_features(payload: &Value) -> Vec<Value> {
+    let empty = Map::new();
+    let obj = |v: Option<&Value>| -> Map<String, Value> {
+        v.and_then(Value::as_object).cloned().unwrap_or_default()
+    };
+    let mut out = vec![];
+    let collections = payload
+        .get("data")
+        .and_then(Value::as_object)
+        .unwrap_or(&empty);
+    for collection in collections.values() {
+        let features = collection.get("features").and_then(Value::as_array);
+        for feature in features.into_iter().flatten() {
+            let props = obj(feature.get("properties"));
+            let basin = obj(props.get("basin"));
+            if basin.get("basin").and_then(Value::as_str) != Some("Ping") {
+                continue;
+            }
+            let geometry = obj(feature.get("geometry"));
+            let coords = geometry
+                .get("coordinates")
+                .and_then(Value::as_array)
+                .cloned()
+                .unwrap_or_default();
+            if coords.len() < 2 {
+                continue;
+            }
+            let station = obj(props.get("station"));
+            let code = station
+                .get("stationCode")
+                .and_then(Value::as_str)
+                .unwrap_or("");
+            let get = |m: &Map<String, Value>, k: &str| m.get(k).cloned().unwrap_or(Value::Null);
+            out.push(json!({
+                "id": format!("thaiwater:{}", py_str(props.get("id").unwrap_or(&Value::Null))),
+                "station_code": code.split_once('-').map_or(code, |(_, rest)| rest),
+                "station_name": get(&station, "station"),
+                "latitude": coords[1],
+                "longitude": coords[0],
+                "timestamp": get(&props, "waterlevelDatetime"),
+                "water_level_msl": get(&props, "waterlevelMsl"),
+                "bank_percent": get(&props, "storagePercent"),
+                "distance_to_bank": get(&props, "diffWlBank"),
+                "river_name": get(&props, "riverName"),
+                "agency": get(&obj(props.get("agency")), "agencyShort"),
+                "source": "ThaiWater",
+            }));
+        }
+    }
+    out
+}
+
+/// GET /sensors/thaiwater: current ThaiWater water-level sensors in the Ping
+/// basin, fetched live (cached for a minute; the feed updates every ~10 min).
+pub async fn thaiwater_sensors(State(st): State<Shared>) -> ApiResult {
+    st.metrics.count("thaiwater_sensors");
+    let Some(key) = st.cfg.web.thaiwater_api_key.clone() else {
+        return Err(ApiError::new(503, "THAIWATER_API_KEY is not configured"));
+    };
+    let (http, timeout) = (st.http.clone(), st.cfg.web.request_timeout_s);
+    let (body, stale) = st
+        .caches
+        .sensors
+        .get("ping".into(), Duration::from_secs(60), None, move || {
+            let (http, key) = (http.clone(), key.clone());
+            async move {
+                let fetch = async {
+                    http.get(THAIWATER_URL)
+                        .header("Accept-Language", "en")
+                        .header("x-api-key", key)
+                        .timeout(Duration::from_secs(timeout))
+                        .send()
+                        .await?
+                        .error_for_status()?
+                        .json::<Value>()
+                        .await
+                };
+                match fetch.await {
+                    Ok(payload) => {
+                        let sensors = parse_ping_features(&payload);
+                        let keep = !sensors.is_empty();
+                        Ok((to_body(&sensors), keep))
+                    }
+                    Err(e) => {
+                        tracing::error!("Error fetching ThaiWater sensors: {e}");
+                        Err(ApiError::new(502, "ThaiWater API unavailable"))
+                    }
+                }
+            }
+        })
+        .await?;
+    ok(body, stale)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn thaiwater_features() {
+        let payload = json!({"data": {"a": {"features": [
+            {"properties": {"id": 2186, "basin": {"basin": "Ping"},
+               "station": {"stationCode": "FOP-MOU298", "station": "Mae Tom Bridge"},
+               "waterlevelDatetime": "2026-10-18T15:10:00+07:00", "waterlevelMsl": 794.41,
+               "storagePercent": -10.76, "diffWlBank": 1.9, "riverName": "Nam Mae Tuen",
+               "agency": {"agencyShort": "FOP"}},
+             "geometry": {"coordinates": [98.36419, 17.799877]}},
+            {"properties": {"id": 1, "basin": {"basin": "Wang"}},
+             "geometry": {"coordinates": [99.0, 18.0]}},
+            {"properties": {"basin": {"basin": "Ping"}}, "geometry": {"coordinates": [99.0]}},
+            {"properties": {"basin": {"basin": "Ping"}}, "geometry": {"coordinates": [99.0, 18.0]}}
+        ]}}});
+        let s = parse_ping_features(&payload);
+        assert_eq!(s.len(), 2);
+        assert_eq!(
+            s[0].to_string(),
+            r#"{"id":"thaiwater:2186","station_code":"MOU298","station_name":"Mae Tom Bridge","latitude":17.799877,"longitude":98.36419,"timestamp":"2026-10-18T15:10:00+07:00","water_level_msl":794.41,"bank_percent":-10.76,"distance_to_bank":1.9,"river_name":"Nam Mae Tuen","agency":"FOP","source":"ThaiWater"}"#
+        );
+        assert_eq!(s[1]["id"], "thaiwater:None");
+        assert_eq!(s[1]["station_code"], "");
+    }
+}

rust/collector/src/web/mod.rs

@@ -0,0 +1,336 @@
+//! The website and read API: the port of the retired FastAPI app.
+//!
+//! Same routes, parameters, JSON fields, status codes, cache headers and
+//! response caches as the Python app had; before the cutover (2026-10-18) a
+//! route-by-route comparison of the two on production data found no
+//! difference. It runs inside `ping-collector run` (WEB_LISTEN)
+//! or alone (`ping-collector serve`). /docs is the same Swagger UI, fed by an
+//! OpenAPI document built from the route table that also validates the
+//! parameters (openapi.rs, params.rs).
+
+mod cache;
+mod data;
+mod error;
+mod forecasts;
+mod hii;
+mod openapi;
+mod params;
+mod pyfmt;
+mod system;
+#[cfg(test)]
+mod tests;
+
+use crate::config::Config;
+use crate::stations::Stations;
+use anyhow::{Context, Result};
+use axum::Router;
+use axum::body::Bytes;
+use axum::extract::{Request, State};
+use axum::handler::HandlerWithoutStateExt;
+use axum::http::{HeaderValue, Method, StatusCode, header};
+use axum::middleware::{self, Next};
+use axum::response::{IntoResponse, Response};
+use axum::routing::{get, post};
+use cache::Swr;
+use serde_json::{Map, Value, json};
+use sqlx::PgPool;
+use std::sync::{Arc, Mutex, RwLock};
+use std::time::{Duration, Instant};
+use tower_http::compression::CompressionLayer;
+use tower_http::compression::predicate::{NotForContentType, Predicate, SizeAbove};
+use tower_http::cors::{AllowHeaders, AllowMethods, AllowOrigin, Any, CorsLayer};
+use tower_http::services::ServeDir;
+
+pub struct AppState {
+    pub cfg: Config,
+    pub pool: PgPool,
+    pub http: reqwest::Client,
+    pub stations: Arc<RwLock<Stations>>,
+    /// The collector's status when it runs in this process; otherwise
+    /// /scraping/status reads SCRAPING_STATUS_FILE, as the Python app did.
+    pub status: Option<Arc<RwLock<Value>>>,
+    /// Wakes the collection loop (POST /scrape/trigger).
+    pub trigger: Option<Arc<tokio::sync::Notify>>,
+    pub dashboard: String,
+    pub metrics: Metrics,
+    pub started: Instant,
+    caches: Caches,
+}
+
+pub type Shared = Arc<AppState>;
+
+struct Caches {
+    latest: Arc<Swr<Bytes>>,
+    history: Arc<Swr<Bytes>>,
+    hii: Arc<Swr<Bytes>>,
+    stats: Arc<Swr<Bytes>>,
+    health: Arc<Swr<Bytes>>,
+    sensors: Arc<Swr<Bytes>>,
+    /// body, older than FORECAST_MAX_AGE_SECONDS
+    forecast: Arc<Swr<(Bytes, bool)>>,
+}
+
+impl AppState {
+    pub fn new(
+        cfg: Config,
+        pool: PgPool,
+        http: reqwest::Client,
+        stations: Arc<RwLock<Stations>>,
+        status: Option<Arc<RwLock<Value>>>,
+        trigger: Option<Arc<tokio::sync::Notify>>,
+    ) -> Shared {
+        let path = cfg.web.static_dir.join("dashboard.html");
+        let dashboard = std::fs::read_to_string(&path).unwrap_or_else(|e| {
+            tracing::error!("could not load the dashboard {}: {e}", path.display());
+            "<h1>Northern Thailand Ping River Monitor API</h1><p>See <code>/docs</code>.</p>".into()
+        });
+        Arc::new(AppState {
+            cfg,
+            pool,
+            http,
+            stations,
+            status,
+            trigger,
+            dashboard,
+            metrics: Metrics::default(),
+            started: Instant::now(),
+            // byte budgets: ~100 MB in all, inside the unit's MemoryMax
+            caches: Caches {
+                latest: Swr::new(4 * MB),
+                history: Swr::new(48 * MB),
+                hii: Swr::new(24 * MB),
+                stats: Swr::new(MB),
+                health: Swr::new(MB),
+                sensors: Swr::new(4 * MB),
+                forecast: Swr::new(4 * MB),
+            },
+        })
+    }
+}
+
+const MB: usize = 1 << 20;
+
+fn secs(s: u64) -> Duration {
+    Duration::from_secs(s)
+}
+
+/// `/metrics`: request counters per endpoint, as the Python app kept them.
+#[derive(Default)]
+pub struct Metrics {
+    counters: Mutex<Map<String, Value>>,
+}
+
+impl Metrics {
+    pub fn count(&self, endpoint: &str) {
+        let key = format!("api_requests{{endpoint={endpoint}}}");
+        let mut c = self.counters.lock().unwrap();
+        let n = c.get(&key).and_then(Value::as_f64).unwrap_or(0.0);
+        c.insert(key, json!(n + 1.0));
+    }
+
+    fn counters(&self) -> Value {
+        Value::Object(self.counters.lock().unwrap().clone())
+    }
+}
+
+/// Cache-Control by path, as the Python app's middleware: lets browsers and
+/// Cloudflare absorb repeat traffic. Max-ages mirror the data cadence.
+const CACHE_CONTROL: &[(&str, &str)] = &[
+    ("/static/", "public, max-age=3600"),
+    ("/measurements/latest", "public, max-age=30"),
+    ("/api/forecast/", "public, max-age=300"),
+    ("/api/hii/", "public, max-age=60"),
+    ("/measurements/history", "public, max-age=300"),
+    ("/forecast", "public, max-age=120"),
+    ("/api/stats", "public, max-age=300"),
+    ("/stations", "public, max-age=300"),
+];
+
+/// Requests reported to Umami (server-side API analytics).
+const UMAMI_PREFIXES: &[&str] = &[
+    "/api/",
+    "/measurements",
+    "/forecast",
+    "/stations",
+    "/sensors",
+];
+
+fn cache_control(path: &str) -> Option<&'static str> {
+    if path == "/" {
+        return Some("public, max-age=120");
+    }
+    CACHE_CONTROL
+        .iter()
+        .find(|(prefix, _)| path.starts_with(prefix))
+        .map(|(_, v)| *v)
+}
+
+async fn finish(State(st): State<Shared>, req: Request, next: Next) -> Response {
+    let path = req.uri().path().to_string();
+    let method = req.method().clone();
+    let (host, user_agent) = {
+        let header = |name: header::HeaderName| {
+            req.headers()
+                .get(name)
+                .and_then(|v| v.to_str().ok())
+                .map(String::from)
+        };
+        (
+            header(header::HOST).unwrap_or_else(|| "water.buildfor.life".into()),
+            header(header::USER_AGENT).unwrap_or_default(),
+        )
+    };
+    let mut resp = next.run(req).await;
+    if method == Method::GET
+        && resp.status() == StatusCode::OK
+        && !resp.headers().contains_key(header::CACHE_CONTROL)
+        && let Some(v) = cache_control(&path)
+    {
+        resp.headers_mut()
+            .insert(header::CACHE_CONTROL, HeaderValue::from_static(v));
+    }
+    let web = &st.cfg.web;
+    if web.umami_track_api
+        && !web.umami_website_id.is_empty()
+        && UMAMI_PREFIXES.iter().any(|p| path.starts_with(p))
+    {
+        let event = json!({
+            "type": "event",
+            "payload": {
+                "website": web.umami_website_id,
+                "url": path,
+                "hostname": host,
+                "name": "api-request",
+                "data": {"method": method.as_str(), "status": resp.status().as_u16()},
+            },
+        });
+        let req = st
+            .http
+            .post(&web.umami_api_url)
+            .timeout(Duration::from_secs(3))
+            .header(
+                header::USER_AGENT,
+                if user_agent.is_empty() {
+                    "api-client".into()
+                } else {
+                    user_agent
+                },
+            )
+            .json(&event);
+        // analytics must never affect the response
+        tokio::spawn(async move {
+            let _ = req.send().await;
+        });
+    }
+    resp
+}
+
+async fn not_found() -> Response {
+    error::ApiError::new(404, "Not Found").into_response()
+}
+
+async fn method_not_allowed() -> Response {
+    error::ApiError::new(405, "Method Not Allowed").into_response()
+}
+
+fn cors(cfg: &Config) -> CorsLayer {
+    // as the Python app: any origin without credentials unless
+    // CORS_ALLOW_ORIGINS lists the allowed ones
+    let origins = &cfg.web.cors_allow_origins;
+    if origins.is_empty() {
+        return CorsLayer::new()
+            .allow_origin(Any)
+            .allow_methods(Any)
+            .allow_headers(Any);
+    }
+    let list: Vec<HeaderValue> = origins.iter().filter_map(|o| o.parse().ok()).collect();
+    CorsLayer::new()
+        .allow_origin(AllowOrigin::list(list))
+        .allow_credentials(true)
+        .allow_methods(AllowMethods::mirror_request())
+        .allow_headers(AllowHeaders::mirror_request())
+}
+
+pub fn router(st: Shared) -> Router {
+    let static_files =
+        ServeDir::new(st.cfg.web.static_dir.clone()).not_found_service(not_found.into_service());
+    // as Starlette's GZipMiddleware(minimum_size=500)
+    let compress = CompressionLayer::new().compress_when(
+        SizeAbove::new(500)
+            .and(NotForContentType::GRPC)
+            .and(NotForContentType::IMAGES)
+            .and(NotForContentType::SSE),
+    );
+    Router::new()
+        .route("/", get(system::root))
+        .route("/robots.txt", get(system::robots))
+        .route("/llms.txt", get(system::llms))
+        .route("/sitemap.xml", get(system::sitemap))
+        .route("/docs", get(openapi::swagger_ui))
+        .route("/redoc", get(openapi::redoc))
+        .route("/openapi.json", get(openapi::spec))
+        .route("/health", get(system::health))
+        .route("/metrics", get(system::metrics))
+        .route(
+            "/stations",
+            get(system::stations).post(system::create_station),
+        )
+        .route(
+            "/stations/{station_id}",
+            get(system::station)
+                .put(system::update_station)
+                .delete(system::delete_station),
+        )
+        .route("/sensors/thaiwater", get(hii::thaiwater_sensors))
+        .route("/api/hii/rainfall/latest", get(hii::rainfall_latest))
+        .route("/api/hii/rainfall/catchment", get(hii::rainfall_catchment))
+        .route("/api/hii/waterlevel/latest", get(hii::waterlevel_latest))
+        .route("/measurements/history/{station_code}", get(data::history))
+        .route("/forecast", get(forecasts::forecast))
+        .route(
+            "/api/forecast/history/{station_code}",
+            get(forecasts::history),
+        )
+        .route("/api/notifications", get(forecasts::notifications))
+        .route("/api/forecast/skill", get(forecasts::skill))
+        .route("/measurements/latest", get(data::latest))
+        .route("/measurements/station/{station_code}", get(data::station))
+        .route("/api/stats", get(data::stats))
+        .route("/scrape/trigger", post(system::trigger))
+        .route("/scraping/status", get(system::scraping_status))
+        .route("/config", get(system::config))
+        .nest_service("/static", static_files)
+        .fallback(not_found)
+        .method_not_allowed_fallback(method_not_allowed)
+        .layer(middleware::from_fn_with_state(st.clone(), finish))
+        .layer(cors(&st.cfg))
+        .layer(compress)
+        .with_state(st)
+}
+
+/// Serve until `shutdown` resolves. While the address is taken (the Python
+/// app still running during a switch-over) binding is retried every 30 s.
+pub async fn serve(
+    st: Shared,
+    listen: String,
+    shutdown: impl std::future::Future<Output = ()> + Send + 'static,
+) -> Result<()> {
+    let mut shutdown = Box::pin(shutdown);
+    let listener = loop {
+        match tokio::net::TcpListener::bind(&listen).await {
+            Ok(l) => break l,
+            Err(e) => {
+                tracing::error!("website: cannot listen on {listen}: {e}; retrying in 30 s");
+                tokio::select! {
+                    _ = tokio::time::sleep(Duration::from_secs(30)) => {}
+                    _ = &mut shutdown => return Ok(()),
+                }
+            }
+        }
+    };
+    tracing::info!("website and API on http://{listen}");
+    axum::serve(listener, router(st))
+        .with_graceful_shutdown(shutdown)
+        .await
+        .context("web server")
+}

rust/collector/src/web/openapi.rs

@@ -0,0 +1,430 @@
+//! /docs, /redoc and /openapi.json. The document is built from the route
+//! table below, whose parameter lists are the same constants the handlers
+//! validate with, so the documented parameters are the accepted ones.
+//! Summaries, operation ids and descriptions follow FastAPI's conventions
+//! (from the Python handler names and docstrings) so /docs reads as before.
+
+use super::params::{Param, python_title};
+use super::{data, forecasts, hii, system};
+use axum::http::header;
+use axum::response::{IntoResponse, Response};
+use serde_json::{Map, Value, json};
+use std::sync::OnceLock;
+
+pub const TITLE: &str = "Northern Thailand Ping River Monitor API";
+const DESCRIPTION: &str =
+    "Real-time water level monitoring system for Northern Thailand's Ping River Basin stations";
+const VERSION: &str = "3.1.3";
+
+enum Resp {
+    Any,
+    Html,
+    Ref(&'static str),
+    ArrayOf(&'static str),
+}
+
+struct Route {
+    method: &'static str,
+    path: &'static str,
+    /// the Python handler's name (summary and operationId)
+    name: &'static str,
+    description: &'static str,
+    params: &'static [Param],
+    body: Option<&'static str>,
+    response: Resp,
+}
+
+const NONE: &[Param] = &[];
+
+const ROUTES: &[Route] = &[
+    Route {
+        method: "get",
+        path: "/",
+        name: "root",
+        description: "Root endpoint with basic dashboard",
+        params: NONE,
+        body: None,
+        response: Resp::Html,
+    },
+    Route {
+        method: "get",
+        path: "/health",
+        name: "get_health",
+        description: "Get system health status",
+        params: NONE,
+        body: None,
+        response: Resp::Ref("HealthResponse"),
+    },
+    Route {
+        method: "get",
+        path: "/metrics",
+        name: "get_metrics",
+        description: "Get application metrics",
+        params: NONE,
+        body: None,
+        response: Resp::Ref("MetricsResponse"),
+    },
+    Route {
+        method: "get",
+        path: "/stations",
+        name: "get_stations",
+        description: "Get list of all monitoring stations",
+        params: NONE,
+        body: None,
+        response: Resp::ArrayOf("StationResponse"),
+    },
+    Route {
+        method: "post",
+        path: "/stations",
+        name: "create_station",
+        description: "Create a new monitoring station",
+        params: system::ADMIN,
+        body: Some("StationCreateModel"),
+        response: Resp::Ref("StationResponse"),
+    },
+    Route {
+        method: "put",
+        path: "/stations/{station_id}",
+        name: "update_station",
+        description: "Update an existing monitoring station",
+        params: system::ADMIN_STATION_ID,
+        body: Some("StationUpdateModel"),
+        response: Resp::Ref("StationResponse"),
+    },
+    Route {
+        method: "delete",
+        path: "/stations/{station_id}",
+        name: "delete_station",
+        description: "Delete a monitoring station",
+        params: system::ADMIN_STATION_ID,
+        body: None,
+        response: Resp::Any,
+    },
+    Route {
+        method: "get",
+        path: "/stations/{station_id}",
+        name: "get_station",
+        description: "Get details of a specific monitoring station",
+        params: system::STATION_ID,
+        body: None,
+        response: Resp::Ref("StationResponse"),
+    },
+    Route {
+        method: "get",
+        path: "/sensors/thaiwater",
+        name: "get_thaiwater_sensors",
+        description: "Get current ThaiWater water-level sensors in the Ping basin.",
+        params: NONE,
+        body: None,
+        response: Resp::Any,
+    },
+    Route {
+        method: "get",
+        path: "/api/hii/rainfall/latest",
+        name: "get_hii_rainfall_latest",
+        description: "Latest rainfall reading per HII station (Ping basin, collected hourly).",
+        params: hii::LATEST,
+        body: None,
+        response: Resp::Any,
+    },
+    Route {
+        method: "get",
+        path: "/api/hii/rainfall/catchment",
+        name: "get_hii_rainfall_catchment",
+        description: "Upper-Ping catchment-mean hourly rain: HII gauges vs the Open-Meteo\nseries the flood model actually uses, plus their agreement over the window.\n\nEvidence-gathering endpoint (docs/FLOOD_FORECASTING.md, HII gauge rain):\nthe gauge table only exists since 2026-08 so it cannot be a training\nfeature yet; this makes the two sources' relationship observable meanwhile.",
+        params: hii::CATCHMENT,
+        body: None,
+        response: Resp::Any,
+    },
+    Route {
+        method: "get",
+        path: "/api/hii/waterlevel/latest",
+        name: "get_hii_waterlevel_latest",
+        description: "Latest water-level reading per HII station (Ping basin, m MSL).",
+        params: hii::LATEST,
+        body: None,
+        response: Resp::Any,
+    },
+    Route {
+        method: "get",
+        path: "/measurements/history/{station_code}",
+        name: "get_postgres_history",
+        description: "Get historical measurements for a station from the configured database.",
+        params: data::HISTORY,
+        body: None,
+        response: Resp::Any,
+    },
+    Route {
+        method: "get",
+        path: "/forecast",
+        name: "get_flood_forecasts",
+        description: "Flood-risk forecasts per station for the 6/12/24 h horizons.",
+        params: NONE,
+        body: None,
+        response: Resp::Any,
+    },
+    Route {
+        method: "get",
+        path: "/api/forecast/history/{station_code}",
+        name: "get_forecast_history",
+        description: "Issued model forecasts for a station — what the model predicted, when.\n\nPopulated by the hourly precompute; enables predicted-vs-actual charts.",
+        params: forecasts::HISTORY,
+        body: None,
+        response: Resp::Any,
+    },
+    Route {
+        method: "get",
+        path: "/api/notifications",
+        name: "get_notifications_config",
+        description: "Public ntfy settings so the dashboard can offer subscribe links.",
+        params: NONE,
+        body: None,
+        response: Resp::Any,
+    },
+    Route {
+        method: "get",
+        path: "/api/forecast/skill",
+        name: "get_forecast_skill",
+        description: "Is the model getting better? Issued forecasts verified against what the\nriver then did, per model version, with a persistence baseline.\n\nRead from forecast_history (what each deployed version predicted, hourly)\njoined to water_measurements; no retraining involved. Cached like the HII\nfeeds because the join is a few hundred correlated subqueries.",
+        params: forecasts::SKILL,
+        body: None,
+        response: Resp::Any,
+    },
+    Route {
+        method: "get",
+        path: "/measurements/latest",
+        name: "get_latest_measurements",
+        description: "Get latest measurements from all stations",
+        params: data::LATEST,
+        body: None,
+        response: Resp::ArrayOf("MeasurementResponse"),
+    },
+    Route {
+        method: "get",
+        path: "/measurements/station/{station_code}",
+        name: "get_station_measurements",
+        description: "Get measurements for a specific station",
+        params: data::STATION,
+        body: None,
+        response: Resp::ArrayOf("MeasurementResponse"),
+    },
+    Route {
+        method: "get",
+        path: "/api/stats",
+        name: "get_database_stats",
+        description: "Get database coverage statistics (totals, date range, hourly coverage)",
+        params: NONE,
+        body: None,
+        response: Resp::Any,
+    },
+    Route {
+        method: "post",
+        path: "/scrape/trigger",
+        name: "trigger_scraping",
+        description: "Trigger manual data scraping",
+        params: system::ADMIN,
+        body: None,
+        response: Resp::Any,
+    },
+    Route {
+        method: "get",
+        path: "/scraping/status",
+        name: "get_scraping_status",
+        description: "Get current scraping status",
+        params: NONE,
+        body: None,
+        response: Resp::Ref("ScrapingStatusResponse"),
+    },
+    Route {
+        method: "get",
+        path: "/config",
+        name: "get_config",
+        description: "Get current configuration (sensitive data masked)",
+        params: system::ADMIN,
+        body: None,
+        response: Resp::Any,
+    },
+];
+
+/// FastAPI's `generate_unique_id`: name + path with non-word characters as
+/// `_`, then the method.
+fn operation_id(r: &Route) -> String {
+    let p: String = format!("{}{}", r.name, r.path)
+        .chars()
+        .map(|c| {
+            if c.is_alphanumeric() || c == '_' {
+                c
+            } else {
+                '_'
+            }
+        })
+        .collect();
+    format!("{p}_{}", r.method)
+}
+
+fn reference(name: &str) -> Value {
+    json!({ "$ref": format!("#/components/schemas/{name}") })
+}
+
+fn operation(r: &Route) -> Value {
+    let op_id = operation_id(r);
+    let ok_content = match r.response {
+        Resp::Html => json!({"text/html": {"schema": {"type": "string"}}}),
+        Resp::Any => json!({"application/json": {"schema": {}}}),
+        Resp::Ref(name) => json!({"application/json": {"schema": reference(name)}}),
+        Resp::ArrayOf(name) => json!({"application/json": {"schema": {
+            "type": "array",
+            "items": reference(name),
+            "title": format!("Response {}", python_title(&op_id.replace('_', " "))),
+        }}}),
+    };
+    let mut responses = Map::new();
+    responses.insert(
+        "200".into(),
+        json!({"description": "Successful Response", "content": ok_content}),
+    );
+    if !r.params.is_empty() || r.body.is_some() {
+        responses.insert(
+            "422".into(),
+            json!({"description": "Validation Error",
+                   "content": {"application/json": {"schema": reference("HTTPValidationError")}}}),
+        );
+    }
+    let mut op = Map::new();
+    op.insert(
+        "summary".into(),
+        json!(python_title(&r.name.replace('_', " "))),
+    );
+    op.insert("description".into(), json!(r.description));
+    op.insert("operationId".into(), json!(op_id));
+    if !r.params.is_empty() {
+        op.insert(
+            "parameters".into(),
+            Value::Array(r.params.iter().map(Param::openapi).collect()),
+        );
+    }
+    if let Some(body) = r.body {
+        op.insert(
+            "requestBody".into(),
+            json!({"required": true, "content": {"application/json": {"schema": reference(body)}}}),
+        );
+    }
+    op.insert("responses".into(), Value::Object(responses));
+    Value::Object(op)
+}
+
+pub fn document() -> &'static str {
+    static DOC: OnceLock<String> = OnceLock::new();
+    DOC.get_or_init(|| {
+        let mut paths = Map::new();
+        for r in ROUTES {
+            let entry = paths
+                .entry(r.path)
+                .or_insert_with(|| Value::Object(Map::new()));
+            entry
+                .as_object_mut()
+                .expect("object")
+                .insert(r.method.into(), operation(r));
+        }
+        let schemas: Value = serde_json::from_str(include_str!("openapi_components.json"))
+            .expect("openapi_components.json is valid JSON");
+        json!({
+            "openapi": "3.1.0",
+            "info": {"title": TITLE, "description": DESCRIPTION, "version": VERSION},
+            "paths": paths,
+            "components": {"schemas": schemas},
+        })
+        .to_string()
+    })
+}
+
+pub async fn spec() -> Response {
+    ([(header::CONTENT_TYPE, "application/json")], document()).into_response()
+}
+
+fn html(body: String) -> Response {
+    ([(header::CONTENT_TYPE, "text/html; charset=utf-8")], body).into_response()
+}
+
+/// The Swagger UI page FastAPI serves (assets from the jsDelivr CDN).
+pub async fn swagger_ui() -> Response {
+    html(format!(
+        r##"<!DOCTYPE html>
+<html>
+<head>
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<link type="text/css" rel="stylesheet" href="https://cdn.jsdelivr.net/npm/swagger-ui-dist@5/swagger-ui.css">
+<title>{TITLE} - Swagger UI</title>
+</head>
+<body>
+<div id="swagger-ui">
+</div>
+<script src="https://cdn.jsdelivr.net/npm/swagger-ui-dist@5/swagger-ui-bundle.js"></script>
+<script>
+const ui = SwaggerUIBundle({{
+    url: '/openapi.json',
+    "dom_id": "#swagger-ui",
+    "layout": "BaseLayout",
+    "deepLinking": true,
+    "showExtensions": true,
+    "showCommonExtensions": true,
+    presets: [
+        SwaggerUIBundle.presets.apis,
+        SwaggerUIBundle.SwaggerUIStandalonePreset
+    ],
+}})
+</script>
+</body>
+</html>
+"##
+    ))
+}
+
+pub async fn redoc() -> Response {
+    html(format!(
+        r#"<!DOCTYPE html>
+<html>
+<head>
+<title>{TITLE} - ReDoc</title>
+<meta charset="utf-8"/>
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<style>body {{ margin: 0; padding: 0; }}</style>
+</head>
+<body>
+<noscript>ReDoc requires Javascript to function. Please enable it to browse the documentation.</noscript>
+<redoc spec-url="/openapi.json"></redoc>
+<script src="https://cdn.jsdelivr.net/npm/redoc@2/bundles/redoc.standalone.js"></script>
+</body>
+</html>
+"#
+    ))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn fastapi_naming() {
+        let doc: Value = serde_json::from_str(document()).unwrap();
+        let op = &doc["paths"]["/measurements/history/{station_code}"]["get"];
+        assert_eq!(
+            op["operationId"],
+            "get_postgres_history_measurements_history__station_code__get"
+        );
+        assert_eq!(op["summary"], "Get Postgres History");
+        assert_eq!(op["parameters"].as_array().unwrap().len(), 5);
+        let latest = &doc["paths"]["/measurements/latest"]["get"]["responses"]["200"];
+        assert_eq!(
+            latest["content"]["application/json"]["schema"]["title"],
+            "Response Get Latest Measurements Measurements Latest Get"
+        );
+        assert!(doc["paths"]["/stations"]["post"]["responses"]["422"].is_object());
+        assert!(
+            doc["paths"]["/health"]["get"]["responses"]
+                .get("422")
+                .is_none()
+        );
+        assert!(doc["components"]["schemas"]["HTTPValidationError"].is_object());
+    }
+}

rust/collector/src/web/openapi_components.json

@@ -0,0 +1,478 @@
+{
+  "HTTPValidationError": {
+    "properties": {
+      "detail": {
+        "items": {
+          "$ref": "#/components/schemas/ValidationError"
+        },
+        "type": "array",
+        "title": "Detail"
+      }
+    },
+    "type": "object",
+    "title": "HTTPValidationError"
+  },
+  "HealthResponse": {
+    "properties": {
+      "overall_status": {
+        "type": "string",
+        "title": "Overall Status"
+      },
+      "timestamp": {
+        "type": "string",
+        "title": "Timestamp"
+      },
+      "checks": {
+        "additionalProperties": {
+          "additionalProperties": true,
+          "type": "object"
+        },
+        "type": "object",
+        "title": "Checks"
+      }
+    },
+    "type": "object",
+    "required": [
+      "overall_status",
+      "timestamp",
+      "checks"
+    ],
+    "title": "HealthResponse"
+  },
+  "MeasurementResponse": {
+    "properties": {
+      "timestamp": {
+        "type": "string",
+        "format": "date-time",
+        "title": "Timestamp"
+      },
+      "station_code": {
+        "type": "string",
+        "title": "Station Code"
+      },
+      "station_name_en": {
+        "type": "string",
+        "title": "Station Name En"
+      },
+      "station_name_th": {
+        "type": "string",
+        "title": "Station Name Th"
+      },
+      "water_level": {
+        "type": "number",
+        "title": "Water Level"
+      },
+      "discharge": {
+        "anyOf": [
+          {
+            "type": "number"
+          },
+          {
+            "type": "null"
+          }
+        ],
+        "title": "Discharge"
+      },
+      "discharge_percent": {
+        "anyOf": [
+          {
+            "type": "number"
+          },
+          {
+            "type": "null"
+          }
+        ],
+        "title": "Discharge Percent"
+      },
+      "status": {
+        "type": "string",
+        "title": "Status",
+        "default": "active"
+      }
+    },
+    "type": "object",
+    "required": [
+      "timestamp",
+      "station_code",
+      "station_name_en",
+      "station_name_th",
+      "water_level"
+    ],
+    "title": "MeasurementResponse"
+  },
+  "MetricsResponse": {
+    "properties": {
+      "counters": {
+        "additionalProperties": {
+          "type": "number"
+        },
+        "type": "object",
+        "title": "Counters"
+      },
+      "gauges": {
+        "additionalProperties": {
+          "type": "number"
+        },
+        "type": "object",
+        "title": "Gauges"
+      },
+      "histograms": {
+        "additionalProperties": {
+          "additionalProperties": {
+            "type": "number"
+          },
+          "type": "object"
+        },
+        "type": "object",
+        "title": "Histograms"
+      }
+    },
+    "type": "object",
+    "required": [
+      "counters",
+      "gauges",
+      "histograms"
+    ],
+    "title": "MetricsResponse"
+  },
+  "ScrapingStatusResponse": {
+    "properties": {
+      "is_running": {
+        "type": "boolean",
+        "title": "Is Running"
+      },
+      "last_run": {
+        "anyOf": [
+          {
+            "type": "string",
+            "format": "date-time"
+          },
+          {
+            "type": "null"
+          }
+        ],
+        "title": "Last Run"
+      },
+      "next_run": {
+        "anyOf": [
+          {
+            "type": "string",
+            "format": "date-time"
+          },
+          {
+            "type": "null"
+          }
+        ],
+        "title": "Next Run"
+      },
+      "total_runs": {
+        "type": "integer",
+        "title": "Total Runs",
+        "default": 0
+      },
+      "successful_runs": {
+        "type": "integer",
+        "title": "Successful Runs",
+        "default": 0
+      },
+      "failed_runs": {
+        "type": "integer",
+        "title": "Failed Runs",
+        "default": 0
+      },
+      "feed_current": {
+        "anyOf": [
+          {
+            "type": "boolean"
+          },
+          {
+            "type": "null"
+          }
+        ],
+        "title": "Feed Current"
+      },
+      "latest_reading": {
+        "anyOf": [
+          {
+            "type": "string",
+            "format": "date-time"
+          },
+          {
+            "type": "null"
+          }
+        ],
+        "title": "Latest Reading"
+      },
+      "readings_saved": {
+        "type": "integer",
+        "title": "Readings Saved",
+        "default": 0
+      }
+    },
+    "type": "object",
+    "required": [
+      "is_running"
+    ],
+    "title": "ScrapingStatusResponse"
+  },
+  "StationCreateModel": {
+    "properties": {
+      "station_code": {
+        "type": "string",
+        "title": "Station Code",
+        "description": "Station code (e.g., P.1, P.20)"
+      },
+      "thai_name": {
+        "type": "string",
+        "title": "Thai Name",
+        "description": "Thai name of the station"
+      },
+      "english_name": {
+        "type": "string",
+        "title": "English Name",
+        "description": "English name of the station"
+      },
+      "latitude": {
+        "anyOf": [
+          {
+            "type": "number",
+            "maximum": 90.0,
+            "minimum": -90.0
+          },
+          {
+            "type": "null"
+          }
+        ],
+        "title": "Latitude",
+        "description": "Latitude coordinate"
+      },
+      "longitude": {
+        "anyOf": [
+          {
+            "type": "number",
+            "maximum": 180.0,
+            "minimum": -180.0
+          },
+          {
+            "type": "null"
+          }
+        ],
+        "title": "Longitude",
+        "description": "Longitude coordinate"
+      },
+      "geohash": {
+        "anyOf": [
+          {
+            "type": "string"
+          },
+          {
+            "type": "null"
+          }
+        ],
+        "title": "Geohash",
+        "description": "Geohash for the location"
+      },
+      "status": {
+        "type": "string",
+        "title": "Status",
+        "description": "Station status",
+        "default": "active"
+      }
+    },
+    "type": "object",
+    "required": [
+      "station_code",
+      "thai_name",
+      "english_name"
+    ],
+    "title": "StationCreateModel"
+  },
+  "StationResponse": {
+    "properties": {
+      "station_id": {
+        "type": "integer",
+        "title": "Station Id"
+      },
+      "station_code": {
+        "type": "string",
+        "title": "Station Code"
+      },
+      "thai_name": {
+        "type": "string",
+        "title": "Thai Name"
+      },
+      "english_name": {
+        "type": "string",
+        "title": "English Name"
+      },
+      "latitude": {
+        "anyOf": [
+          {
+            "type": "number"
+          },
+          {
+            "type": "null"
+          }
+        ],
+        "title": "Latitude"
+      },
+      "longitude": {
+        "anyOf": [
+          {
+            "type": "number"
+          },
+          {
+            "type": "null"
+          }
+        ],
+        "title": "Longitude"
+      },
+      "geohash": {
+        "anyOf": [
+          {
+            "type": "string"
+          },
+          {
+            "type": "null"
+          }
+        ],
+        "title": "Geohash"
+      },
+      "status": {
+        "type": "string",
+        "title": "Status",
+        "default": "active"
+      }
+    },
+    "type": "object",
+    "required": [
+      "station_id",
+      "station_code",
+      "thai_name",
+      "english_name"
+    ],
+    "title": "StationResponse"
+  },
+  "StationUpdateModel": {
+    "properties": {
+      "thai_name": {
+        "anyOf": [
+          {
+            "type": "string"
+          },
+          {
+            "type": "null"
+          }
+        ],
+        "title": "Thai Name",
+        "description": "Thai name of the station"
+      },
+      "english_name": {
+        "anyOf": [
+          {
+            "type": "string"
+          },
+          {
+            "type": "null"
+          }
+        ],
+        "title": "English Name",
+        "description": "English name of the station"
+      },
+      "latitude": {
+        "anyOf": [
+          {
+            "type": "number",
+            "maximum": 90.0,
+            "minimum": -90.0
+          },
+          {
+            "type": "null"
+          }
+        ],
+        "title": "Latitude",
+        "description": "Latitude coordinate"
+      },
+      "longitude": {
+        "anyOf": [
+          {
+            "type": "number",
+            "maximum": 180.0,
+            "minimum": -180.0
+          },
+          {
+            "type": "null"
+          }
+        ],
+        "title": "Longitude",
+        "description": "Longitude coordinate"
+      },
+      "geohash": {
+        "anyOf": [
+          {
+            "type": "string"
+          },
+          {
+            "type": "null"
+          }
+        ],
+        "title": "Geohash",
+        "description": "Geohash for the location"
+      },
+      "status": {
+        "anyOf": [
+          {
+            "type": "string"
+          },
+          {
+            "type": "null"
+          }
+        ],
+        "title": "Status",
+        "description": "Station status"
+      }
+    },
+    "type": "object",
+    "title": "StationUpdateModel"
+  },
+  "ValidationError": {
+    "properties": {
+      "loc": {
+        "items": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "integer"
+            }
+          ]
+        },
+        "type": "array",
+        "title": "Location"
+      },
+      "msg": {
+        "type": "string",
+        "title": "Message"
+      },
+      "type": {
+        "type": "string",
+        "title": "Error Type"
+      },
+      "input": {
+        "title": "Input"
+      },
+      "ctx": {
+        "type": "object",
+        "title": "Context"
+      }
+    },
+    "type": "object",
+    "required": [
+      "loc",
+      "msg",
+      "type"
+    ],
+    "title": "ValidationError"
+  }
+}

rust/collector/src/web/params.rs

@@ -0,0 +1,450 @@
+//! Query/path parameters: one declaration per route drives both the
+//! validation (pydantic-style 422 errors, as FastAPI) and the OpenAPI
+//! document behind /docs, so the two cannot drift apart.
+
+use super::error::ApiError;
+use chrono::{NaiveDate, NaiveDateTime};
+use serde_json::{Map, Value, json};
+use std::collections::HashMap;
+
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub enum In {
+    Query,
+    Path,
+    Header,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub enum Kind {
+    Int,
+    Str,
+    Date,
+}
+
+#[derive(Clone, Copy, Debug)]
+pub enum Default {
+    None,
+    Int(i64),
+    Str(&'static str),
+}
+
+#[derive(Clone, Copy, Debug)]
+pub struct Param {
+    pub name: &'static str,
+    pub loc: In,
+    pub kind: Kind,
+    pub required: bool,
+    /// `Optional[...] = None`
+    pub nullable: bool,
+    pub default: Default,
+    pub ge: Option<i64>,
+    pub le: Option<i64>,
+    pub description: Option<&'static str>,
+}
+
+impl Param {
+    const fn new(name: &'static str, loc: In, kind: Kind) -> Self {
+        Param {
+            name,
+            loc,
+            kind,
+            required: matches!(loc, In::Path),
+            nullable: false,
+            default: Default::None,
+            ge: None,
+            le: None,
+            description: None,
+        }
+    }
+    pub const fn path_str(name: &'static str) -> Self {
+        Self::new(name, In::Path, Kind::Str)
+    }
+    pub const fn path_int(name: &'static str) -> Self {
+        Self::new(name, In::Path, Kind::Int)
+    }
+    pub const fn int(name: &'static str, default: i64) -> Self {
+        Param {
+            default: Default::Int(default),
+            ..Self::new(name, In::Query, Kind::Int)
+        }
+    }
+    pub const fn opt_int(name: &'static str) -> Self {
+        Param {
+            nullable: true,
+            ..Self::new(name, In::Query, Kind::Int)
+        }
+    }
+    pub const fn str(name: &'static str, default: &'static str) -> Self {
+        Param {
+            default: Default::Str(default),
+            ..Self::new(name, In::Query, Kind::Str)
+        }
+    }
+    pub const fn opt_date(name: &'static str, description: &'static str) -> Self {
+        Param {
+            nullable: true,
+            description: Some(description),
+            ..Self::new(name, In::Query, Kind::Date)
+        }
+    }
+    /// The optional `X-API-Key` header of the admin endpoints.
+    pub const fn api_key() -> Self {
+        Param {
+            nullable: true,
+            ..Self::new("X-API-Key", In::Header, Kind::Str)
+        }
+    }
+    pub const fn ge(self, v: i64) -> Self {
+        Param {
+            ge: Some(v),
+            ..self
+        }
+    }
+    pub const fn le(self, v: i64) -> Self {
+        Param {
+            le: Some(v),
+            ..self
+        }
+    }
+
+    fn loc_name(&self) -> &'static str {
+        match self.loc {
+            In::Query => "query",
+            In::Path => "path",
+            In::Header => "header",
+        }
+    }
+
+    /// pydantic's default title: `snake_case` / `X-API-Key` -> `Snake Case` / `X-Api-Key`
+    pub fn title(&self) -> String {
+        python_title(&self.name.replace('_', " "))
+    }
+
+    /// The OpenAPI parameter object, as FastAPI writes it.
+    pub fn openapi(&self) -> Value {
+        let mut base = Map::new();
+        base.insert(
+            "type".into(),
+            json!(match self.kind {
+                Kind::Int => "integer",
+                Kind::Str | Kind::Date => "string",
+            }),
+        );
+        if self.kind == Kind::Date {
+            base.insert("format".into(), json!("date"));
+        }
+        if let Some(le) = self.le {
+            base.insert("maximum".into(), json!(le));
+        }
+        if let Some(ge) = self.ge {
+            base.insert("minimum".into(), json!(ge));
+        }
+        let mut schema = if self.nullable {
+            let mut m = Map::new();
+            m.insert(
+                "anyOf".into(),
+                json!([Value::Object(base), {"type": "null"}]),
+            );
+            m
+        } else {
+            base
+        };
+        match self.default {
+            Default::Int(d) => {
+                schema.insert("default".into(), json!(d));
+            }
+            Default::Str(d) => {
+                schema.insert("default".into(), json!(d));
+            }
+            Default::None => {}
+        }
+        if let Some(d) = self.description {
+            schema.insert("description".into(), json!(d));
+        }
+        schema.insert("title".into(), json!(self.title()));
+        let mut p = Map::new();
+        p.insert("name".into(), json!(self.name));
+        p.insert("in".into(), json!(self.loc_name()));
+        p.insert("required".into(), json!(self.required));
+        p.insert("schema".into(), Value::Object(schema));
+        if let Some(d) = self.description {
+            p.insert("description".into(), json!(d));
+        }
+        Value::Object(p)
+    }
+}
+
+/// Python's `str.title()` for ASCII.
+pub fn python_title(s: &str) -> String {
+    let mut out = String::with_capacity(s.len());
+    let mut prev_letter = false;
+    for c in s.chars() {
+        out.extend(if prev_letter {
+            c.to_lowercase().collect::<Vec<_>>()
+        } else {
+            c.to_uppercase().collect::<Vec<_>>()
+        });
+        prev_letter = c.is_alphabetic();
+    }
+    out
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub enum Val {
+    Int(i64),
+    Str(String),
+    Date(NaiveDate),
+    None,
+}
+
+#[derive(Debug, Default)]
+pub struct Args(HashMap<&'static str, Val>);
+
+impl Args {
+    pub fn int(&self, name: &str) -> i64 {
+        match self.0.get(name) {
+            Some(Val::Int(v)) => *v,
+            other => panic!("parameter {name}: expected an int, got {other:?}"),
+        }
+    }
+    pub fn opt_int(&self, name: &str) -> Option<i64> {
+        match self.0.get(name) {
+            Some(Val::Int(v)) => Some(*v),
+            _ => None,
+        }
+    }
+    pub fn str(&self, name: &str) -> &str {
+        match self.0.get(name) {
+            Some(Val::Str(v)) => v,
+            other => panic!("parameter {name}: expected a string, got {other:?}"),
+        }
+    }
+    pub fn date(&self, name: &str) -> Option<NaiveDate> {
+        match self.0.get(name) {
+            Some(Val::Date(d)) => Some(*d),
+            _ => None,
+        }
+    }
+    /// Python's `str()` of the value (cache keys built like the Python app's).
+    pub fn py_str(&self, name: &str) -> String {
+        match self.0.get(name) {
+            Some(Val::Int(v)) => v.to_string(),
+            Some(Val::Str(s)) => s.clone(),
+            Some(Val::Date(d)) => d.format("%Y-%m-%d").to_string(),
+            Some(Val::None) | None => "None".into(),
+        }
+    }
+}
+
+/// The query string as a map; a repeated name keeps its last value (as
+/// Starlette's `QueryParams.get`).
+pub fn query_map(raw: Option<&str>) -> HashMap<String, String> {
+    form_urlencoded::parse(raw.unwrap_or("").as_bytes())
+        .map(|(k, v)| (k.into_owned(), v.into_owned()))
+        .collect()
+}
+
+fn error(kind: &str, p: &Param, msg: String, input: Value, ctx: Option<Value>) -> Value {
+    let mut e = Map::new();
+    e.insert("type".into(), json!(kind));
+    e.insert("loc".into(), json!([p.loc_name(), p.name]));
+    e.insert("msg".into(), json!(msg));
+    e.insert("input".into(), input);
+    if let Some(c) = ctx {
+        e.insert("ctx".into(), c);
+    }
+    Value::Object(e)
+}
+
+fn parse_date(s: &str) -> Result<NaiveDate, &'static str> {
+    if s.len() < 10 {
+        return Err("input is too short");
+    }
+    if let Ok(d) = NaiveDate::parse_from_str(s, "%Y-%m-%d") {
+        return Ok(d);
+    }
+    // pydantic accepts a datetime at midnight for a date
+    for fmt in [
+        "%Y-%m-%dT%H:%M:%S%.f",
+        "%Y-%m-%dT%H:%M:%S",
+        "%Y-%m-%d %H:%M:%S",
+    ] {
+        if let Ok(t) = NaiveDateTime::parse_from_str(s, fmt) {
+            if t.time() == chrono::NaiveTime::MIN {
+                return Ok(t.date());
+            }
+            return Err("Datetimes provided to dates should have zero time - e.g. be exact dates");
+        }
+    }
+    Err("invalid date format, expected YYYY-MM-DD")
+}
+
+/// Validate every declared parameter; all problems are reported at once.
+pub fn parse(
+    spec: &[Param],
+    query: &HashMap<String, String>,
+    path: &[(&str, &str)],
+    headers: &axum::http::HeaderMap,
+) -> Result<Args, ApiError> {
+    let mut args = Args::default();
+    let mut errors = vec![];
+    for p in spec {
+        let raw: Option<String> = match p.loc {
+            In::Query => query.get(p.name).cloned(),
+            In::Path => path
+                .iter()
+                .find(|(k, _)| *k == p.name)
+                .map(|(_, v)| v.to_string()),
+            In::Header => headers
+                .get(p.name)
+                .and_then(|v| v.to_str().ok())
+                .map(String::from),
+        };
+        let Some(raw) = raw else {
+            if p.required {
+                errors.push(error(
+                    "missing",
+                    p,
+                    "Field required".into(),
+                    Value::Null,
+                    None,
+                ));
+                continue;
+            }
+            let v = match p.default {
+                Default::Int(d) => Val::Int(d),
+                Default::Str(d) => Val::Str(d.into()),
+                Default::None => Val::None,
+            };
+            args.0.insert(p.name, v);
+            continue;
+        };
+        let v = match p.kind {
+            Kind::Str => Val::Str(raw),
+            Kind::Int => match raw.trim().parse::<i64>() {
+                Err(_) => {
+                    errors.push(error(
+                        "int_parsing",
+                        p,
+                        "Input should be a valid integer, unable to parse string as an integer"
+                            .into(),
+                        json!(raw),
+                        None,
+                    ));
+                    continue;
+                }
+                Ok(v) => {
+                    if let Some(ge) = p.ge
+                        && v < ge
+                    {
+                        errors.push(error(
+                            "greater_than_equal",
+                            p,
+                            format!("Input should be greater than or equal to {ge}"),
+                            json!(raw),
+                            Some(json!({ "ge": ge })),
+                        ));
+                        continue;
+                    }
+                    if let Some(le) = p.le
+                        && v > le
+                    {
+                        errors.push(error(
+                            "less_than_equal",
+                            p,
+                            format!("Input should be less than or equal to {le}"),
+                            json!(raw),
+                            Some(json!({ "le": le })),
+                        ));
+                        continue;
+                    }
+                    Val::Int(v)
+                }
+            },
+            Kind::Date => match parse_date(&raw) {
+                Ok(d) => Val::Date(d),
+                Err(why) => {
+                    errors.push(error(
+                        "date_from_datetime_parsing",
+                        p,
+                        format!("Input should be a valid date or datetime, {why}"),
+                        json!(raw),
+                        Some(json!({ "error": why })),
+                    ));
+                    continue;
+                }
+            },
+        };
+        args.0.insert(p.name, v);
+    }
+    if errors.is_empty() {
+        Ok(args)
+    } else {
+        Err(ApiError::validation(errors))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    const SPEC: &[Param] = &[
+        Param::path_str("station_code"),
+        Param::int("hours", 168).ge(1),
+        Param::int("limit", 50000).ge(1).le(100000),
+        Param::opt_date("start", "First day (overrides hours)"),
+        Param::opt_int("horizon").ge(1).le(48),
+    ];
+
+    fn run(q: &str) -> Result<Args, ApiError> {
+        parse(
+            SPEC,
+            &query_map(Some(q)),
+            &[("station_code", "P.1")],
+            &axum::http::HeaderMap::new(),
+        )
+    }
+
+    #[test]
+    fn defaults_and_values() {
+        let a = run("hours=6&start=2026-10-15&hours=12").unwrap();
+        assert_eq!(a.int("hours"), 12); // last wins
+        assert_eq!(a.int("limit"), 50000);
+        assert_eq!(a.str("station_code"), "P.1");
+        assert_eq!(a.date("start"), NaiveDate::from_ymd_opt(2026, 10, 15));
+        assert_eq!(a.opt_int("horizon"), None);
+        assert_eq!(a.py_str("horizon"), "None");
+    }
+
+    #[test]
+    fn errors_like_fastapi() {
+        // FastAPI: {"detail":[{"type":"greater_than_equal","loc":["query","hours"],
+        // "msg":"Input should be greater than or equal to 1","input":"0","ctx":{"ge":1}}]}
+        let e = run("hours=0&limit=x").unwrap_err();
+        assert_eq!(e.status.as_u16(), 422);
+        assert_eq!(
+            e.detail,
+            json!([
+                {"type": "greater_than_equal", "loc": ["query", "hours"],
+                 "msg": "Input should be greater than or equal to 1", "input": "0", "ctx": {"ge": 1}},
+                {"type": "int_parsing", "loc": ["query", "limit"],
+                 "msg": "Input should be a valid integer, unable to parse string as an integer",
+                 "input": "x"}
+            ])
+        );
+    }
+
+    #[test]
+    fn openapi_parameter_objects() {
+        assert_eq!(
+            SPEC[2].openapi().to_string(),
+            r#"{"name":"limit","in":"query","required":false,"schema":{"type":"integer","maximum":100000,"minimum":1,"default":50000,"title":"Limit"}}"#
+        );
+        assert_eq!(
+            SPEC[4].openapi().to_string(),
+            r#"{"name":"horizon","in":"query","required":false,"schema":{"anyOf":[{"type":"integer","maximum":48,"minimum":1},{"type":"null"}],"title":"Horizon"}}"#
+        );
+        assert_eq!(Param::api_key().title(), "X-Api-Key");
+        assert_eq!(python_title("station code"), "Station Code");
+    }
+}

rust/collector/src/web/pyfmt.rs

@@ -0,0 +1,158 @@
+//! Output conventions of the Python app that the API keeps: `isoformat()`,
+//! `round()`, numpy's summation order and Python's `str()` of JSON scalars.
+
+use chrono::{NaiveDate, NaiveDateTime, Timelike};
+use serde_json::Value;
+
+/// `datetime.isoformat()`: microseconds only when they are not zero.
+pub fn isoformat(t: NaiveDateTime) -> String {
+    let base = t.format("%Y-%m-%dT%H:%M:%S");
+    match t.nanosecond() / 1000 {
+        0 => base.to_string(),
+        us => format!("{base}.{us:06}"),
+    }
+}
+
+/// `str(datetime)`: as `isoformat()` with a space instead of the `T`.
+pub fn py_str_datetime(t: NaiveDateTime) -> String {
+    isoformat(t).replacen('T', " ", 1)
+}
+
+/// `datetime.combine(d, datetime.max.time())`
+pub fn end_of_day(d: NaiveDate) -> NaiveDateTime {
+    d.and_hms_micro_opt(23, 59, 59, 999_999)
+        .expect("valid time")
+}
+
+pub fn start_of_day(d: NaiveDate) -> NaiveDateTime {
+    d.and_hms_opt(0, 0, 0).expect("valid time")
+}
+
+/// `round(x, ndigits)` for floats: the correctly rounded decimal, ties to
+/// even (Rust's fixed-precision formatting rounds the same way).
+pub fn py_round(x: f64, ndigits: usize) -> f64 {
+    if !x.is_finite() {
+        return x;
+    }
+    format!("{x:.ndigits$}")
+        .parse()
+        .expect("formatted float parses")
+}
+
+/// numpy's `np.add.reduce` on a contiguous float64 array (pairwise
+/// summation, 8 accumulators per block of up to 128), which pandas'
+/// `Series.mean()` uses without bottleneck.
+pub fn numpy_sum(a: &[f64]) -> f64 {
+    0.0 + pairwise(a)
+}
+
+fn pairwise(a: &[f64]) -> f64 {
+    let n = a.len();
+    if n < 8 {
+        let mut res = -0.0;
+        for x in a {
+            res += x;
+        }
+        res
+    } else if n <= 128 {
+        let mut r = [0.0; 8];
+        r.copy_from_slice(&a[..8]);
+        let mut i = 8;
+        while i < n - (n % 8) {
+            for j in 0..8 {
+                r[j] += a[i + j];
+            }
+            i += 8;
+        }
+        let mut res = ((r[0] + r[1]) + (r[2] + r[3])) + ((r[4] + r[5]) + (r[6] + r[7]));
+        while i < n {
+            res += a[i];
+            i += 1;
+        }
+        res
+    } else {
+        let mut n2 = n / 2;
+        n2 -= n2 % 8;
+        pairwise(&a[..n2]) + pairwise(&a[n2..])
+    }
+}
+
+/// `Series.mean()` (NaN-free input).
+pub fn numpy_mean(a: &[f64]) -> f64 {
+    numpy_sum(a) / a.len() as f64
+}
+
+/// `str(x)` of a JSON scalar as Python would print the decoded value.
+pub fn py_str(v: &Value) -> String {
+    match v {
+        Value::Null => "None".into(),
+        Value::Bool(true) => "True".into(),
+        Value::Bool(false) => "False".into(),
+        Value::String(s) => s.clone(),
+        other => other.to_string(),
+    }
+}
+
+/// A float as JSON (`NaN` and infinities, which Python refuses to encode,
+/// become null).
+pub fn num(x: Option<f64>) -> Value {
+    x.filter(|x| x.is_finite()).map_or(Value::Null, Value::from)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn isoformat_drops_zero_microseconds() {
+        let d = NaiveDate::from_ymd_opt(2026, 10, 18).unwrap();
+        assert_eq!(
+            isoformat(d.and_hms_opt(14, 0, 0).unwrap()),
+            "2026-10-18T14:00:00"
+        );
+        assert_eq!(
+            isoformat(d.and_hms_micro_opt(15, 30, 38, 454_399).unwrap()),
+            "2026-10-18T15:30:38.454399"
+        );
+        assert_eq!(isoformat(end_of_day(d)), "2026-10-18T23:59:59.999999");
+        assert_eq!(
+            py_str_datetime(d.and_hms_opt(21, 0, 0).unwrap()),
+            "2026-10-18 21:00:00"
+        );
+    }
+
+    #[test]
+    fn rounding_matches_python() {
+        // values from CPython 3.11: round(x, n)
+        for (x, n, want) in [
+            (0.125, 2, 0.12),
+            (0.375, 2, 0.38),
+            (2.675, 2, 2.67),
+            (1.0005, 3, 1.0),
+            (0.0875, 3, 0.087),
+            (-0.0049, 2, -0.0),
+            (93.35, 1, 93.3),
+            (24.6, 2, 24.6),
+            (1.2345e-7, 3, 0.0),
+        ] {
+            assert_eq!(py_round(x, n), want, "round({x}, {n})");
+        }
+    }
+
+    #[test]
+    fn numpy_pairwise_sum_order() {
+        // np.add.reduce / pd.Series.mean (numpy 1.26), where a naive loop
+        // gives 20.000000000000014, 1027.6428571428573, 1481.0200000000002
+        assert_eq!(numpy_sum(&[0.1; 200]), 20.0);
+        let series = |n: usize| -> Vec<f64> {
+            (0..n)
+                .map(|i| ((i * 7919) % 1000) as f64 / 7.0 * 0.013 + 0.1)
+                .collect()
+        };
+        assert_eq!(numpy_sum(&series(1000)), 1027.642857142857);
+        assert_eq!(numpy_sum(&series(1441)), 1481.02);
+        assert_eq!(numpy_mean(&series(1441)), 1.0277723802914642);
+        assert_eq!(numpy_sum(&series(300)), 308.84999999999997);
+        assert_eq!(numpy_sum(&[]), 0.0);
+    }
+}

rust/collector/src/web/system.rs

@@ -0,0 +1,635 @@
+//! The dashboard, health, metrics, collector status and the admin endpoints.
+
+use super::Shared;
+use super::error::{ApiError, ApiResult, ok, ok_json, to_body};
+use super::params::{self, Param};
+use super::pyfmt::{isoformat, py_str_datetime};
+use crate::stations::StationInfo;
+use crate::util::bangkok_now;
+use axum::body::Bytes;
+use axum::extract::{Path, State};
+use axum::http::{HeaderMap, StatusCode, header};
+use axum::response::{IntoResponse, Response};
+use serde_json::{Map, Value, json};
+use std::time::{Duration, Instant};
+
+/// GET /: the dashboard (src/static/dashboard.html, loaded at start).
+pub async fn root(State(st): State<Shared>) -> Response {
+    (
+        [(header::CONTENT_TYPE, "text/html; charset=utf-8")],
+        st.dashboard.clone(),
+    )
+        .into_response()
+}
+
+async fn static_file(st: &Shared, name: &str, content_type: &'static str) -> Response {
+    match tokio::fs::read(st.cfg.web.static_dir.join(name)).await {
+        Ok(bytes) => ([(header::CONTENT_TYPE, content_type)], bytes).into_response(),
+        Err(_) => ApiError::new(404, "Not Found").into_response(),
+    }
+}
+
+/// Crawler/indexing files at the domain root.
+pub async fn robots(State(st): State<Shared>) -> Response {
+    static_file(&st, "robots.txt", "text/plain; charset=utf-8").await
+}
+
+pub async fn llms(State(st): State<Shared>) -> Response {
+    static_file(&st, "llms.txt", "text/plain; charset=utf-8").await
+}
+
+pub async fn sitemap(State(st): State<Shared>) -> Response {
+    static_file(&st, "sitemap.xml", "application/xml").await
+}
+
+// ---------------------------------------------------------------- /health
+
+struct Check {
+    status: &'static str,
+    message: String,
+}
+
+fn check(status: &'static str, message: impl Into<String>) -> Check {
+    Check {
+        status,
+        message: message.into(),
+    }
+}
+
+/// Data freshness. As the Python check it judges the newest reading of the
+/// first station (`get_latest_measurements(limit=1)`: P.1) against 2 h. One
+/// change, as in the notifier: while RID's report lags (2-10 h at night) but
+/// the ThaiWater telemetry of the same gauges is current, the data is current.
+async fn database_check(st: &Shared) -> Check {
+    let latest: Result<Option<(chrono::NaiveDateTime,)>, _> = sqlx::query_as(
+        "SELECT m.timestamp
+         FROM water_measurements m
+         JOIN stations s ON m.station_id = s.id
+         INNER JOIN (
+             SELECT station_id, MAX(timestamp) as max_timestamp
+             FROM water_measurements GROUP BY station_id
+         ) latest ON m.station_id = latest.station_id AND m.timestamp = latest.max_timestamp
+         ORDER BY s.station_code LIMIT 1",
+    )
+    .fetch_optional(&st.pool)
+    .await;
+    let latest = match latest {
+        Ok(l) => l.map(|(t,)| t),
+        Err(e) => return check("unhealthy", format!("Database check failed: {e}")),
+    };
+    let now = bangkok_now();
+    let max_age = chrono::TimeDelta::hours(2);
+    if let Some(ts) = latest
+        && now - ts > max_age
+    {
+        let codes = &ping_forecast::config::config().hii_fill_stations;
+        if let Ok(Some(hii)) = crate::db::latest_hii_timestamp(&st.pool, codes).await
+            && now - hii <= max_age
+        {
+            return check(
+                "healthy",
+                format!(
+                    "Database connection OK; RID report lags (latest {}), ThaiWater telemetry current (latest {})",
+                    py_str_datetime(ts),
+                    py_str_datetime(hii)
+                ),
+            );
+        }
+        return check(
+            "degraded",
+            format!("Latest data is old: {}", py_str_datetime(ts)),
+        );
+    }
+    check("healthy", "Database connection OK")
+}
+
+/// The RID API answers (5 s, a liveness probe must fail fast).
+async fn api_check(st: &Shared) -> Check {
+    match st
+        .http
+        .get(&st.cfg.rid_api_url)
+        .timeout(Duration::from_secs(5))
+        .send()
+        .await
+    {
+        Ok(r) if r.status() == StatusCode::OK => check("healthy", "API connection OK"),
+        Ok(r) => check(
+            "degraded",
+            format!("API returned status {}", r.status().as_u16()),
+        ),
+        Err(e) => check("unhealthy", format!("API check failed: {e}")),
+    }
+}
+
+/// Resident memory of this process in MB (Linux).
+pub fn rss_mb() -> Option<f64> {
+    let status = std::fs::read_to_string("/proc/self/status").ok()?;
+    let kb: f64 = status
+        .lines()
+        .find(|l| l.starts_with("VmRSS:"))?
+        .split_whitespace()
+        .nth(1)?
+        .parse()
+        .ok()?;
+    Some(kb / 1024.0)
+}
+
+async fn memory_check() -> Check {
+    match rss_mb() {
+        Some(mb) if mb > 1000.0 => check("degraded", format!("High memory usage: {mb:.1}MB")),
+        Some(mb) => check("healthy", format!("Memory usage OK: {mb:.1}MB")),
+        None => check("healthy", "Memory check skipped (no /proc/self/status)"),
+    }
+}
+
+async fn timed<F: std::future::Future<Output = Check>>(
+    f: F,
+) -> (Check, f64, chrono::NaiveDateTime) {
+    let t = Instant::now();
+    let c = f.await;
+    (c, t.elapsed().as_secs_f64() * 1000.0, bangkok_now())
+}
+
+async fn compute_health(st: Shared) -> super::cache::Computed<Bytes> {
+    let results = [
+        ("database", timed(database_check(&st)).await),
+        ("api", timed(api_check(&st)).await),
+        ("memory", timed(memory_check()).await),
+    ];
+    let statuses: Vec<&str> = results.iter().map(|(_, (c, _, _))| c.status).collect();
+    let overall = if statuses.contains(&"unhealthy") {
+        "unhealthy"
+    } else if statuses.contains(&"degraded") {
+        "degraded"
+    } else {
+        "healthy"
+    };
+    let mut checks = Map::new();
+    for (name, (c, ms, at)) in results {
+        checks.insert(
+            name.into(),
+            json!({
+                "status": c.status,
+                "message": c.message,
+                "response_time_ms": ms,
+                "timestamp": isoformat(at),
+            }),
+        );
+    }
+    let body = json!({
+        "overall_status": overall,
+        "timestamp": isoformat(bangkok_now()),
+        "checks": checks,
+    });
+    Ok((to_body(&body), true))
+}
+
+/// GET /health: cached briefly (it probes the RID API), never older than 4 TTLs.
+pub async fn health(State(st): State<Shared>) -> ApiResult {
+    st.metrics.count("health");
+    let ttl = st.cfg.web.health_cache_ttl_s;
+    let me = st.clone();
+    let (body, _) = st
+        .caches
+        .health
+        .get(
+            "health".into(),
+            Duration::from_secs(ttl),
+            Some(Duration::from_secs(4 * ttl)),
+            move || compute_health(me.clone()),
+        )
+        .await?;
+    ok(body, false)
+}
+
+/// GET /metrics
+pub async fn metrics(State(st): State<Shared>) -> ApiResult {
+    st.metrics.count("metrics");
+    let mut gauges = Map::new();
+    gauges.insert(
+        "uptime_seconds".into(),
+        json!(st.started.elapsed().as_secs_f64()),
+    );
+    if let Some(mb) = rss_mb() {
+        gauges.insert("memory_rss_mb".into(), json!(mb));
+    }
+    gauges.insert("db_pool_connections".into(), json!(st.pool.size() as f64));
+    ok_json(&json!({
+        "counters": st.metrics.counters(),
+        "gauges": gauges,
+        "histograms": {},
+    }))
+}
+
+// -------------------------------------------------------------- stations
+
+fn station_json(id: u32, s: &StationInfo, geohash: bool, status: &str) -> Value {
+    json!({
+        "station_id": id,
+        "station_code": s.code,
+        "thai_name": s.thai_name,
+        "english_name": s.english_name,
+        "latitude": s.latitude,
+        "longitude": s.longitude,
+        "geohash": if geohash { json!(s.geohash) } else { Value::Null },
+        "status": status,
+    })
+}
+
+/// GET /stations: the RID gauges (report column -> station).
+pub async fn stations(State(st): State<Shared>) -> ApiResult {
+    st.metrics.count("stations");
+    let list: Vec<Value> = st
+        .stations
+        .read()
+        .unwrap()
+        .iter()
+        .map(|(id, s)| station_json(*id, s, false, "active"))
+        .collect();
+    ok_json(&list)
+}
+
+pub const STATION_ID: &[Param] = &[Param::path_int("station_id")];
+pub const ADMIN_STATION_ID: &[Param] = &[Param::path_int("station_id"), Param::api_key()];
+pub const ADMIN: &[Param] = &[Param::api_key()];
+
+fn station_id(id: &str, h: &HeaderMap) -> Result<u32, ApiError> {
+    let a = params::parse(STATION_ID, &Default::default(), &[("station_id", id)], h)?;
+    // ids beyond u32 cannot exist
+    Ok(u32::try_from(a.int("station_id")).unwrap_or(u32::MAX))
+}
+
+fn not_found() -> ApiError {
+    ApiError::new(404, "Station not found")
+}
+
+/// GET /stations/{station_id}
+pub async fn station(State(st): State<Shared>, Path(id): Path<String>, h: HeaderMap) -> ApiResult {
+    st.metrics.count("get_station");
+    let id = station_id(&id, &h)?;
+    let stations = st.stations.read().unwrap();
+    let s = stations.info(id).ok_or_else(not_found)?;
+    ok_json(&station_json(id, s, true, "active"))
+}
+
+/// `require_admin_key`: the X-API-Key header must match ADMIN_API_KEY;
+/// with no key configured the admin endpoints are disabled.
+fn require_admin(st: &Shared, h: &HeaderMap) -> Result<(), ApiError> {
+    let Some(want) = &st.cfg.web.admin_api_key else {
+        return Err(ApiError::new(
+            503,
+            "Admin API disabled: ADMIN_API_KEY is not configured on the server",
+        ));
+    };
+    let given = h.get("x-api-key").map(|v| v.as_bytes()).unwrap_or_default();
+    // constant time, as secrets.compare_digest
+    let same = given.len() == want.len()
+        && given
+            .iter()
+            .zip(want.as_bytes())
+            .fold(0u8, |acc, (a, b)| acc | (a ^ b))
+            == 0;
+    if same {
+        Ok(())
+    } else {
+        Err(ApiError::new(401, "Invalid or missing X-API-Key"))
+    }
+}
+
+fn body_object(body: &Bytes) -> Result<Map<String, Value>, ApiError> {
+    match serde_json::from_slice::<Value>(body) {
+        Ok(Value::Object(m)) => Ok(m),
+        Ok(other) => Err(ApiError::validation(vec![json!({
+            "type": "model_attributes_type", "loc": ["body"],
+            "msg": "Input should be a valid dictionary or object to extract fields from",
+            "input": other,
+        })])),
+        Err(e) => Err(ApiError::validation(vec![json!({
+            "type": "json_invalid", "loc": ["body", e.column()],
+            "msg": "JSON decode error", "input": {}, "ctx": {"error": e.to_string()},
+        })])),
+    }
+}
+
+/// Field checks of StationCreateModel / StationUpdateModel.
+struct Fields<'a> {
+    body: &'a Map<String, Value>,
+    errors: Vec<Value>,
+}
+
+impl<'a> Fields<'a> {
+    fn err(&mut self, kind: &str, field: &str, msg: String, ctx: Option<Value>) {
+        let mut e = json!({
+            "type": kind, "loc": ["body", field], "msg": msg,
+            "input": self.body.get(field).cloned().unwrap_or(Value::Object(self.body.clone())),
+        });
+        if let Some(c) = ctx {
+            e["ctx"] = c;
+        }
+        self.errors.push(e);
+    }
+
+    fn string(&mut self, field: &str, required: bool) -> Option<String> {
+        match self.body.get(field) {
+            None if required => {
+                self.err("missing", field, "Field required".into(), None);
+                None
+            }
+            None | Some(Value::Null) if !required => None,
+            Some(Value::String(s)) => Some(s.clone()),
+            _ => {
+                self.err(
+                    "string_type",
+                    field,
+                    "Input should be a valid string".into(),
+                    None,
+                );
+                None
+            }
+        }
+    }
+
+    fn coord(&mut self, field: &str, limit: f64) -> Option<f64> {
+        let v = match self.body.get(field) {
+            None | Some(Value::Null) => return None,
+            Some(Value::Number(n)) => n.as_f64(),
+            Some(Value::String(s)) => s.trim().parse().ok(),
+            _ => None,
+        };
+        let Some(x) = v else {
+            self.err(
+                "float_parsing",
+                field,
+                "Input should be a valid number, unable to parse string as a number".into(),
+                None,
+            );
+            return None;
+        };
+        if x < -limit {
+            self.err(
+                "greater_than_equal",
+                field,
+                format!("Input should be greater than or equal to {}", -limit),
+                Some(json!({"ge": -limit})),
+            );
+        } else if x > limit {
+            self.err(
+                "less_than_equal",
+                field,
+                format!("Input should be less than or equal to {limit}"),
+                Some(json!({"le": limit})),
+            );
+        }
+        Some(x)
+    }
+
+    fn finish(self) -> Result<(), ApiError> {
+        if self.errors.is_empty() {
+            Ok(())
+        } else {
+            Err(ApiError::validation(self.errors))
+        }
+    }
+}
+
+fn persist(st: &Shared) -> bool {
+    let stations = st.stations.read().unwrap();
+    match stations.save(&st.cfg.station_config_path) {
+        Ok(()) => true,
+        Err(e) => {
+            tracing::error!(
+                "Failed to persist station mapping to {}: {e:#}",
+                st.cfg.station_config_path.display()
+            );
+            false
+        }
+    }
+}
+
+/// POST /stations (admin)
+pub async fn create_station(State(st): State<Shared>, h: HeaderMap, body: Bytes) -> ApiResult {
+    st.metrics.count("create_station");
+    require_admin(&st, &h)?;
+    let body = body_object(&body)?;
+    let mut f = Fields {
+        body: &body,
+        errors: vec![],
+    };
+    let code = f.string("station_code", true);
+    let thai = f.string("thai_name", true);
+    let english = f.string("english_name", true);
+    let latitude = f.coord("latitude", 90.0);
+    let longitude = f.coord("longitude", 180.0);
+    let geohash = f.string("geohash", false);
+    let status = f.string("status", false).unwrap_or_else(|| "active".into());
+    f.finish()?;
+    let info = StationInfo {
+        code: code.unwrap_or_default(),
+        thai_name: thai.unwrap_or_default(),
+        english_name: english.unwrap_or_default(),
+        latitude,
+        longitude,
+        geohash,
+    };
+    let id = {
+        let mut stations = st.stations.write().unwrap();
+        let id = stations.next_id();
+        stations.insert(id, info.clone());
+        id
+    };
+    if !persist(&st) {
+        st.stations.write().unwrap().remove(id);
+        return Err(ApiError::new(500, "Failed to persist new station"));
+    }
+    tracing::info!("Created new station: {} ({})", info.code, info.english_name);
+    ok_json(&station_json(id, &info, true, &status))
+}
+
+/// PUT /stations/{station_id} (admin)
+pub async fn update_station(
+    State(st): State<Shared>,
+    Path(id): Path<String>,
+    h: HeaderMap,
+    body: Bytes,
+) -> ApiResult {
+    st.metrics.count("update_station");
+    require_admin(&st, &h)?;
+    let id = station_id(&id, &h)?;
+    let body = body_object(&body)?;
+    let mut f = Fields {
+        body: &body,
+        errors: vec![],
+    };
+    let thai = f.string("thai_name", false);
+    let english = f.string("english_name", false);
+    let latitude = f.coord("latitude", 90.0);
+    let longitude = f.coord("longitude", 180.0);
+    let geohash = f.string("geohash", false);
+    let status = f.string("status", false);
+    f.finish()?;
+    let (original, updated) = {
+        let mut stations = st.stations.write().unwrap();
+        let s = stations.info_mut(id).ok_or_else(not_found)?;
+        let original = s.clone();
+        if let Some(v) = thai {
+            s.thai_name = v;
+        }
+        if let Some(v) = english {
+            s.english_name = v;
+        }
+        if latitude.is_some() {
+            s.latitude = latitude;
+        }
+        if longitude.is_some() {
+            s.longitude = longitude;
+        }
+        if geohash.is_some() {
+            s.geohash = geohash;
+        }
+        (original, s.clone())
+    };
+    if !persist(&st) {
+        st.stations.write().unwrap().insert(id, original);
+        return Err(ApiError::new(500, "Failed to persist station update"));
+    }
+    tracing::info!("Updated station {id}: {}", updated.code);
+    ok_json(&station_json(
+        id,
+        &updated,
+        true,
+        status
+            .as_deref()
+            .filter(|s| !s.is_empty())
+            .unwrap_or("active"),
+    ))
+}
+
+/// DELETE /stations/{station_id} (admin)
+pub async fn delete_station(
+    State(st): State<Shared>,
+    Path(id): Path<String>,
+    h: HeaderMap,
+) -> ApiResult {
+    st.metrics.count("delete_station");
+    require_admin(&st, &h)?;
+    let id = station_id(&id, &h)?;
+    let removed = st
+        .stations
+        .write()
+        .unwrap()
+        .remove(id)
+        .ok_or_else(not_found)?;
+    if !persist(&st) {
+        st.stations.write().unwrap().insert(id, removed);
+        return Err(ApiError::new(500, "Failed to persist station deletion"));
+    }
+    tracing::info!("Deleted station {id}: {}", removed.code);
+    ok_json(&json!({"message": format!("Station {} deleted successfully", removed.code)}))
+}
+
+// ------------------------------------------------------------ collection
+
+/// GET /scraping/status: the collector's last cycle (in this process, or
+/// SCRAPING_STATUS_FILE when serving alone).
+pub async fn scraping_status(State(st): State<Shared>) -> ApiResult {
+    st.metrics.count("scraping_status");
+    let status: Value = match &st.status {
+        Some(s) => s.read().unwrap().clone(),
+        None => st
+            .cfg
+            .status_file
+            .as_ref()
+            .and_then(|p| std::fs::read_to_string(p).ok())
+            .and_then(|t| serde_json::from_str(&t).ok())
+            .unwrap_or_else(|| json!({"is_running": false})),
+    };
+    // ScrapingStatusResponse's fields and defaults first, then what the
+    // collector adds (version, forecast run)
+    let defaults = [
+        ("is_running", json!(false)),
+        ("last_run", Value::Null),
+        ("next_run", Value::Null),
+        ("total_runs", json!(0)),
+        ("successful_runs", json!(0)),
+        ("failed_runs", json!(0)),
+        ("feed_current", Value::Null),
+        ("latest_reading", Value::Null),
+        ("readings_saved", json!(0)),
+    ];
+    let mut out = Map::new();
+    for (k, d) in defaults {
+        out.insert(k.into(), status.get(k).cloned().unwrap_or(d));
+    }
+    for (k, v) in status.as_object().into_iter().flatten() {
+        if !out.contains_key(k) {
+            out.insert(k.clone(), v.clone());
+        }
+    }
+    ok_json(&Value::Object(out))
+}
+
+/// POST /scrape/trigger (admin): poll every source now.
+pub async fn trigger(State(st): State<Shared>, h: HeaderMap) -> ApiResult {
+    st.metrics.count("scrape_trigger");
+    require_admin(&st, &h)?;
+    let Some(trigger) = &st.trigger else {
+        return Err(ApiError::new(
+            409,
+            "Collection does not run in this process (ping-collector serve)",
+        ));
+    };
+    let running = st
+        .status
+        .as_ref()
+        .is_some_and(|s| s.read().unwrap()["is_running"] == true);
+    if running {
+        return Err(ApiError::new(409, "Scraping already in progress"));
+    }
+    trigger.notify_one();
+    ok_json(&json!({"message": "Scraping triggered", "status": "started"}))
+}
+
+/// GET /config (admin): the settings, secrets masked.
+pub async fn config(State(st): State<Shared>, h: HeaderMap) -> ApiResult {
+    st.metrics.count("config");
+    require_admin(&st, &h)?;
+    let c = &st.cfg;
+    let w = &c.web;
+    let mut out = json!({
+        "DB_TYPE": "postgresql",
+        "DATABASE_URL": c.redacted_database_url(),
+        "API_URL": c.rid_api_url,
+        "REQUEST_TIMEOUT": w.request_timeout_s,
+        "RID_POLL_OFFSET_MINUTES": c.poll_offset_minutes,
+        "RID_RETRY_MINUTES": c.retry_minutes,
+        "GAP_FILL_DAYS_ON_START": c.gap_fill_days_on_start,
+        "ENABLE_HII_COLLECTION": c.hii_enabled,
+        "ENABLE_RESERVOIR_COLLECTION": c.dams_enabled,
+        "ENABLE_RAIN_COLLECTION": c.rain_enabled,
+        "FORECAST_IN_COLLECTOR": c.forecast_enabled,
+        "MODELS_DIR": c.models_dir.display().to_string(),
+        "STATION_CONFIG_PATH": c.station_config_path.display().to_string(),
+        "SCRAPING_STATUS_FILE": c.status_file.as_ref().map(|p| p.display().to_string()),
+        "NTFY_SERVER": c.ntfy_server,
+        "NTFY_TOPIC_PREFIX": c.ntfy_topic_prefix,
+        "MATRIX_ENABLED": c.matrix.is_some(),
+        "WEB_LISTEN": w.listen,
+        "STATIC_DIR": w.static_dir.display().to_string(),
+        "CORS_ALLOW_ORIGINS": w.cors_allow_origins,
+        "HII_CACHE_TTL_SECONDS": w.hii_cache_ttl_s,
+        "LATEST_CACHE_TTL_SECONDS": w.latest_cache_ttl_s,
+        "HEALTH_CACHE_TTL_SECONDS": w.health_cache_ttl_s,
+        "FORECAST_MAX_AGE_SECONDS": w.forecast_max_age_s,
+        "UMAMI_TRACK_API": w.umami_track_api,
+        "collector": concat!("ping-collector ", env!("CARGO_PKG_VERSION")),
+    });
+    if let Some(m) = out.as_object_mut() {
+        for (k, v) in m.iter_mut() {
+            let k = k.to_lowercase();
+            if (k.contains("password") || k.contains("secret")) && !v.is_null() {
+                *v = json!("********");
+            }
+        }
+    }
+    ok_json(&out)
+}

rust/collector/src/web/tests.rs

@@ -0,0 +1,293 @@
+//! Route tests: the router as the server runs it, against a throwaway
+//! schema (TEST_DATABASE_URL; see db_tests.rs) or, for the routes that do
+//! not touch the database, a pool that never connects. Output formats are
+//! the retired FastAPI app's (field order, isoformat, errors).
+
+use super::{AppState, router};
+use crate::config::Config;
+use crate::db_tests::TestDb;
+use crate::stations::Stations;
+use axum::Router;
+use axum::body::Body;
+use axum::http::{HeaderMap, Request, StatusCode};
+use serde_json::{Value, json};
+use std::collections::HashMap;
+use std::sync::{Arc, RwLock};
+use tower::ServiceExt;
+
+fn config(extra: &[(&str, &str)]) -> Config {
+    let static_dir = concat!(env!("CARGO_MANIFEST_DIR"), "/../../src/static");
+    let mut vars: HashMap<String, String> = [
+        (
+            "POSTGRES_CONNECTION_STRING",
+            "postgres://test@localhost:1/none",
+        ),
+        ("UMAMI_TRACK_API", "false"),
+        ("STATIC_DIR", static_dir),
+        ("NTFY_SERVER", "https://ntfy.example"),
+        ("STATION_CONFIG_PATH", "/nonexistent/stations.json"),
+    ]
+    .iter()
+    .map(|(k, v)| (k.to_string(), v.to_string()))
+    .collect();
+    for (k, v) in extra {
+        vars.insert(k.to_string(), v.to_string());
+    }
+    Config::from_vars(&|k| vars.get(k).cloned()).unwrap()
+}
+
+fn app(cfg: Config, pool: sqlx::PgPool) -> Router {
+    let state = AppState::new(
+        cfg,
+        pool,
+        reqwest::Client::new(),
+        Arc::new(RwLock::new(Stations::bundled())),
+        None,
+        None,
+    );
+    router(state)
+}
+
+async fn call(
+    app: &Router,
+    method: &str,
+    uri: &str,
+    headers: &[(&str, &str)],
+) -> (StatusCode, HeaderMap, Vec<u8>) {
+    let mut req = Request::builder().method(method).uri(uri);
+    for (k, v) in headers {
+        req = req.header(*k, *v);
+    }
+    let resp = app
+        .clone()
+        .oneshot(req.body(Body::empty()).unwrap())
+        .await
+        .unwrap();
+    let (parts, body) = resp.into_parts();
+    let bytes = axum::body::to_bytes(body, usize::MAX).await.unwrap();
+    (parts.status, parts.headers, bytes.to_vec())
+}
+
+async fn get_json(app: &Router, uri: &str) -> (StatusCode, HeaderMap, Value) {
+    let (s, h, b) = call(app, "GET", uri, &[]).await;
+    let v = serde_json::from_slice(&b)
+        .unwrap_or_else(|e| panic!("{uri}: {e}: {}", String::from_utf8_lossy(&b)));
+    (s, h, v)
+}
+
+fn header<'a>(h: &'a HeaderMap, name: &str) -> Option<&'a str> {
+    h.get(name).and_then(|v| v.to_str().ok())
+}
+
+#[tokio::test]
+async fn routes_without_database() {
+    let pool = sqlx::PgPool::connect_lazy("postgres://test@localhost:1/none").unwrap();
+    let app = app(config(&[]), pool);
+
+    let (s, h, stations) = get_json(&app, "/stations").await;
+    assert_eq!(s, StatusCode::OK);
+    assert_eq!(header(&h, "cache-control"), Some("public, max-age=300"));
+    assert_eq!(header(&h, "content-type"), Some("application/json"));
+    assert_eq!(stations.as_array().unwrap().len(), 16);
+    // the list leaves geohash out, the single station has it (as FastAPI)
+    assert_eq!(stations[7]["station_code"], "P.1");
+    assert_eq!(stations[7]["geohash"], Value::Null);
+    let (_, _, p1) = get_json(&app, "/stations/8").await;
+    assert_eq!(
+        p1.as_object().unwrap().keys().collect::<Vec<_>>(),
+        [
+            "station_id",
+            "station_code",
+            "thai_name",
+            "english_name",
+            "latitude",
+            "longitude",
+            "geohash",
+            "status"
+        ]
+    );
+
+    let (s, _, e) = get_json(&app, "/stations/abc").await;
+    assert_eq!(s, StatusCode::UNPROCESSABLE_ENTITY);
+    assert_eq!(e["detail"][0]["type"], "int_parsing");
+    assert_eq!(e["detail"][0]["loc"], json!(["path", "station_id"]));
+    assert_eq!(
+        get_json(&app, "/stations/99").await.2,
+        json!({"detail": "Station not found"})
+    );
+    assert_eq!(
+        get_json(&app, "/nope").await.2,
+        json!({"detail": "Not Found"})
+    );
+    let (s, _, b) = call(&app, "DELETE", "/forecast", &[]).await;
+    assert_eq!(s, StatusCode::METHOD_NOT_ALLOWED);
+    assert_eq!(b, br#"{"detail":"Method Not Allowed"}"#);
+
+    let (s, _, e) = get_json(&app, "/config").await;
+    assert_eq!(s, StatusCode::SERVICE_UNAVAILABLE);
+    assert_eq!(
+        e["detail"],
+        "Admin API disabled: ADMIN_API_KEY is not configured on the server"
+    );
+    let (s, _, e) = get_json(&app, "/measurements/history/P.1?hours=0").await;
+    assert_eq!(s, StatusCode::UNPROCESSABLE_ENTITY);
+    assert_eq!(e["detail"][0]["ctx"], json!({"ge": 1}));
+
+    let (_, _, n) = get_json(&app, "/api/notifications").await;
+    assert_eq!(n["topics"]["warning"], "ping-warning");
+    assert_eq!(n["server"], "https://ntfy.example");
+
+    let (s, h, spec) = get_json(&app, "/openapi.json").await;
+    assert_eq!(s, StatusCode::OK);
+    assert_eq!(header(&h, "content-encoding"), None);
+    assert!(spec["paths"]["/forecast"]["get"].is_object());
+
+    let (s, h, html) = call(&app, "GET", "/", &[("accept-encoding", "gzip")]).await;
+    assert_eq!(s, StatusCode::OK);
+    assert_eq!(header(&h, "cache-control"), Some("public, max-age=120"));
+    assert_eq!(header(&h, "content-encoding"), Some("gzip"));
+    assert!(html.len() > 100);
+    let (s, h, _) = call(&app, "GET", "/docs", &[]).await;
+    assert_eq!(s, StatusCode::OK);
+    assert_eq!(header(&h, "content-type"), Some("text/html; charset=utf-8"));
+    let (s, h, _) = call(&app, "GET", "/static/custom-markers.json", &[]).await;
+    assert_eq!(s, StatusCode::OK);
+    assert_eq!(header(&h, "cache-control"), Some("public, max-age=3600"));
+    let (s, h, _) = call(&app, "GET", "/robots.txt", &[]).await;
+    assert_eq!(s, StatusCode::OK);
+    assert_eq!(
+        header(&h, "content-type"),
+        Some("text/plain; charset=utf-8")
+    );
+    assert_eq!(
+        call(&app, "GET", "/static/nope.json", &[]).await.0,
+        StatusCode::NOT_FOUND
+    );
+
+    // admin key
+    let app = super::router(AppState::new(
+        config(&[("ADMIN_API_KEY", "k3y")]),
+        sqlx::PgPool::connect_lazy("postgres://test@localhost:1/none").unwrap(),
+        reqwest::Client::new(),
+        Arc::new(RwLock::new(Stations::bundled())),
+        None,
+        None,
+    ));
+    assert_eq!(
+        call(&app, "GET", "/config", &[("x-api-key", "nope")])
+            .await
+            .0,
+        StatusCode::UNAUTHORIZED
+    );
+    let (s, _, b) = call(&app, "GET", "/config", &[("x-api-key", "k3y")]).await;
+    assert_eq!(s, StatusCode::OK, "{}", String::from_utf8_lossy(&b));
+    assert!(String::from_utf8_lossy(&b).contains("postgres://test:***@localhost:1/none"));
+    let (s, _, b) = call(&app, "POST", "/scrape/trigger", &[("x-api-key", "k3y")]).await;
+    assert_eq!(s, StatusCode::CONFLICT, "{}", String::from_utf8_lossy(&b));
+}
+
+async fn seed(pool: &sqlx::PgPool) {
+    for sql in [
+        "INSERT INTO stations (id, station_code, thai_name, english_name) VALUES
+             (1, 'P.20', 'บ้านเชียงดาว', 'Ban Chiang Dao'), (8, 'P.1', 'สะพานนวรัฐ', 'Nawarat Bridge')",
+        // P.1 hourly 2026-10-01 00:00 .. 10-03 23:00, one missing discharge, a hole at 10-02 05:00
+        "INSERT INTO water_measurements (timestamp, station_id, water_level, discharge, discharge_percent, status)
+         SELECT t, 8, round((1.2 + extract(hour from t) / 100.0)::numeric, 3),
+                CASE WHEN t = timestamp '2026-10-02 03:00' THEN NULL ELSE 55.5 END, 12.94, 'active'
+         FROM generate_series(timestamp '2026-10-01 00:00', timestamp '2026-10-03 23:00', interval '1 hour') t
+         WHERE t <> timestamp '2026-10-02 05:00'",
+        "INSERT INTO water_measurements (timestamp, station_id, water_level, discharge, discharge_percent, status)
+         VALUES (timestamp '2026-10-03 22:00', 1, 0.95, NULL, NULL, 'active')",
+        "INSERT INTO forecast_history (as_of, station_code, horizon_hours, predicted_max_level, p_warning,
+                                       p_danger, current_level, model_version, source)
+         SELECT t, 'P.1', 6, 1.5, 0.01, 0.0, 1.2, CASE WHEN t < timestamp '2026-10-02 00:00' THEN 'v1' ELSE 'v2' END, 'model'
+         FROM generate_series(timestamp '2026-10-01 00:00', timestamp '2026-10-03 12:00', interval '1 hour') t",
+        "INSERT INTO forecast_current (id, computed_at, model_versions, payload)
+         VALUES ('all', timestamp '2026-10-03 23:10', 'v2', '[{\"station_code\":\"P.1\",\"horizon_hours\":6}]')",
+    ] {
+        sqlx::query(sql).execute(pool).await.unwrap();
+    }
+}
+
+#[tokio::test]
+async fn routes_against_the_database() {
+    let Some(db) = TestDb::new().await else {
+        return;
+    };
+    seed(&db.pool).await;
+    let app = app(
+        config(&[("FORECAST_MAX_AGE_SECONDS", "60")]),
+        db.pool.clone(),
+    );
+
+    let (s, h, latest) = get_json(&app, "/measurements/latest?limit=5").await;
+    assert_eq!(s, StatusCode::OK);
+    assert_eq!(header(&h, "cache-control"), Some("public, max-age=30"));
+    // ORDER BY station_code: P.1 before P.20
+    assert_eq!(
+        latest,
+        json!([
+            {"timestamp": "2026-10-03T23:00:00", "station_code": "P.1", "station_name_en": "Nawarat Bridge",
+             "station_name_th": "สะพานนวรัฐ", "water_level": 1.43, "discharge": 55.5,
+             "discharge_percent": 12.94, "status": "active"},
+            {"timestamp": "2026-10-03T22:00:00", "station_code": "P.20", "station_name_en": "Ban Chiang Dao",
+             "station_name_th": "บ้านเชียงดาว", "water_level": 0.95, "discharge": null,
+             "discharge_percent": null, "status": "active"}
+        ])
+    );
+
+    let (_, _, h) = get_json(
+        &app,
+        "/measurements/history/P.1?start=2026-10-02&end=2026-10-02&limit=6",
+    )
+    .await;
+    let rows = h.as_array().unwrap();
+    assert_eq!(rows.len(), 6);
+    assert_eq!(rows[0]["timestamp"], "2026-10-02T00:00:00");
+    // discharge missing in the database: 20 * H, rounded
+    assert_eq!(rows[3]["discharge"], json!(24.6));
+    assert_eq!(rows[5]["timestamp"], "2026-10-02T06:00:00"); // the hole at 05:00
+    let (s, _, e) = get_json(
+        &app,
+        "/measurements/history/P.1?start=2026-10-03&end=2026-10-02",
+    )
+    .await;
+    assert_eq!(s, StatusCode::BAD_GATEWAY);
+    assert_eq!(e["detail"], "Measurement history unavailable");
+
+    let (_, _, stats) = get_json(&app, "/api/stats").await;
+    assert_eq!(stats["rid_measurements"], 72);
+    assert_eq!(stats["rid_station_count"], 2);
+    assert_eq!(stats["first_timestamp"], "2026-10-01T00:00:00");
+    assert_eq!(stats["days_spanned"], 3);
+    assert_eq!(stats["coverage_percent"], json!(98.6)); // 71 of 72 hours
+
+    let (_, _, fh) = get_json(&app, "/api/forecast/history/P.1?start=2026-10-03&horizon=6").await;
+    assert_eq!(fh.as_array().unwrap().len(), 13);
+    assert_eq!(fh[0]["predicted_max_level"], json!(1.5));
+
+    let (s, h, f) = get_json(&app, "/forecast").await;
+    assert_eq!(s, StatusCode::OK);
+    assert_eq!(f[0]["station_code"], "P.1");
+    // computed long ago: served, marked
+    assert_eq!(header(&h, "x-data-stale"), Some("true"));
+
+    let (_, _, sk) = get_json(&app, "/api/forecast/skill?station_code=P.1&horizon=6").await;
+    let versions = sk["versions"].as_array().unwrap();
+    assert_eq!(versions.len(), 2);
+    assert_eq!(versions[0]["model_version"], "v1");
+    assert_eq!(versions[0]["n"], 24);
+    assert_eq!(versions[0]["enough_data"], true);
+    assert_eq!(sk["current"]["model_version"], "v2");
+    assert_eq!(sk["trend"]["previous_version"], "v1");
+
+    let (_, _, health) = get_json(&app, "/health").await;
+    assert_eq!(health["checks"]["database"]["status"], "degraded"); // data from 2026-10-03
+    assert!(
+        health["checks"]["database"]["message"]
+            .as_str()
+            .unwrap()
+            .starts_with("Latest data is old: 2026-10-03 23:00:00")
+    );
+    db.drop().await;
+}

rust/collector/testdata/gen_envfile_cases.py

@@ -0,0 +1,122 @@
+"""Generate rust/collector/testdata/envfile_python_dotenv.json: python-dotenv's
+own parse + interpolation of tricky .env inputs, used by the Rust envfile
+tests so the collector reads .env exactly as the Python app does.
+
+    uv run python rust/collector/testdata/gen_envfile_cases.py
+"""
+
+import io
+import json
+import os
+import random
+from pathlib import Path
+
+import dotenv
+from dotenv.main import DotEnv
+
+FIXED = [
+    "A=abc'def\nB=x\"y\nC=1\n",
+    "A=1\nPOSTGRES_PASSWORD='abc\nB=2\n",
+    "A=1\nPOSTGRES_PASSWORD= \nB=2\n",
+    "POSTGRES_PASSWORD=\"ab'c\nB=2\n",
+    "  export  A = spaced value   # comment\nB=a#b\n# c\n\nC= \nD\n",
+    "A='x\\'y'\nB=\"l1\\nl2\\t\\\"q\\\"\"\nC='multi\nline'\nD='a\\\\b'\n",
+    "A='ab\\'",
+    "A='x'y\nB=1\n",
+    "HOME_FROM_ENV=file\nA=${HOME_FROM_ENV}/x\nB=${A}-${MISSING:-dflt}-${NOPE}\nC=$A ${A:x}\n",
+    "A=1\r\nB=2\r\nA=3\r\n",
+    "C=  # c\nD=#x\nE= #x\n",
+    "A#B=1\nexport\nexportA=2\n'q k'=3\n",
+    "\ufeffA=bom\n",
+    "A=\u00e9\u0e01\u0e02 # thai\n",
+]
+
+KEYS = [
+    "A",
+    "B",
+    "C",
+    "D",
+    "E",
+    "export A",
+    "'A B'",
+    "HOME_FROM_ENV",
+    "#A",
+    " A",
+    "A B",
+]
+SEPS = ["=", " = ", "=  ", "", "\t=\t", "=="]
+VALUE_BITS = [
+    "abc",
+    "'",
+    '"',
+    "\\",
+    "#",
+    " #c",
+    "${A}",
+    "${B:-d}",
+    "${",
+    "}",
+    " ",
+    "\t",
+    "'q'",
+    '"q"',
+    "\\'",
+    '\\"',
+    "\\n",
+    "x y",
+    "${HOME_FROM_ENV}",
+    "${:-z}",
+    "$A",
+    "\u0e01",
+]
+EOLS = ["\n", "\r\n", "\r", ""]
+
+
+def random_case(rng: random.Random) -> str:
+    lines = []
+    for _ in range(rng.randint(1, 5)):
+        kind = rng.random()
+        if kind < 0.1:
+            lines.append(rng.choice(["", "   ", "# comment", "  # c ' \""]))
+        else:
+            value = "".join(rng.choice(VALUE_BITS) for _ in range(rng.randint(0, 4)))
+            lines.append(rng.choice(KEYS) + rng.choice(SEPS) + value)
+        lines.append(rng.choice(EOLS))
+    return "".join(lines)
+
+
+def expected(content: str):
+    values = DotEnv(
+        None, stream=io.StringIO(content), interpolate=True, override=False
+    ).dict()
+    return [[k, v] for k, v in values.items()]
+
+
+def main():
+    os.environ["HOME_FROM_ENV"] = "/home/x"
+    for k in ("A", "B", "C", "D", "E", "MISSING", "NOPE"):
+        os.environ.pop(k, None)
+    rng = random.Random(20261018)
+    inputs = FIXED + [random_case(rng) for _ in range(600)]
+    cases = [{"input": c, "expected": expected(c)} for c in inputs]
+    out = Path(__file__).with_name("envfile_python_dotenv.json")
+    out.write_text(
+        json.dumps(
+            {
+                "python_dotenv": dotenv.__dict__.get("__version__", "1.1.1"),
+                "env": {"HOME_FROM_ENV": "/home/x"},
+                "cases": cases,
+            },
+            ensure_ascii=True,
+            indent=0,
+        ),
+        encoding="utf-8",
+    )
+    print(f"{len(cases)} cases -> {out}")
+
+
+if __name__ == "__main__":
+    import logging
+
+    logging.disable(logging.WARNING)  # python-dotenv warns on every bad line
+    main()

rust/forecast/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "ping-forecast"
+version = "0.1.0"
+description = "Ping River flood forecast inference (port of src/ml: features, HistGradientBoosting trees, heuristic fallback)"
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+publish.workspace = true
+repository.workspace = true
+
+[dependencies]
+anyhow.workspace = true
+chrono.workspace = true
+flate2.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+tracing.workspace = true
+
+[dev-dependencies]
+flate2.workspace = true

rust/forecast/flood_config.json

@@ -0,0 +1,348 @@
+{
+ "thresholds": {
+  "*": [
+   3.0,
+   4.5
+  ],
+  "P.1": [
+   3.7,
+   4.2
+  ],
+  "P.103": [
+   5.95,
+   6.75
+  ],
+  "P.20": [
+   2.35,
+   2.8
+  ],
+  "P.21": [
+   3.2,
+   3.6
+  ],
+  "P.4A": [
+   3.4,
+   3.9
+  ],
+  "P.5": [
+   4.55,
+   4.95
+  ],
+  "P.67": [
+   2.45,
+   2.9
+  ],
+  "P.75": [
+   3.2,
+   3.65
+  ],
+  "P.76": [
+   5.35,
+   5.45
+  ],
+  "P.77": [
+   4.3,
+   4.9
+  ],
+  "P.81": [
+   5.15,
+   6.3
+  ],
+  "P.82": [
+   3.4,
+   3.75
+  ],
+  "P.84": [
+   3.45,
+   3.9
+  ],
+  "P.85": [
+   2.9,
+   3.35
+  ],
+  "P.87": [
+   3.75,
+   4.05
+  ],
+  "P.92": [
+   2.95,
+   3.6
+  ]
+ },
+ "flood_stages": {
+  "P.1": [
+   {
+    "stage": 1,
+    "level": 3.7,
+    "discharge_cms": 405.0
+   },
+   {
+    "stage": 2,
+    "level": 3.9,
+    "discharge_cms": 438.0
+   },
+   {
+    "stage": 3,
+    "level": 4.0,
+    "discharge_cms": 458.0
+   },
+   {
+    "stage": 4,
+    "level": 4.1,
+    "discharge_cms": 478.0
+   },
+   {
+    "stage": 5,
+    "level": 4.2,
+    "discharge_cms": 493.0
+   },
+   {
+    "stage": 6,
+    "level": 4.3,
+    "discharge_cms": 508.0
+   },
+   {
+    "stage": 7,
+    "level": 4.6,
+    "discharge_cms": 558.0
+   }
+  ]
+ },
+ "upstream_leads": {
+  "P.1": [
+   [
+    "P.103",
+    1
+   ],
+   [
+    "P.67",
+    7
+   ],
+   [
+    "P.21",
+    9
+   ],
+   [
+    "P.75",
+    12
+   ],
+   [
+    "P.4A",
+    12
+   ],
+   [
+    "P.92",
+    15
+   ],
+   [
+    "P.20",
+    17
+   ]
+  ],
+  "P.103": [
+   [
+    "P.67",
+    6
+   ],
+   [
+    "P.21",
+    8
+   ],
+   [
+    "P.75",
+    11
+   ],
+   [
+    "P.4A",
+    11
+   ],
+   [
+    "P.92",
+    14
+   ],
+   [
+    "P.20",
+    16
+   ]
+  ],
+  "P.21": [
+   [
+    "P.67",
+    1
+   ],
+   [
+    "P.75",
+    3
+   ],
+   [
+    "P.4A",
+    3
+   ],
+   [
+    "P.92",
+    6
+   ],
+   [
+    "P.20",
+    8
+   ]
+  ],
+  "P.67": [
+   [
+    "P.75",
+    5
+   ],
+   [
+    "P.4A",
+    5
+   ],
+   [
+    "P.92",
+    8
+   ],
+   [
+    "P.20",
+    10
+   ]
+  ],
+  "P.75": [
+   [
+    "P.92",
+    3
+   ],
+   [
+    "P.20",
+    5
+   ]
+  ],
+  "P.4A": [
+   [
+    "P.92",
+    3
+   ],
+   [
+    "P.20",
+    5
+   ]
+  ],
+  "P.92": [
+   [
+    "P.20",
+    2
+   ]
+  ],
+  "P.20": [],
+  "P.5": [
+   [
+    "P.1",
+    12
+   ],
+   [
+    "P.103",
+    13
+   ]
+  ],
+  "P.81": [
+   [
+    "P.1",
+    4
+   ],
+   [
+    "P.103",
+    5
+   ]
+  ],
+  "P.82": [],
+  "P.84": [],
+  "P.87": [],
+  "P.77": [],
+  "P.85": [],
+  "P.76": []
+ },
+ "dam_stations": [
+  "P.1",
+  "P.103",
+  "P.21",
+  "P.5",
+  "P.67",
+  "P.81"
+ ],
+ "monsoon_months": [
+  6,
+  7,
+  8,
+  9,
+  10
+ ],
+ "ffill_limit_h": 3,
+ "basin_anchor": "P.1",
+ "forward_rain_h": 24,
+ "horizons": [
+  6,
+  12,
+  24
+ ],
+ "stale_after_h": 6.0,
+ "heuristic_sigma": 0.3,
+ "heuristic_version": "heuristic-v1",
+ "latest_window_h": 336,
+ "hii_fill_stations": [
+  "P.1",
+  "P.103",
+  "P.20",
+  "P.4A",
+  "P.67",
+  "P.75",
+  "P.82",
+  "P.84",
+  "P.92",
+  "P.81"
+ ],
+ "hii_exact_mirrors": [
+  "P.1",
+  "P.103",
+  "P.20",
+  "P.4A",
+  "P.67",
+  "P.75",
+  "P.82",
+  "P.84",
+  "P.92"
+ ],
+ "hii_min_overlap_hours": 168,
+ "dam": {
+  "dam_id": "200103",
+  "report_hour": 7,
+  "ffill_limit_h": 48,
+  "serving_days": 21
+ },
+ "notify": {
+  "clear_margin_m": 0.1,
+  "capacity_guard_min_pct": 60.0,
+  "capacity_guard_exempt": [
+   "P.1"
+  ],
+  "outlook_on": 0.5,
+  "outlook_off": 0.25,
+  "outlook_horizon": 24,
+  "feed_stale_after_h": 3.0,
+  "station_names": {
+   "P.1": "Nawarat Bridge, Chiang Mai city",
+   "P.103": "Ring Road Bridge 3, Chiang Mai",
+   "P.67": "Ban Tae (Mae Taeng)",
+   "P.21": "Ban Rim Tai (Mae Rim)",
+   "P.75": "Ban Chai Lat",
+   "P.92": "Ban Muang Aut",
+   "P.20": "Ban Chiang Dao",
+   "P.4A": "Ban Mae Taeng",
+   "P.5": "Tha Nang Bridge (downstream)",
+   "P.81": "Ban Pong (downstream)",
+   "P.82": "Ban Sob Win",
+   "P.84": "Ban Panton",
+   "P.87": "Ban Pa Sang",
+   "P.77": "Ban Sop Mae Sapuat",
+   "P.85": "Ban Lai Kaew",
+   "P.76": "Ban Mae I Hai"
+  }
+ }
+}

rust/forecast/src/config.rs

@@ -0,0 +1,106 @@
+//! Constants shared with Python, generated from it: `flood_config.json` is
+//! written by `uv run python -m src.ml.export --config`, and
+//! tests/test_model_export.py fails while it differs from the Python source.
+
+use serde::Deserialize;
+use std::collections::BTreeMap;
+use std::sync::OnceLock;
+
+#[derive(Debug, Deserialize)]
+pub struct Stage {
+    pub stage: u32,
+    pub level: f64,
+    pub discharge_cms: f64,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct DamConfig {
+    pub dam_id: String,
+    pub report_hour: u32,
+    pub ffill_limit_h: usize,
+    pub serving_days: i64,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct NotifyConfig {
+    pub clear_margin_m: f64,
+    pub capacity_guard_min_pct: f64,
+    pub capacity_guard_exempt: Vec<String>,
+    pub outlook_on: f64,
+    pub outlook_off: f64,
+    pub outlook_horizon: u32,
+    pub feed_stale_after_h: f64,
+    pub station_names: BTreeMap<String, String>,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct FloodConfig {
+    thresholds: BTreeMap<String, (f64, f64)>,
+    pub flood_stages: BTreeMap<String, Vec<Stage>>,
+    upstream_leads: BTreeMap<String, Vec<(String, i64)>>,
+    pub dam_stations: Vec<String>,
+    pub monsoon_months: Vec<u32>,
+    pub ffill_limit_h: usize,
+    pub basin_anchor: String,
+    pub forward_rain_h: usize,
+    pub horizons: Vec<u32>,
+    pub stale_after_h: f64,
+    pub heuristic_sigma: f64,
+    pub heuristic_version: String,
+    pub latest_window_h: i64,
+    pub hii_fill_stations: Vec<String>,
+    pub hii_exact_mirrors: Vec<String>,
+    pub hii_min_overlap_hours: usize,
+    pub dam: DamConfig,
+    pub notify: NotifyConfig,
+}
+
+impl FloodConfig {
+    /// (warning, danger) for a station, or the "*" default.
+    pub fn thresholds(&self, station: &str) -> (f64, f64) {
+        self.thresholds
+            .get(station)
+            .or_else(|| self.thresholds.get("*"))
+            .copied()
+            .unwrap_or((f64::INFINITY, f64::INFINITY))
+    }
+
+    /// (upstream station, lead hours) pairs, in Python's order.
+    pub fn upstream_leads(&self, station: &str) -> &[(String, i64)] {
+        self.upstream_leads.get(station).map_or(&[], Vec::as_slice)
+    }
+
+    /// The stations the models are built for (Python's `UPSTREAM_LEADS` keys).
+    pub fn forecast_stations(&self) -> impl Iterator<Item = &str> {
+        self.upstream_leads.keys().map(String::as_str)
+    }
+
+    pub fn is_dam_station(&self, station: &str) -> bool {
+        self.dam_stations.iter().any(|s| s == station)
+    }
+}
+
+pub fn config() -> &'static FloodConfig {
+    static CONFIG: OnceLock<FloodConfig> = OnceLock::new();
+    CONFIG.get_or_init(|| {
+        serde_json::from_str(include_str!("../flood_config.json"))
+            .expect("flood_config.json is generated by src/ml/export.py and must parse")
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn generated_config_parses_and_has_the_basin() {
+        let c = config();
+        assert_eq!(c.thresholds("P.1"), (3.7, 4.2));
+        assert_eq!(c.thresholds("P.999"), (3.0, 4.5));
+        assert_eq!(c.upstream_leads("P.1")[0], ("P.103".to_string(), 1));
+        assert!(c.upstream_leads("P.82").is_empty());
+        assert_eq!(c.forecast_stations().count(), 16);
+        assert_eq!(c.horizons, vec![6, 12, 24]);
+        assert!(c.is_dam_station("P.1") && !c.is_dam_station("P.20"));
+    }
+}

rust/forecast/src/dam.rs

@@ -0,0 +1,98 @@
+//! Mae Ngat reservoir state for the dam features (`src/ml/dam.py`).
+
+use crate::config::config;
+use crate::rolling::ffill;
+use chrono::{Duration, NaiveDate, NaiveDateTime};
+use std::collections::BTreeMap;
+
+/// One `rid_reservoir_daily` row; NaN = missing.
+#[derive(Clone, Debug, PartialEq)]
+pub struct DamDay {
+    pub date: NaiveDate,
+    pub storage_pct: f64,
+    pub inflow_mcm: f64,
+    pub outflow_mcm: f64,
+}
+
+/// Hourly dam state: each daily report valid from 07:00 of its date,
+/// forward-filled at most 48 h (`dam.hourly_frame`).
+#[derive(Clone, Debug, Default)]
+pub struct DamFrame {
+    pub hours: BTreeMap<NaiveDateTime, [f64; 3]>,
+}
+
+impl DamFrame {
+    /// Columns storage_pct, inflow_mcm, outflow_mcm on the given hours
+    /// (`DataFrame.reindex`); an empty frame gives NaN columns.
+    pub fn reindex(&self, times: &[NaiveDateTime]) -> [Vec<f64>; 3] {
+        let col = |k: usize| {
+            times
+                .iter()
+                .map(|t| self.hours.get(t).map_or(f64::NAN, |r| r[k]))
+                .collect()
+        };
+        [col(0), col(1), col(2)]
+    }
+}
+
+/// `dam.hourly_frame`; `None` for no rows (Python then serves an empty frame).
+pub fn hourly_frame(days: &[DamDay]) -> Option<DamFrame> {
+    let cfg = &config().dam;
+    let mut by_time: BTreeMap<NaiveDateTime, [f64; 3]> = BTreeMap::new();
+    for d in days {
+        let t = d.date.and_hms_opt(cfg.report_hour, 0, 0)?;
+        by_time.insert(t, [d.storage_pct, d.inflow_mcm, d.outflow_mcm]); // keep last
+    }
+    let first = *by_time.keys().next()?;
+    let last = *by_time.keys().next_back()? + Duration::hours(cfg.ffill_limit_h as i64);
+    let n = ((last - first).num_hours() + 1) as usize;
+    let times: Vec<NaiveDateTime> = (0..n).map(|i| first + Duration::hours(i as i64)).collect();
+    let cols: Vec<Vec<f64>> = (0..3)
+        .map(|k| {
+            let raw: Vec<f64> = times
+                .iter()
+                .map(|t| by_time.get(t).map_or(f64::NAN, |r| r[k]))
+                .collect();
+            ffill(&raw, cfg.ffill_limit_h)
+        })
+        .collect();
+    Some(DamFrame {
+        hours: times
+            .iter()
+            .enumerate()
+            .map(|(i, t)| (*t, [cols[0][i], cols[1][i], cols[2][i]]))
+            .collect(),
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn day(d: u32, storage: f64) -> DamDay {
+        DamDay {
+            date: NaiveDate::from_ymd_opt(2026, 10, d).unwrap(),
+            storage_pct: storage,
+            inflow_mcm: 1.0,
+            outflow_mcm: 2.0,
+        }
+    }
+
+    #[test]
+    fn report_valid_from_seven_and_filled_two_days() {
+        let f = hourly_frame(&[day(1, 50.0), day(5, 60.0)]).unwrap();
+        let at = |d: u32, h: u32| {
+            NaiveDate::from_ymd_opt(2026, 10, d)
+                .unwrap()
+                .and_hms_opt(h, 0, 0)
+                .unwrap()
+        };
+        assert!(!f.hours.contains_key(&at(1, 6)));
+        assert_eq!(f.hours[&at(1, 7)][0], 50.0);
+        assert_eq!(f.hours[&at(3, 7)][0], 50.0); // +48 h
+        assert!(f.hours[&at(3, 8)][0].is_nan());
+        assert_eq!(f.hours[&at(7, 7)][0], 60.0);
+        assert_eq!(f.hours.keys().next_back(), Some(&at(7, 7)));
+        assert!(hourly_frame(&[]).is_none());
+    }
+}

rust/forecast/src/features.rs

@@ -0,0 +1,166 @@
+//! `features.build_features`: the feature columns for one target station,
+//! in Python's construction order. Features are causal except `rain_fc24`,
+//! the forward sum of the weather forecast.
+
+use crate::config::config;
+use crate::dam::DamFrame;
+use crate::grid::Grid;
+use crate::rain::{RainSeries, reindex};
+use crate::rolling::*;
+use chrono::{Datelike, Duration};
+use std::f64::consts::PI;
+
+pub type Columns = Vec<(String, Series)>;
+
+/// `rain`: `None` omits the rain columns (Python's `rain=None`), an empty
+/// series gives NaN columns. Same for `dam`, which only DAM_STATIONS receive.
+pub fn build_features(
+    grid: &Grid,
+    station: &str,
+    rain: Option<&RainSeries>,
+    dam: Option<&DamFrame>,
+) -> Columns {
+    let cfg = config();
+    let n = grid.len();
+    let ffill_h = cfg.ffill_limit_h;
+    let filled = |s: &str| -> Series {
+        grid.level
+            .get(s)
+            .map_or_else(|| nan_series(n), |v| ffill(v, ffill_h))
+    };
+    let mut cols: Columns = Vec::with_capacity(80);
+    let mut put = |name: String, s: Series| cols.push((name, s));
+
+    let level = filled(station);
+    let discharge = grid
+        .discharge
+        .get(station)
+        .cloned()
+        .unwrap_or_else(|| nan_series(n));
+    let mask: Vec<bool> = grid.level.get(station).map_or_else(
+        || vec![false; n],
+        |v| v.iter().map(|x| !x.is_nan()).collect(),
+    );
+
+    put("level".into(), level.clone());
+    for k in [1, 2, 3, 6, 12, 24, 48, 72] {
+        put(format!("level_lag_{k}"), shift(&level, k));
+    }
+    for k in [1, 3, 6, 12, 24] {
+        put(format!("rise_{k}"), sub(&level, &shift(&level, k)));
+    }
+    put("roll_mean_6".into(), rolling_mean(&level, 6));
+    put("roll_mean_24".into(), rolling_mean(&level, 24));
+    put("roll_max_6".into(), rolling_max(&level, 6));
+    put("roll_max_24".into(), rolling_max(&level, 24));
+    put("roll_max_72".into(), rolling_max(&level, 72));
+    put("roll_min_24".into(), rolling_min(&level, 24));
+
+    put("discharge".into(), discharge.clone());
+    put("discharge_lag_6".into(), shift(&discharge, 6));
+    put("discharge_lag_24".into(), shift(&discharge, 24));
+    put(
+        "discharge_rise_6".into(),
+        sub(&discharge, &shift(&discharge, 6)),
+    );
+
+    // hours since the last observed level, NaN beyond the ffill limit
+    let mut last: Option<usize> = None;
+    let age: Series = mask
+        .iter()
+        .enumerate()
+        .map(|(i, &m)| {
+            if m {
+                last = Some(i);
+            }
+            match last {
+                Some(l) if i - l <= ffill_h => (i - l) as f64,
+                _ => f64::NAN,
+            }
+        })
+        .collect();
+    put("obs_age_h".into(), age);
+    let mask_f: Series = mask.iter().map(|&m| if m { 1.0 } else { 0.0 }).collect();
+    put("cov_24h".into(), rolling_mean(&mask_f, 24));
+
+    for (up, lead) in cfg.upstream_leads(station) {
+        let lead = *lead;
+        let u = filled(up);
+        let u_rise_6 = sub(&u, &shift(&u, 6));
+        let u_rollmax_24 = rolling_max(&u, 24);
+        let near = (lead - 3).max(0);
+        put(format!("{up}_level_lag_{near}"), shift(&u, near));
+        put(format!("{up}_level_lag_{lead}"), shift(&u, lead));
+        put(format!("{up}_level_lag_{}", lead + 3), shift(&u, lead + 3));
+        put(format!("{up}_rise_6_lag_{lead}"), shift(&u_rise_6, lead));
+        put(
+            format!("{up}_rollmax_24_lag_{near}"),
+            shift(&u_rollmax_24, near),
+        );
+    }
+
+    if station != cfg.basin_anchor {
+        let p1 = filled(&cfg.basin_anchor);
+        let rollmax = rolling_max(&p1, 24);
+        let rise = sub(&p1, &shift(&p1, 24));
+        put("P1_level".into(), p1);
+        put("P1_rollmax_24".into(), rollmax);
+        put("P1_rise_24".into(), rise);
+    }
+
+    let mut put = |name: &str, s: Series| cols.push((name.to_string(), s));
+    let doy: Vec<f64> = grid.times.iter().map(|t| t.ordinal() as f64).collect();
+    put(
+        "doy_sin",
+        doy.iter().map(|d| (2.0 * PI * d / 365.25).sin()).collect(),
+    );
+    put(
+        "doy_cos",
+        doy.iter().map(|d| (2.0 * PI * d / 365.25).cos()).collect(),
+    );
+    put(
+        "is_monsoon",
+        grid.times
+            .iter()
+            .map(|t| {
+                if cfg.monsoon_months.contains(&t.month()) {
+                    1.0
+                } else {
+                    0.0
+                }
+            })
+            .collect(),
+    );
+
+    if let Some(rain) = rain {
+        let r = reindex(rain, &grid.times);
+        put("rain_6h", rolling_sum(&r, 6));
+        put("rain_24h", rolling_sum(&r, 24));
+        put("rain_72h", rolling_sum(&r, 72));
+        // forward sum over (t, t+24], on the grid extended past its last row
+        // so the newest row sees the weather forecast
+        let mut ext_times = grid.times.clone();
+        if let Some(&last) = grid.times.last() {
+            ext_times.extend((1..=cfg.forward_rain_h as i64).map(|h| last + Duration::hours(h)));
+        }
+        let mut fc = forward_sum(&reindex(rain, &ext_times), cfg.forward_rain_h);
+        fc.truncate(grid.len());
+        put("rain_fc24", fc);
+    }
+
+    if let Some(dam) = dam
+        && cfg.is_dam_station(station)
+    {
+        let [storage, inflow, outflow] = dam.reindex(&grid.times);
+        put("dam_storage_pct", storage.clone());
+        put("dam_storage_pct_d3", sub(&storage, &shift(&storage, 72)));
+        put("dam_inflow", inflow);
+        put("dam_outflow", outflow);
+    }
+    cols
+}
+
+/// The row at index `i` as name -> value.
+pub fn row_at(cols: &Columns, i: usize) -> std::collections::HashMap<&str, f64> {
+    cols.iter().map(|(k, v)| (k.as_str(), v[i])).collect()
+}

rust/forecast/src/grid.rs

@@ -0,0 +1,86 @@
+//! Long readings -> complete hourly grid (`features.make_hourly_grid`).
+
+use crate::rolling::{Series, nan_series};
+use chrono::{Duration, NaiveDateTime, Timelike};
+use std::collections::BTreeMap;
+
+/// One reading in long format; NaN = missing.
+#[derive(Clone, Debug, PartialEq)]
+pub struct Row {
+    pub timestamp: NaiveDateTime,
+    pub station_code: String,
+    pub water_level: f64,
+    pub discharge: f64,
+}
+
+pub fn floor_hour(t: NaiveDateTime) -> NaiveDateTime {
+    t.with_nanosecond(0)
+        .and_then(|t| t.with_second(0))
+        .and_then(|t| t.with_minute(0))
+        .unwrap_or(t)
+}
+
+/// Every hour from the first to the last reading; per station the observed
+/// (unfilled) level and discharge, NaN where nothing was recorded.
+pub struct Grid {
+    pub times: Vec<NaiveDateTime>,
+    pub level: BTreeMap<String, Series>,
+    pub discharge: BTreeMap<String, Series>,
+}
+
+impl Grid {
+    /// Timestamps are floored to the hour; for a duplicate (station, hour)
+    /// the later row wins (pandas `drop_duplicates(keep="last")`).
+    pub fn from_rows(rows: &[Row]) -> Grid {
+        let mut cells: BTreeMap<(String, NaiveDateTime), (f64, f64)> = BTreeMap::new();
+        for r in rows {
+            cells.insert(
+                (r.station_code.clone(), floor_hour(r.timestamp)),
+                (r.water_level, r.discharge),
+            );
+        }
+        let (Some(first), Some(last)) = (
+            cells.keys().map(|k| k.1).min(),
+            cells.keys().map(|k| k.1).max(),
+        ) else {
+            return Grid {
+                times: vec![],
+                level: BTreeMap::new(),
+                discharge: BTreeMap::new(),
+            };
+        };
+        let n = ((last - first).num_hours() + 1) as usize;
+        let times: Vec<NaiveDateTime> = (0..n).map(|i| first + Duration::hours(i as i64)).collect();
+        let mut level: BTreeMap<String, Series> = BTreeMap::new();
+        let mut discharge: BTreeMap<String, Series> = BTreeMap::new();
+        for ((code, t), (l, q)) in cells {
+            let i = (t - first).num_hours() as usize;
+            level.entry(code.clone()).or_insert_with(|| nan_series(n))[i] = l;
+            discharge.entry(code).or_insert_with(|| nan_series(n))[i] = q;
+        }
+        Grid {
+            times,
+            level,
+            discharge,
+        }
+    }
+
+    pub fn len(&self) -> usize {
+        self.times.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.times.is_empty()
+    }
+
+    pub fn index_of(&self, t: NaiveDateTime) -> Option<usize> {
+        let first = *self.times.first()?;
+        let i = (t - first).num_hours();
+        (i >= 0 && (i as usize) < self.len() && self.times[i as usize] == t).then_some(i as usize)
+    }
+
+    /// The station's last hour with an observed level.
+    pub fn last_observed(&self, station: &str) -> Option<usize> {
+        self.level.get(station)?.iter().rposition(|x| !x.is_nan())
+    }
+}

rust/forecast/src/hgb.rs

@@ -0,0 +1,145 @@
+//! Inference for scikit-learn HistGradientBoosting heads exported as flat
+//! tree arrays (src/ml/export.py). Mirrors sklearn's `_predictor.pyx`: NaN
+//! follows `missing_go_to_left`, otherwise `x <= threshold` goes left; raw =
+//! baseline + leaf values summed in tree order (sklearn's order, so the
+//! regression output is bit-identical).
+
+use anyhow::{Result, bail};
+use serde::Deserialize;
+
+/// One node, packed (24 bytes): `x` is the split threshold of an internal
+/// node and the output of a leaf.
+#[derive(Debug, Clone, Copy)]
+struct Node {
+    x: f64,
+    left: u32,
+    right: u32,
+    feature: u16,
+    leaf: bool,
+    missing_left: bool,
+}
+
+/// The exported arrays, as in sklearn's predictor node table.
+#[derive(Deserialize)]
+struct RawTree {
+    feature: Vec<usize>,
+    threshold: Vec<Option<f64>>,
+    missing_left: Vec<bool>,
+    left: Vec<usize>,
+    right: Vec<usize>,
+    leaf: Vec<bool>,
+    value: Vec<f64>,
+}
+
+#[derive(Debug)]
+pub struct Tree {
+    nodes: Vec<Node>,
+}
+
+impl<'de> Deserialize<'de> for Tree {
+    fn deserialize<D: serde::Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
+        use serde::de::Error;
+        let r = RawTree::deserialize(d)?;
+        let n = r.value.len();
+        if n == 0
+            || [
+                r.feature.len(),
+                r.threshold.len(),
+                r.missing_left.len(),
+                r.left.len(),
+                r.right.len(),
+                r.leaf.len(),
+            ]
+            .iter()
+            .any(|&l| l != n)
+        {
+            return Err(D::Error::custom("tree arrays have different lengths"));
+        }
+        let nodes = (0..n)
+            .map(|i| {
+                let leaf = r.leaf[i];
+                if !leaf && (r.left[i] >= n || r.right[i] >= n || r.feature[i] > u16::MAX as usize)
+                {
+                    return Err(D::Error::custom(format!(
+                        "tree node {i} points outside the tree"
+                    )));
+                }
+                Ok(Node {
+                    // +inf thresholds ("missing vs everything") are exported as null
+                    x: if leaf {
+                        r.value[i]
+                    } else {
+                        r.threshold[i].unwrap_or(f64::INFINITY)
+                    },
+                    left: r.left[i] as u32,
+                    right: r.right[i] as u32,
+                    feature: r.feature[i] as u16,
+                    leaf,
+                    missing_left: r.missing_left[i],
+                })
+            })
+            .collect::<Result<_, _>>()?;
+        Ok(Tree { nodes })
+    }
+}
+
+impl Tree {
+    #[inline]
+    fn predict(&self, x: &[f64]) -> f64 {
+        let mut node = &self.nodes[0];
+        while !node.leaf {
+            let v = x[node.feature as usize];
+            let go_left = if v.is_nan() {
+                node.missing_left
+            } else {
+                v <= node.x
+            };
+            node = &self.nodes[if go_left { node.left } else { node.right } as usize];
+        }
+        node.x
+    }
+
+    fn validate(&self, n_features: usize) -> Result<()> {
+        if let Some(i) = self
+            .nodes
+            .iter()
+            .position(|n| !n.leaf && n.feature as usize >= n_features)
+        {
+            bail!("tree node {i} reads feature outside the feature vector");
+        }
+        Ok(())
+    }
+}
+
+#[derive(Debug, Deserialize, PartialEq, Eq, Clone, Copy)]
+#[serde(rename_all = "lowercase")]
+pub enum Kind {
+    Regressor,
+    Classifier,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct Head {
+    pub kind: Kind,
+    baseline: f64,
+    trees: Vec<Tree>,
+}
+
+impl Head {
+    /// Regressor: the prediction. Classifier: P(class 1), as
+    /// `predict_proba(X)[:, 1]` (scipy's `expit`).
+    pub fn predict(&self, x: &[f64]) -> f64 {
+        let mut raw = self.baseline;
+        for t in &self.trees {
+            raw += t.predict(x);
+        }
+        match self.kind {
+            Kind::Regressor => raw,
+            Kind::Classifier => 1.0 / (1.0 + (-raw).exp()),
+        }
+    }
+
+    pub fn validate(&self, n_features: usize) -> Result<()> {
+        self.trees.iter().try_for_each(|t| t.validate(n_features))
+    }
+}

rust/forecast/src/hii.rs

@@ -0,0 +1,231 @@
+//! Gap fill from the HII/ThaiWater mirror telemetry (`data.fill_from_hii`).
+//!
+//! For stations whose HII twin is the same sensor, hours missing from the RID
+//! series (or carrying no level) are filled from HII, shifted by the median
+//! offset between the two series over their overlap. This is also what keeps
+//! the forecast current at night while RID's report lags.
+
+use crate::config::config;
+use crate::grid::{Row, floor_hour};
+use chrono::NaiveDateTime;
+use std::collections::{BTreeMap, BTreeSet, HashSet};
+
+/// One `hii_waterlevel` row joined to its RID station code; NaN = missing.
+#[derive(Clone, Debug, PartialEq)]
+pub struct HiiRow {
+    pub timestamp: NaiveDateTime,
+    pub station_code: String,
+    pub wl_msl: f64,
+    pub discharge: f64,
+}
+
+/// `data._prepare_hii_levels`: rows without a level dropped, one row per
+/// station-hour, the latest reading of the hour winning (stable on ties).
+pub fn prepare(rows: &[HiiRow]) -> Vec<HiiRow> {
+    let mut kept: Vec<&HiiRow> = rows.iter().filter(|r| !r.wl_msl.is_nan()).collect();
+    kept.sort_by_key(|r| r.timestamp); // stable
+    let mut by_hour: BTreeMap<(String, NaiveDateTime), HiiRow> = BTreeMap::new();
+    for r in kept {
+        let t = floor_hour(r.timestamp);
+        by_hour.insert(
+            (r.station_code.clone(), t),
+            HiiRow {
+                timestamp: t,
+                ..r.clone()
+            },
+        );
+    }
+    by_hour.into_values().collect()
+}
+
+/// The HII station codes worth fetching for these RID rows.
+pub fn fill_candidates(rid: &[Row]) -> Vec<String> {
+    let present: HashSet<&str> = rid.iter().map(|r| r.station_code.as_str()).collect();
+    config()
+        .hii_fill_stations
+        .iter()
+        .filter(|c| present.contains(c.as_str()))
+        .cloned()
+        .collect()
+}
+
+/// Per-station result of the fill, for logs and the notifier.
+#[derive(Clone, Debug, PartialEq)]
+pub struct FillInfo {
+    pub station_code: String,
+    pub offset: f64,
+    pub filled_hours: usize,
+    /// The newest filled hour and its level.
+    pub latest: Option<(NaiveDateTime, f64)>,
+}
+
+/// pandas `Series.median` (numpy: mean of the two middle values when even).
+fn median(mut v: Vec<f64>) -> f64 {
+    v.sort_by(f64::total_cmp);
+    let n = v.len();
+    if n % 2 == 1 {
+        v[n / 2]
+    } else {
+        (v[n / 2 - 1] + v[n / 2]) / 2.0
+    }
+}
+
+/// `data.fill_from_hii`: `rid` as loaded (already one row per station-hour),
+/// `hii` from [`prepare`]. Returns the merged rows (fill rows replace RID rows
+/// that have no level) and what was filled.
+pub fn fill(rid: &[Row], hii: &[HiiRow]) -> (Vec<Row>, Vec<FillInfo>) {
+    let cfg = config();
+    let candidates: BTreeSet<String> = fill_candidates(rid).into_iter().collect();
+    let mut mirrors: BTreeMap<&str, Vec<&HiiRow>> = BTreeMap::new();
+    for h in hii {
+        if candidates.contains(&h.station_code) {
+            mirrors.entry(h.station_code.as_str()).or_default().push(h);
+        }
+    }
+    let mut fills: Vec<Row> = Vec::new();
+    let mut info = Vec::new();
+    for (code, mirror) in mirrors {
+        let base: BTreeMap<NaiveDateTime, f64> = rid
+            .iter()
+            .filter(|r| r.station_code == code)
+            .map(|r| (r.timestamp, r.water_level))
+            .collect();
+        let diffs: Vec<f64> = mirror
+            .iter()
+            .filter_map(|m| match base.get(&m.timestamp) {
+                Some(level) if !level.is_nan() => Some(m.wl_msl - level),
+                _ => None,
+            })
+            .collect();
+        if diffs.len() < cfg.hii_min_overlap_hours {
+            continue;
+        }
+        let offset = median(diffs);
+        let exact = cfg.hii_exact_mirrors.iter().any(|c| c == code);
+        let before = fills.len();
+        for m in mirror {
+            if base.get(&m.timestamp).is_some_and(|l| !l.is_nan()) {
+                continue;
+            }
+            fills.push(Row {
+                timestamp: m.timestamp,
+                station_code: code.to_string(),
+                water_level: m.wl_msl - offset,
+                discharge: if exact { m.discharge } else { f64::NAN },
+            });
+        }
+        if fills.len() > before {
+            info.push(FillInfo {
+                station_code: code.to_string(),
+                offset,
+                filled_hours: fills.len() - before,
+                latest: fills
+                    .iter()
+                    .skip(before)
+                    .max_by_key(|r| r.timestamp)
+                    .map(|r| (r.timestamp, r.water_level)),
+            });
+        }
+    }
+    if fills.is_empty() {
+        return (rid.to_vec(), info);
+    }
+    // concat + drop_duplicates(keep="last"): a fill replaces a level-less RID row
+    let mut merged: BTreeMap<(String, NaiveDateTime), Row> = BTreeMap::new();
+    for r in rid.iter().chain(fills.iter()) {
+        merged.insert((r.station_code.clone(), r.timestamp), r.clone());
+    }
+    (merged.into_values().collect(), info)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use chrono::{Duration, NaiveDate};
+
+    fn t0() -> NaiveDateTime {
+        NaiveDate::from_ymd_opt(2026, 10, 1)
+            .unwrap()
+            .and_hms_opt(0, 0, 0)
+            .unwrap()
+    }
+
+    fn rid(code: &str, hours: std::ops::Range<i64>) -> Vec<Row> {
+        hours
+            .map(|h| Row {
+                timestamp: t0() + Duration::hours(h),
+                station_code: code.into(),
+                water_level: 1.0 + h as f64 * 0.01,
+                discharge: 50.0,
+            })
+            .collect()
+    }
+
+    fn mirror(code: &str, hours: std::ops::Range<i64>, offset: f64) -> Vec<HiiRow> {
+        hours
+            .map(|h| HiiRow {
+                timestamp: t0() + Duration::hours(h),
+                station_code: code.into(),
+                wl_msl: 1.0 + h as f64 * 0.01 + offset,
+                discharge: 51.0,
+            })
+            .collect()
+    }
+
+    #[test]
+    fn trailing_hours_filled_with_the_overlap_offset() {
+        let r = rid("P.1", 0..200);
+        let h = prepare(&mirror("P.1", 0..203, 300.0));
+        let (merged, info) = fill(&r, &h);
+        assert_eq!(merged.len(), 203);
+        assert_eq!(info[0].filled_hours, 3);
+        let last = merged.last().unwrap();
+        assert_eq!(last.timestamp, t0() + Duration::hours(202));
+        assert!((last.water_level - 3.02).abs() < 1e-9);
+        assert_eq!(last.discharge, 51.0); // exact mirror keeps discharge
+    }
+
+    #[test]
+    fn short_overlap_or_other_sensor_is_not_merged() {
+        let r = rid("P.1", 0..100);
+        assert_eq!(
+            fill(&r, &prepare(&mirror("P.1", 0..103, 300.0))).0.len(),
+            100
+        );
+        let r = rid("P.77", 0..200);
+        assert_eq!(
+            fill(&r, &prepare(&mirror("P.77", 0..203, 300.0))).0.len(),
+            200
+        );
+    }
+
+    #[test]
+    fn p81_is_bias_corrected_without_discharge() {
+        let (merged, _) = fill(&rid("P.81", 0..200), &prepare(&mirror("P.81", 0..201, 5.0)));
+        assert!(merged.last().unwrap().discharge.is_nan());
+    }
+
+    #[test]
+    fn latest_reading_of_an_hour_wins() {
+        let mut rows = mirror("P.1", 0..1, 300.0);
+        rows.push(HiiRow {
+            timestamp: t0() + Duration::minutes(40),
+            wl_msl: 302.0,
+            ..rows[0].clone()
+        });
+        rows.push(HiiRow {
+            timestamp: t0() + Duration::minutes(10),
+            wl_msl: 301.5,
+            ..rows[0].clone()
+        });
+        rows.push(HiiRow {
+            wl_msl: f64::NAN,
+            timestamp: t0() + Duration::minutes(50),
+            ..rows[0].clone()
+        });
+        let p = prepare(&rows);
+        assert_eq!(p.len(), 1);
+        assert_eq!(p[0].wl_msl, 302.0);
+        assert_eq!(p[0].timestamp, t0());
+    }
+}

rust/forecast/src/lib.rs

@@ -0,0 +1,24 @@
+//! Flood forecast inference: a port of the serving path of `src/ml`
+//! (`features.py`, `predict.py`, the HII gap fill of `data.py`, `dam.py`).
+//!
+//! Training stays in Python (scikit-learn). `src/ml/export.py` writes each
+//! trained bundle as `models/flood_<station>.json.gz` and the shared constants
+//! as `flood_config.json` (compiled in). Given the same inputs, this crate
+//! reproduces Python's forecasts: features and levels bit for bit,
+//! probabilities to the last bits of `exp` (see `tests/parity.rs`).
+
+pub mod config;
+pub mod dam;
+pub mod features;
+pub mod grid;
+pub mod hgb;
+pub mod hii;
+pub mod model;
+pub mod predict;
+pub mod rain;
+pub mod rolling;
+
+pub use config::config;
+pub use grid::Row;
+pub use model::{Bundle, ModelSet, ModelStore};
+pub use predict::{Forecast, StageProbability, forecasts};