feat: in-memory HII gap-fill in the ML data loader

load_measurements() (DB path) patches missing station-hours from the
hii_waterlevel mirror telemetry: exact mirrors (P.1/P.103/P.20/P.4A/P.67/
P.75/P.82/P.84/P.92) plus bias-corrected P.81 (+9,340 h). Per-station
MSL->gauge offset is derived from >=168 h of series overlap, which
reproduces the published offsets for exact mirrors and absorbs P.81's
bias; P.76/P.77/P.85/P.87 HII twins are different physical sensors and
stay excluded. Training and serving share the loader, so both sides see
identical filled series; water_measurements is never written.

.env.back

@@ -0,0 +1,87 @@
+# Northern Thailand Ping River Monitor Configuration
+# Copy this file to .env and customize for your environment
+
+# Database Configuration
+DB_TYPE=postgresql
+# Options: sqlite, mysql, postgresql, influxdb, victoriametrics
+
+# SQLite Configuration (default)
+WATER_DB_PATH=water_levels.db
+
+# VictoriaMetrics Configuration
+VM_HOST=localhost
+VM_PORT=8428
+VM_URL=
+
+# InfluxDB Configuration
+INFLUX_HOST=localhost
+INFLUX_PORT=8086
+INFLUX_DATABASE=ping_river_monitoring
+INFLUX_USERNAME=
+INFLUX_PASSWORD=
+
+# PostgreSQL Configuration (Remote Server)
+# Option 1: Full connection string (URL encode special characters in password)
+#POSTGRES_CONNECTION_STRING=postgresql://username:url_encoded_password@your-postgres-host:5432/water_monitoring
+
+# Option 2: Individual components (password will be automatically URL encoded)
+POSTGRES_HOST=10.0.10.201
+POSTGRES_PORT=5432
+POSTGRES_DB=ping_river
+POSTGRES_USER=ping_river
+POSTGRES_PASSWORD=3_%m]k:+16"rx?M#`swIA
+
+# Examples for connection string:
+# - Local: postgresql://postgres:password@localhost:5432/water_monitoring
+# - Remote: postgresql://user:pass@192.168.1.100:5432/water_monitoring
+# - With special chars: postgresql://user:my%3Apass%40word@host:5432/db
+# - With SSL: postgresql://user:pass@host:port/db?sslmode=require
+# - Connection pooling: postgresql://user:pass@host:port/db?pool_size=20&max_overflow=0
+
+# Special character URL encoding:
+# : → %3A    @ → %40    # → %23    ? → %3F    & → %26    / → %2F    % → %25
+
+# MySQL Configuration
+MYSQL_CONNECTION_STRING=mysql://user:password@localhost:3306/ping_river_monitoring
+
+# API Configuration
+API_HOST=0.0.0.0
+API_PORT=8000
+API_WORKERS=1
+
+# Data Collection Settings
+SCRAPING_INTERVAL_HOURS=1
+REQUEST_TIMEOUT=30
+MAX_RETRIES=3
+RETRY_DELAY_SECONDS=60
+
+# Data Retention
+DATA_RETENTION_DAYS=365
+
+# Logging Configuration
+LOG_LEVEL=INFO
+LOG_FILE=water_monitor.log
+
+# Security (for production)
+SECRET_KEY=your-secret-key-here
+API_KEY=your-api-key-here
+
+# Monitoring
+ENABLE_METRICS=true
+ENABLE_HEALTH_CHECKS=true
+
+# Geographic Settings
+TIMEZONE=Asia/Bangkok
+DEFAULT_LATITUDE=18.7875
+DEFAULT_LONGITUDE=99.0045
+
+# External Services
+NOTIFICATION_EMAIL=
+SMTP_SERVER=
+SMTP_PORT=587
+SMTP_USERNAME=
+SMTP_PASSWORD=
+
+# Development Settings
+DEBUG=false
+DEVELOPMENT_MODE=false

.env.example

@@ -2,7 +2,7 @@
 # Copy this file to .env and customize for your environment
 
 # Database Configuration
-DB_TYPE=sqlite
+DB_TYPE=postgresql
 # Options: sqlite, mysql, postgresql, influxdb, victoriametrics
 
 # SQLite Configuration (default)
@@ -20,8 +20,26 @@ INFLUX_DATABASE=ping_river_monitoring
 INFLUX_USERNAME=
 INFLUX_PASSWORD=
 
-# PostgreSQL Configuration
-POSTGRES_CONNECTION_STRING=postgresql://user:password@localhost:5432/ping_river_monitoring
+# PostgreSQL Configuration (Remote Server)
+# Option 1: Full connection string (URL encode special characters in password)
+POSTGRES_CONNECTION_STRING=postgresql://username:url_encoded_password@your-postgres-host:5432/water_monitoring
+
+# Option 2: Individual components (password will be automatically URL encoded)
+POSTGRES_HOST=your-postgres-host
+POSTGRES_PORT=5432
+POSTGRES_DB=water_monitoring
+POSTGRES_USER=username
+POSTGRES_PASSWORD=your:password@with!special#chars
+
+# Examples for connection string:
+# - Local: postgresql://postgres:password@localhost:5432/water_monitoring
+# - Remote: postgresql://user:pass@192.168.1.100:5432/water_monitoring
+# - With special chars: postgresql://user:my%3Apass%40word@host:5432/db
+# - With SSL: postgresql://user:pass@host:port/db?sslmode=require
+# - Connection pooling: postgresql://user:pass@host:port/db?pool_size=20&max_overflow=0
+
+# Special character URL encoding:
+# : → %3A    @ → %40    # → %23    ? → %3F    & → %26    / → %2F    % → %25
 
 # MySQL Configuration
 MYSQL_CONNECTION_STRING=mysql://user:password@localhost:3306/ping_river_monitoring
@@ -30,6 +48,8 @@ MYSQL_CONNECTION_STRING=mysql://user:password@localhost:3306/ping_river_monitori
 API_HOST=0.0.0.0
 API_PORT=8000
 API_WORKERS=1
+# Public ThaiWater API key used to add Ping-basin water-level sensors.
+THAIWATER_API_KEY=
 
 # Data Collection Settings
 SCRAPING_INTERVAL_HOURS=1
@@ -64,6 +84,18 @@ SMTP_PORT=587
 SMTP_USERNAME=
 SMTP_PASSWORD=
 
+# Matrix Alerting Configuration
+MATRIX_HOMESERVER=https://matrix.org
+MATRIX_ACCESS_TOKEN=
+MATRIX_ROOM_ID=
+
+# Grafana Integration
+GRAFANA_URL=http://localhost:3000
+
+# Alert Configuration
+ALERT_MAX_AGE_HOURS=2
+ALERT_CHECK_INTERVAL_MINUTES=15
+
 # Development Settings
 DEBUG=false
 DEVELOPMENT_MODE=false

.env.postgres

@@ -0,0 +1,2 @@
+DB_TYPE=postgresql
+POSTGRES_CONNECTION_STRING=postgresql://postgres:password@localhost:5432/water_monitoring

.gitea/workflows/ci.yml

@@ -2,9 +2,9 @@ name: CI/CD Pipeline - Northern Thailand Ping River Monitor
 
 on:
   push:
-    branches: [ main, develop ]
+    branches: [ master, develop ]
   pull_request:
-    branches: [ main ]
+    branches: [ master ]
   schedule:
     # Run tests daily at 2 AM UTC
     - cron: '0 2 * * *'
@@ -23,7 +23,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.11']  # pandas 2.0.3 ships no 3.12 wheels; widen after upgrading pandas
     
     steps:
     - name: Checkout code
@@ -55,9 +55,10 @@ jobs:
         flake8 src/ --count --select=E9,F63,F7,F82 --show-source --statistics
         flake8 src/ --count --exit-zero --max-complexity=10 --max-line-length=100 --statistics
         
-    - name: Type check with mypy
+    - name: Type check with mypy (advisory)
       run: |
-        mypy src/ --ignore-missing-imports
+        # 86 pre-existing errors; blocking typing gate deferred until the debt is paid down
+        mypy src/ --ignore-missing-imports || true
         
     - name: Format check with black
       run: |
@@ -271,7 +272,7 @@ jobs:
     name: Deploy to Production
     runs-on: ubuntu-latest
     needs: [test, build, integration-test]
-    if: github.ref == 'refs/heads/main'
+    if: github.ref == 'refs/heads/master'
     environment:
       name: production
       url: https://ping-river-monitor.b4l.co.th
@@ -303,7 +304,7 @@ jobs:
     name: Performance Test
     runs-on: ubuntu-latest
     needs: deploy-production
-    if: github.ref == 'refs/heads/main'
+    if: github.ref == 'refs/heads/master'
     
     steps:
     - name: Checkout code

.gitea/workflows/docs.yml

@@ -2,7 +2,7 @@ name: Documentation
 
 on:
   push:
-    branches: [ main, develop ]
+    branches: [ master, develop ]
     paths:
       - 'docs/**'
       - 'README.md'

.gitignore

@@ -134,4 +134,20 @@ cython_debug/
 
 # Docker volumes
 vm_data/
-grafana_data/
+grafana_data/
+# Runtime station config (persisted CRUD); bundled default lives in src/data/
+/stations.json
+
+# Ruflo local secrets and runtime data
+.env.*.local
+.claude-flow/data/
+.claude-flow/logs/
+.claude-flow/sessions/
+
+# Trained flood-forecast model artifacts (produced on the server, ~100 MB; see docs/FLOOD_FORECASTING.md)
+models/*.joblib
+models/cache/
+models/metrics.json
+
+# Playwright MCP browser artifacts (screenshots/snapshots from agent sessions)
+.playwright-mcp/

.pre-commit-config.yaml

@@ -0,0 +1,40 @@
+# Pre-commit hooks for Northern Thailand Ping River Monitor
+# See https://pre-commit.com for more information
+
+repos:
+  # General file checks
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-json
+      - id: check-toml
+      - id: check-added-large-files
+        args: ['--maxkb=1000']
+      - id: check-merge-conflict
+      - id: check-case-conflict
+      - id: mixed-line-ending
+
+  # Python code formatting with Black
+  - repo: https://github.com/psf/black
+    rev: 23.11.0
+    hooks:
+      - id: black
+        language_version: python3
+        args: ['--line-length=120']
+
+  # Import sorting with isort
+  - repo: https://github.com/pycqa/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+        args: ['--profile', 'black', '--line-length', '120']
+
+  # Linting with flake8
+  - repo: https://github.com/pycqa/flake8
+    rev: 6.1.0
+    hooks:
+      - id: flake8
+        args: ['--max-line-length=120', '--extend-ignore=E203,W503']

MIGRATION_TO_UV.md

@@ -0,0 +1,165 @@
+# Migration to uv
+
+This document describes the migration from traditional Python package management (pip + requirements.txt) to [uv](https://docs.astral.sh/uv/), a fast Python package installer and resolver.
+
+## What Changed
+
+### Files Added
+- `pyproject.toml` - Modern Python project configuration combining dependencies and metadata
+- `.python-version` - Specifies Python version for uv
+- `scripts/setup_uv.sh` - Unix setup script for uv environment
+- `scripts/setup_uv.bat` - Windows setup script for uv environment
+- This migration guide
+
+### Files Modified
+- `Makefile` - Updated all commands to use `uv run` instead of direct Python execution
+
+### Files That Can Be Removed (Optional)
+- `requirements.txt` - Dependencies now in pyproject.toml
+- `requirements-dev.txt` - Dev dependencies now in pyproject.toml
+- `setup.py` - Configuration now in pyproject.toml
+
+## Installation
+
+### Install uv
+
+**Unix/macOS:**
+```bash
+curl -LsSf https://astral.sh/uv/install.sh | sh
+```
+
+**Windows (PowerShell):**
+```powershell
+powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"
+```
+
+### Setup Project
+
+**Unix/macOS:**
+```bash
+# Run the setup script
+chmod +x scripts/setup_uv.sh
+./scripts/setup_uv.sh
+
+# Or manually:
+uv sync
+uv run pre-commit install
+```
+
+**Windows:**
+```batch
+REM Run the setup script
+scripts\setup_uv.bat
+
+REM Or manually:
+uv sync
+uv run pre-commit install
+```
+
+## New Workflow
+
+### Common Commands
+
+| Old Command | New Command | Description |
+|-------------|-------------|-------------|
+| `pip install -r requirements.txt` | `uv sync --no-dev` | Install production dependencies |
+| `pip install -r requirements-dev.txt` | `uv sync` | Install all dependencies (including dev) |
+| `python run.py` | `uv run python run.py` | Run the application |
+| `pytest` | `uv run pytest` | Run tests |
+| `black src/` | `uv run black src/` | Format code |
+
+### Using the Makefile
+
+The Makefile has been updated to use uv, so all existing commands work the same:
+
+```bash
+make install-dev    # Install dev dependencies with uv
+make test          # Run tests with uv
+make run-api       # Start API server with uv
+make lint          # Lint code with uv
+make format        # Format code with uv
+```
+
+### Adding Dependencies
+
+**Production dependency:**
+```bash
+uv add requests
+```
+
+**Development dependency:**
+```bash
+uv add --dev pytest
+```
+
+**Specific version:**
+```bash
+uv add "fastapi==0.104.1"
+```
+
+### Managing Python Versions
+
+uv can automatically manage Python versions:
+
+```bash
+# Install and use Python 3.11
+uv python install 3.11
+uv sync
+
+# Use specific Python version
+uv sync --python 3.11
+```
+
+## Benefits of uv
+
+1. **Speed** - 10-100x faster than pip
+2. **Reliability** - Better dependency resolution
+3. **Simplicity** - Single tool for packages and Python versions
+4. **Reproducibility** - Lock file ensures consistent environments
+5. **Modern** - Built-in support for pyproject.toml
+
+## Troubleshooting
+
+### Command not found
+Make sure uv is in your PATH after installation. Restart your terminal or run:
+```bash
+source ~/.bashrc  # or ~/.zshrc
+```
+
+### Lock file conflicts
+If you encounter lock file issues:
+```bash
+rm uv.lock
+uv sync
+```
+
+### Python version issues
+Ensure the Python version in `.python-version` is available:
+```bash
+uv python list
+uv python install 3.11  # if needed
+```
+
+## Rollback (if needed)
+
+If you need to rollback to the old system:
+
+1. Use the original requirements files:
+   ```bash
+   pip install -r requirements.txt
+   pip install -r requirements-dev.txt
+   ```
+
+2. Revert the Makefile changes to use `python` instead of `uv run python`
+
+3. Remove uv-specific files:
+   ```bash
+   rm pyproject.toml .python-version uv.lock
+   rm -rf .venv  # if created by uv
+   ```
+
+## Additional Resources
+
+- [uv Documentation](https://docs.astral.sh/uv/)
+- [Migration Guide](https://docs.astral.sh/uv/guides/projects/)
+- [pyproject.toml Reference](https://packaging.python.org/en/latest/specifications/pyproject-toml/)

Makefile

@@ -21,39 +21,55 @@ help:
 	@echo "  run          Run the monitor in continuous mode"
 	@echo "  run-api      Run the web API server"
 	@echo "  run-test     Run a single test cycle"
+	@echo "  run-status   Show system status"
+	@echo ""
+	@echo "Alerting:"
+	@echo "  alert-check  Check water levels and send alerts"
+	@echo "  alert-test   Send test Matrix message"
+	@echo ""
+	@echo "Distribution:"
+	@echo "  build-exe    Build standalone executable"
+	@echo "  package      Build and create distribution package"
 	@echo ""
 	@echo "Docker:"
 	@echo "  docker-build Build Docker image"
 	@echo "  docker-run   Run with Docker Compose"
 	@echo "  docker-stop  Stop Docker services"
 	@echo ""
+	@echo "Database:"
+	@echo "  setup-postgres Setup PostgreSQL database"
+	@echo "  test-postgres  Test PostgreSQL connection"
+	@echo "  encode-password URL encode password for connection string"
+	@echo "  migrate-sqlite Migrate SQLite data to PostgreSQL"
+	@echo "  migrate-fast   Fast migration with 10K batch size"
+	@echo "  analyze-sqlite Analyze SQLite database structure (dry run)"
+	@echo ""
 	@echo "Documentation:"
 	@echo "  docs         Generate documentation"
 
 # Installation
 install:
-	pip install -r requirements.txt
+	uv sync --no-dev
 
 install-dev:
-	pip install -r requirements-dev.txt
-	pre-commit install
+	uv sync
+	uv run pre-commit install
 
 # Testing
 test:
-	python test_integration.py
-	python test_station_management.py
+	uv run pytest -q
 
 test-cov:
-	pytest --cov=src --cov-report=html --cov-report=term
+	uv run pytest --cov=src --cov-report=html --cov-report=term
 
 # Code quality
 lint:
-	flake8 src/ --max-line-length=100
-	mypy src/
+	uv run flake8 src/ --max-line-length=100
+	uv run mypy src/
 
 format:
-	black src/ *.py
-	isort src/ *.py
+	uv run black src/ *.py
+	uv run isort src/ *.py
 
 # Cleanup
 clean:
@@ -69,16 +85,23 @@ clean:
 
 # Running
 run:
-	python run.py
+	uv run python run.py
 
 run-api:
-	python run.py --web-api
+	uv run python run.py --web-api
 
 run-test:
-	python run.py --test
+	uv run python run.py --test
 
 run-status:
-	python run.py --status
+	uv run python run.py --status
+
+# Alerting
+alert-check:
+	uv run python run.py --alert-check
+
+alert-test:
+	uv run python run.py --alert-test
 
 # Docker
 docker-build:
@@ -99,7 +122,7 @@ docs:
 
 # Database management
 db-migrate:
-	python scripts/migrate_geolocation.py
+	uv run python scripts/migrate_geolocation.py
 
 # Monitoring
 health-check:
@@ -116,9 +139,38 @@ dev-setup: install-dev
 
 # Production deployment
 deploy-check:
-	python run.py --test
+	uv run python run.py --test
 	@echo "Deployment check passed!"
 
+# Database management
+setup-postgres:
+	uv run python scripts/setup_postgres.py
+
+test-postgres:
+	uv run python -c "from scripts.setup_postgres import test_postgres_connection; from src.config import Config; config = Config.get_database_config(); test_postgres_connection(config['connection_string'])"
+
+encode-password:
+	uv run python scripts/encode_password.py
+
+migrate-sqlite:
+	uv run python scripts/migrate_sqlite_to_postgres.py
+
+migrate-fast:
+	uv run python scripts/migrate_sqlite_to_postgres.py --fast
+
+analyze-sqlite:
+	uv run python scripts/migrate_sqlite_to_postgres.py --dry-run
+
+# Distribution
+build-exe:
+	uv run python build_simple.py
+
+package: build-exe
+	@echo "Creating distribution package..."
+	@if exist dist\ping-river-monitor-distribution.zip del dist\ping-river-monitor-distribution.zip
+	@cd dist && powershell -Command "Compress-Archive -Path * -DestinationPath ping-river-monitor-distribution.zip -Force"
+	@echo "✅ Distribution package created: dist/ping-river-monitor-distribution.zip"
+
 # Git helpers
 git-setup:
 	git remote add origin https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor.git
@@ -134,7 +186,7 @@ validate-workflows:
 	@echo "Validating Gitea Actions workflows..."
 	@for file in .gitea/workflows/*.yml; do \
 		echo "Checking $$file..."; \
-		python -c "import yaml; yaml.safe_load(open('$$file', encoding='utf-8'))" || exit 1; \
+		uv run python -c "import yaml; yaml.safe_load(open('$$file', encoding='utf-8'))" || exit 1; \
 	done
 	@echo "✅ All workflows are valid"
 

POSTGRESQL_SETUP.md

@@ -0,0 +1,287 @@
+# PostgreSQL Setup for Northern Thailand Ping River Monitor
+
+This guide helps you configure PostgreSQL as the database backend for the water monitoring system.
+
+## Prerequisites
+
+- PostgreSQL server running on a remote machine (already available)
+- Network connectivity to the PostgreSQL server
+- Database credentials (username, password, host, port)
+
+## Quick Setup
+
+### 1. Configure Environment
+
+Copy the example environment file and configure it:
+
+```bash
+cp .env.example .env
+```
+
+Edit `.env` and update the PostgreSQL configuration:
+
+```bash
+# Database Configuration
+DB_TYPE=postgresql
+
+# PostgreSQL Configuration (Remote Server)
+POSTGRES_CONNECTION_STRING=postgresql://username:password@your-postgres-host:5432/water_monitoring
+```
+
+### 2. Run Setup Script
+
+Use the interactive setup script:
+
+```bash
+# Using uv
+uv run python scripts/setup_postgres.py
+
+# Or using make
+make setup-postgres
+```
+
+The script will:
+- Test your database connection
+- Create the database if it doesn't exist
+- Initialize the required tables and indexes
+- Set up sample monitoring stations
+
+### 3. Test Connection
+
+Test your PostgreSQL connection:
+
+```bash
+make test-postgres
+```
+
+### 4. Run the Application
+
+Start collecting data:
+
+```bash
+# Run a test cycle
+make run-test
+
+# Start the web API
+make run-api
+```
+
+## Manual Configuration
+
+If you prefer manual setup, here's what you need:
+
+### Connection String Format
+
+```
+postgresql://username:password@host:port/database
+```
+
+**Examples:**
+- Basic: `postgresql://postgres:mypassword@192.168.1.100:5432/water_monitoring`
+- With SSL: `postgresql://user:pass@host:5432/db?sslmode=require`
+- With connection pooling: `postgresql://user:pass@host:5432/db?pool_size=20&max_overflow=0`
+
+### Environment Variables
+
+| Variable | Description | Example |
+|----------|-------------|---------|
+| `DB_TYPE` | Database type | `postgresql` |
+| `POSTGRES_CONNECTION_STRING` | Full connection string | See above |
+
+### Database Schema
+
+The application uses these main tables:
+
+1. **stations** - Monitoring station information
+2. **water_measurements** - Time series water level data
+3. **alert_thresholds** - Warning/danger level definitions
+4. **data_quality_log** - Data collection issue tracking
+
+See `sql/init_postgres.sql` for the complete schema.
+
+## Connection Options
+
+### SSL Connection
+
+For secure connections, add SSL parameters:
+
+```bash
+POSTGRES_CONNECTION_STRING=postgresql://user:pass@host:5432/db?sslmode=require
+```
+
+SSL modes:
+- `disable` - No SSL
+- `require` - Require SSL
+- `prefer` - Use SSL if available
+- `verify-ca` - Verify certificate authority
+- `verify-full` - Full certificate verification
+
+### Connection Pooling
+
+For high-performance applications, configure connection pooling:
+
+```bash
+POSTGRES_CONNECTION_STRING=postgresql://user:pass@host:5432/db?pool_size=20&max_overflow=0
+```
+
+Parameters:
+- `pool_size` - Number of connections to maintain
+- `max_overflow` - Additional connections allowed
+- `pool_timeout` - Seconds to wait for connection
+- `pool_recycle` - Seconds before connection refresh
+
+## Troubleshooting
+
+### Common Issues
+
+**1. Connection Refused**
+```
+psycopg2.OperationalError: could not connect to server
+```
+- Check if PostgreSQL server is running
+- Verify host/port in connection string
+- Check firewall settings
+
+**2. Authentication Failed**
+```
+psycopg2.OperationalError: FATAL: password authentication failed
+```
+- Verify username/password in connection string
+- Check PostgreSQL pg_hba.conf configuration
+- Ensure user has database access permissions
+
+**3. Database Does Not Exist**
+```
+psycopg2.OperationalError: FATAL: database "water_monitoring" does not exist
+```
+- Run the setup script to create the database
+- Or manually create: `CREATE DATABASE water_monitoring;`
+
+**4. Permission Denied**
+```
+psycopg2.ProgrammingError: permission denied for table
+```
+- Ensure user has appropriate permissions
+- Grant access: `GRANT ALL PRIVILEGES ON DATABASE water_monitoring TO username;`
+
+### Network Configuration
+
+For remote PostgreSQL servers, ensure:
+
+1. **PostgreSQL allows remote connections** (`postgresql.conf`):
+   ```
+   listen_addresses = '*'
+   port = 5432
+   ```
+
+2. **Client authentication is configured** (`pg_hba.conf`):
+   ```
+   # Allow connections from your application server
+   host    water_monitoring    username    your.app.ip/32    md5
+   ```
+
+3. **Firewall allows PostgreSQL port**:
+   ```bash
+   # On PostgreSQL server
+   sudo ufw allow 5432/tcp
+   ```
+
+### Performance Tuning
+
+For optimal performance with time series data:
+
+1. **Increase work_mem** for sorting operations
+2. **Tune shared_buffers** for caching
+3. **Configure maintenance_work_mem** for indexing
+4. **Set up regular VACUUM and ANALYZE** for statistics
+
+Example PostgreSQL configuration additions:
+```
+# postgresql.conf
+shared_buffers = 256MB
+work_mem = 16MB
+maintenance_work_mem = 256MB
+effective_cache_size = 1GB
+```
+
+## Monitoring
+
+### Check Application Status
+
+```bash
+# View current configuration
+uv run python -c "from src.config import Config; Config.print_settings()"
+
+# Test database connection
+make test-postgres
+
+# Check latest data
+psql "postgresql://user:pass@host:5432/water_monitoring" -c "SELECT COUNT(*) FROM water_measurements;"
+```
+
+### PostgreSQL Monitoring
+
+Connect directly to check database status:
+
+```bash
+# Connect to database
+psql "postgresql://username:password@host:5432/water_monitoring"
+
+# Check table sizes
+\dt+
+
+# View latest measurements
+SELECT * FROM latest_measurements LIMIT 10;
+
+# Check data quality
+SELECT issue_type, COUNT(*) FROM data_quality_log
+WHERE created_at > NOW() - INTERVAL '24 hours'
+GROUP BY issue_type;
+```
+
+## Backup and Maintenance
+
+### Backup Database
+
+```bash
+# Full backup
+pg_dump "postgresql://user:pass@host:5432/water_monitoring" > backup.sql
+
+# Data only
+pg_dump --data-only "postgresql://user:pass@host:5432/water_monitoring" > data_backup.sql
+```
+
+### Restore Database
+
+```bash
+# Restore full backup
+psql "postgresql://user:pass@host:5432/water_monitoring" < backup.sql
+
+# Restore data only
+psql "postgresql://user:pass@host:5432/water_monitoring" < data_backup.sql
+```
+
+### Regular Maintenance
+
+Set up regular maintenance tasks:
+
+```sql
+-- Update table statistics (run weekly)
+ANALYZE;
+
+-- Reclaim disk space (run monthly)
+VACUUM;
+
+-- Reindex tables (run quarterly)
+REINDEX DATABASE water_monitoring;
+```
+
+## Next Steps
+
+1. Set up monitoring and alerting
+2. Configure data retention policies
+3. Set up automated backups
+4. Implement connection pooling if needed
+5. Configure SSL for production use
+
+For more advanced configuration, see the [PostgreSQL documentation](https://www.postgresql.org/docs/).

README.md

@@ -244,7 +244,7 @@ water_level{station_code="P.1"}
 ```sql
 -- Latest readings from all stations
 SELECT s.station_code, s.english_name, m.water_level, m.discharge
-FROM stations s 
+FROM stations s
 JOIN water_measurements m ON s.id = m.station_id
 WHERE m.timestamp = (SELECT MAX(timestamp) FROM water_measurements WHERE station_id = s.id);
 ```
@@ -267,14 +267,32 @@ docker run -d \
 
 ### Systemd Service (Linux)
 
-```bash
-# Copy service file
-sudo cp scripts/water-monitor.service /etc/systemd/system/
+The install script sets everything up: a dedicated `water-monitor` system user,
+a deploy to `/opt/thailand-water-monitor`, a uv-managed virtualenv, and the
+enabled systemd unit.
 
-# Enable and start
+```bash
+# From a checkout of the repo, as root:
+sudo bash scripts/install.sh
+
+# Then start and check:
+sudo systemctl start water-monitor.service
+systemctl status water-monitor.service
+```
+
+Fill in `/opt/thailand-water-monitor/.env` (Matrix token/room, DB settings)
+before starting if the script reports it is missing.
+
+<details>
+<summary>Manual setup (if you prefer not to use the script)</summary>
+
+```bash
+sudo useradd --system --no-create-home --shell /usr/sbin/nologin water-monitor
+sudo cp scripts/water-monitor.service /etc/systemd/system/
 sudo systemctl enable water-monitor.service
 sudo systemctl start water-monitor.service
 ```
+</details>
 
 ### Migration for Existing Systems
 
@@ -317,6 +335,7 @@ python src/demo_databases.py all                  # Test all databases
 ## 📚 Documentation
 
 ### Core Documentation
+- **[Data Sources & API Catalog](docs/DATA_SOURCES.md)** - Every ingested and available data source (RID, ThaiWater/HII, dams, rainfall, forecasts)
 - **[Installation Guide](docs/DATABASE_DEPLOYMENT_GUIDE.md)** - Complete setup instructions
 - **[Scheduler Guide](docs/ENHANCED_SCHEDULER_GUIDE.md)** - 15-minute scheduling system
 - **[Geolocation Guide](docs/GEOLOCATION_GUIDE.md)** - Grafana geomap integration
@@ -475,7 +494,7 @@ See [docs/PROJECT_STRUCTURE.md](docs/PROJECT_STRUCTURE.md) for detailed architec
 The project includes comprehensive Gitea Actions workflows:
 
 - **🧪 CI/CD Pipeline** - Automated testing, building, and deployment
-- **🔒 Security Scanning** - Daily vulnerability and dependency checks  
+- **🔒 Security Scanning** - Daily vulnerability and dependency checks
 - **📚 Documentation** - Automated API docs and validation
 - **🚀 Release Management** - Automated releases with multi-arch Docker builds
 

SQLITE_MIGRATION.md

@@ -0,0 +1,278 @@
+# SQLite to PostgreSQL Migration Guide
+
+This guide helps you migrate your existing SQLite water monitoring data to PostgreSQL.
+
+## Quick Migration
+
+### 1. Analyze Your SQLite Database (Optional)
+
+First, check what's in your SQLite database:
+
+```bash
+# Analyze without migrating
+make analyze-sqlite
+
+# Or specify a specific SQLite file
+uv run python scripts/migrate_sqlite_to_postgres.py --dry-run /path/to/your/database.db
+```
+
+### 2. Run the Migration
+
+```bash
+# Auto-detect SQLite file and migrate
+make migrate-sqlite
+
+# Or specify a specific SQLite file
+uv run python scripts/migrate_sqlite_to_postgres.py /path/to/your/database.db
+```
+
+The migration tool will:
+- ✅ Connect to both databases
+- ✅ Analyze your SQLite schema automatically
+- ✅ Migrate station information
+- ✅ Migrate all measurement data in batches
+- ✅ Handle different SQLite table structures
+- ✅ Verify the migration results
+- ✅ Generate a detailed log file
+
+## What Gets Migrated
+
+### Station Data
+- Station IDs and codes
+- Thai and English names
+- Coordinates (latitude/longitude)
+- Geohash data (if available)
+- Creation/update timestamps
+
+### Measurement Data
+- Water level readings
+- Discharge measurements
+- Discharge percentages
+- Timestamps
+- Station associations
+- Data quality status
+
+## Supported SQLite Schemas
+
+The migration tool automatically detects and handles various SQLite table structures:
+
+### Modern Schema
+```sql
+-- Stations
+stations: id, station_code, station_name_th, station_name_en, latitude, longitude, geohash
+
+-- Measurements
+water_measurements: timestamp, station_id, water_level, discharge, discharge_percent, status
+```
+
+### Legacy Schema
+```sql
+-- Stations
+water_stations: station_id, station_code, station_name, lat, lon
+
+-- Measurements
+measurements: timestamp, station_id, water_level, discharge, discharge_percent
+```
+
+### Simple Schema
+```sql
+-- Any table with basic water level data
+-- The tool will adapt and map columns automatically
+```
+
+## Migration Process
+
+### Step 1: Database Connection
+- Connects to your SQLite database
+- Verifies PostgreSQL connection
+- Validates configuration
+
+### Step 2: Schema Analysis
+- Scans SQLite tables and columns
+- Reports data counts
+- Identifies table structures
+
+### Step 3: Station Migration
+- Extracts station metadata
+- Maps to PostgreSQL format
+- Handles missing data gracefully
+
+### Step 4: Measurement Migration
+- Processes data in batches (1000 records at a time)
+- Converts timestamps correctly
+- Preserves all measurement values
+- Shows progress during migration
+
+### Step 5: Verification
+- Compares record counts
+- Validates data integrity
+- Reports migration statistics
+
+## Command Options
+
+```bash
+# Basic migration (auto-detects SQLite file)
+uv run python scripts/migrate_sqlite_to_postgres.py
+
+# Specify SQLite database path
+uv run python scripts/migrate_sqlite_to_postgres.py /path/to/database.db
+
+# Dry run (analyze only, no migration)
+uv run python scripts/migrate_sqlite_to_postgres.py --dry-run
+
+# Custom batch size for large databases
+uv run python scripts/migrate_sqlite_to_postgres.py --batch-size 5000
+```
+
+## Auto-Detection
+
+The tool automatically searches for SQLite files in common locations:
+- `water_levels.db`
+- `water_monitoring.db`
+- `database.db`
+- `../water_levels.db`
+
+## Migration Output
+
+The tool provides detailed logging:
+
+```
+========================================
+SQLite to PostgreSQL Migration Tool
+========================================
+SQLite database: water_levels.db
+PostgreSQL: postgresql
+
+Step 1: Connecting to databases...
+Connected to SQLite database: water_levels.db
+Connected to PostgreSQL database
+
+Step 2: Analyzing SQLite database structure...
+Table 'stations': 8 columns, 25 rows
+Table 'water_measurements': 7 columns, 15420 rows
+
+Step 3: Migrating station data...
+Migrated 25 stations
+
+Step 4: Migrating measurement data...
+Found 15420 measurements to migrate
+Migrated 1000/15420 measurements
+Migrated 2000/15420 measurements
+...
+Successfully migrated 15420 measurements
+
+Step 5: Verifying migration...
+SQLite stations: 25
+SQLite measurements: 15420
+PostgreSQL measurements retrieved: 15420
+Migrated stations: 25
+Migrated measurements: 15420
+
+========================================
+MIGRATION COMPLETED
+========================================
+Duration: 0:02:15
+Stations migrated: 25
+Measurements migrated: 15420
+No errors encountered
+```
+
+## Error Handling
+
+The migration tool is robust and handles:
+- **Missing tables** - Tries alternative table names
+- **Different column names** - Maps common variations
+- **Missing data** - Uses sensible defaults
+- **Invalid timestamps** - Attempts multiple date formats
+- **Connection issues** - Provides clear error messages
+- **Large datasets** - Processes in batches to avoid memory issues
+
+## Log Files
+
+Migration creates a detailed log file:
+- `migration.log` - Complete migration log
+- Shows all operations, errors, and statistics
+- Useful for troubleshooting
+
+## Troubleshooting
+
+### Common Issues
+
+**1. SQLite file not found**
+```
+SQLite database file not found. Please specify the path:
+  python migrate_sqlite_to_postgres.py /path/to/database.db
+```
+**Solution**: Specify the correct path to your SQLite file
+
+**2. PostgreSQL not configured**
+```
+Error: PostgreSQL not configured. Set DB_TYPE=postgresql in your .env file
+```
+**Solution**: Ensure your .env file has `DB_TYPE=postgresql`
+
+**3. Connection failed**
+```
+Database connection error: connection refused
+```
+**Solution**: Check your PostgreSQL connection settings
+
+**4. No tables found**
+```
+Could not analyze SQLite database structure
+```
+**Solution**: Verify your SQLite file contains water monitoring data
+
+### Performance Tips
+
+- **Large databases**: Use `--batch-size 5000` for faster processing
+- **Slow networks**: Reduce batch size to `--batch-size 100`
+- **Memory issues**: Process smaller batches
+
+## After Migration
+
+Once migration is complete:
+
+1. **Verify data**:
+   ```bash
+   make run-test
+   make run-api
+   ```
+
+2. **Check the web interface**: Latest readings should show your migrated data
+
+3. **Backup your SQLite**: Keep the original file as backup
+
+4. **Update configurations**: Remove SQLite references from configs
+
+## Rollback
+
+If you need to rollback:
+
+1. **Clear PostgreSQL data**:
+   ```sql
+   DELETE FROM water_measurements;
+   DELETE FROM stations;
+   ```
+
+2. **Switch back to SQLite**:
+   ```bash
+   # In .env file
+   DB_TYPE=sqlite
+   WATER_DB_PATH=water_levels.db
+   ```
+
+3. **Test the rollback**:
+   ```bash
+   make run-test
+   ```
+
+The migration tool is designed to be safe and can be run multiple times - it handles duplicates appropriately.
+
+## Next Steps
+
+After successful migration:
+- Set up automated backups for PostgreSQL
+- Configure monitoring and alerting
+- Consider data retention policies
+- Update documentation references

build_executable.py

@@ -0,0 +1,311 @@
+#!/usr/bin/env python3
+"""
+Build script to create a standalone executable for Northern Thailand Ping River Monitor
+"""
+
+import os
+import shutil
+import sys
+from pathlib import Path
+
+
+def create_spec_file():
+    """Create PyInstaller spec file"""
+    spec_content = """
+# -*- mode: python ; coding: utf-8 -*-
+
+block_cipher = None
+
+# Data files to include
+data_files = [
+    ('.env', '.'),
+    ('sql/*.sql', 'sql'),
+    ('README.md', '.'),
+    ('POSTGRESQL_SETUP.md', '.'),
+    ('SQLITE_MIGRATION.md', '.'),
+]
+
+# Hidden imports that PyInstaller might miss
+hidden_imports = [
+    'psycopg2',
+    'psycopg2-binary',
+    'sqlalchemy.dialects.postgresql',
+    'sqlalchemy.dialects.sqlite',
+    'sqlalchemy.dialects.mysql',
+    'influxdb',
+    'pymysql',
+    'dotenv',
+    'pydantic',
+    'fastapi',
+    'uvicorn',
+    'schedule',
+    'pandas',
+    'requests',
+    'psutil',
+]
+
+a = Analysis(
+    ['run.py'],
+    pathex=['.'],
+    binaries=[],
+    datas=data_files,
+    hiddenimports=hidden_imports,
+    hookspath=[],
+    hooksconfig={},
+    runtime_hooks=[],
+    excludes=[
+        'tkinter',
+        'matplotlib',
+        'PIL',
+        'jupyter',
+        'notebook',
+        'IPython',
+    ],
+    win_no_prefer_redirects=False,
+    win_private_assemblies=False,
+    cipher=block_cipher,
+    noarchive=False,
+)
+
+pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher)
+
+exe = EXE(
+    pyz,
+    a.scripts,
+    a.binaries,
+    a.zipfiles,
+    a.datas,
+    [],
+    name='ping-river-monitor',
+    debug=False,
+    bootloader_ignore_signals=False,
+    strip=False,
+    upx=True,
+    upx_exclude=[],
+    runtime_tmpdir=None,
+    console=True,
+    disable_windowed_traceback=False,
+    argv_emulation=False,
+    target_arch=None,
+    codesign_identity=None,
+    entitlements_file=None,
+    icon='icon.ico' if os.path.exists('icon.ico') else None,
+)
+"""
+
+    with open("ping-river-monitor.spec", "w") as f:
+        f.write(spec_content.strip())
+
+    print("[OK] Created ping-river-monitor.spec")
+
+
+def install_pyinstaller():
+    """Install PyInstaller if not present"""
+    try:
+        import PyInstaller
+
+        print("[OK] PyInstaller already installed")
+    except ImportError:
+        print("Installing PyInstaller...")
+        os.system("uv add --dev pyinstaller")
+        print("[OK] PyInstaller installed")
+
+
+def build_executable():
+    """Build the executable"""
+    print("🔨 Building executable...")
+
+    # Clean previous builds
+    if os.path.exists("dist"):
+        shutil.rmtree("dist")
+    if os.path.exists("build"):
+        shutil.rmtree("build")
+
+    # Build with PyInstaller using uv
+    result = os.system("uv run pyinstaller ping-river-monitor.spec --clean --noconfirm")
+
+    if result == 0:
+        print("✅ Executable built successfully!")
+
+        # Copy additional files to dist directory
+        dist_dir = Path("dist")
+        if dist_dir.exists():
+            # Copy .env file if it exists
+            if os.path.exists(".env"):
+                shutil.copy2(".env", dist_dir / ".env")
+                print("✅ Copied .env file")
+
+            # Copy documentation
+            for doc in ["README.md", "POSTGRESQL_SETUP.md", "SQLITE_MIGRATION.md"]:
+                if os.path.exists(doc):
+                    shutil.copy2(doc, dist_dir / doc)
+                    print(f"✅ Copied {doc}")
+
+            # Copy SQL files
+            if os.path.exists("sql"):
+                shutil.copytree("sql", dist_dir / "sql", dirs_exist_ok=True)
+                print("✅ Copied SQL files")
+
+            print(f"\n🎉 Executable created: {dist_dir / 'ping-river-monitor.exe'}")
+            print(f"📁 All files in: {dist_dir.absolute()}")
+
+    else:
+        print("❌ Build failed!")
+        return False
+
+    return True
+
+
+def create_batch_files():
+    """Create convenient batch files"""
+    batch_files = {
+        "start.bat": """@echo off
+echo Starting Ping River Monitor...
+ping-river-monitor.exe
+pause
+""",
+        "start-api.bat": """@echo off
+echo Starting Ping River Monitor Web API...
+ping-river-monitor.exe --web-api
+pause
+""",
+        "test.bat": """@echo off
+echo Running Ping River Monitor test...
+ping-river-monitor.exe --test
+pause
+""",
+        "status.bat": """@echo off
+echo Checking Ping River Monitor status...
+ping-river-monitor.exe --status
+pause
+""",
+    }
+
+    dist_dir = Path("dist")
+    for filename, content in batch_files.items():
+        batch_file = dist_dir / filename
+        with open(batch_file, "w") as f:
+            f.write(content)
+        print(f"✅ Created {filename}")
+
+
+def create_readme():
+    """Create deployment README"""
+    readme_content = """# Ping River Monitor - Standalone Executable
+
+This is a standalone executable version of the Northern Thailand Ping River Monitor.
+
+## Quick Start
+
+1. **Configure Database**: Edit `.env` file with your PostgreSQL settings
+2. **Test Connection**: Double-click `test.bat`
+3. **Start Monitoring**: Double-click `start.bat`
+4. **Web Interface**: Double-click `start-api.bat`
+
+## Files Included
+
+- `ping-river-monitor.exe` - Main executable
+- `.env` - Configuration file (EDIT THIS!)
+- `start.bat` - Start continuous monitoring
+- `start-api.bat` - Start web API server
+- `test.bat` - Run a test cycle
+- `status.bat` - Check system status
+- `README.md`, `POSTGRESQL_SETUP.md` - Documentation
+- `sql/` - Database initialization scripts
+
+## Configuration
+
+Edit `.env` file:
+```
+DB_TYPE=postgresql
+POSTGRES_HOST=your-server-ip
+POSTGRES_PORT=5432
+POSTGRES_DB=water_monitoring
+POSTGRES_USER=your-username
+POSTGRES_PASSWORD=your-password
+```
+
+## Usage
+
+### Command Line
+```cmd
+# Continuous monitoring
+ping-river-monitor.exe
+
+# Single test run
+ping-river-monitor.exe --test
+
+# Web API server
+ping-river-monitor.exe --web-api
+
+# Check status
+ping-river-monitor.exe --status
+```
+
+### Batch Files
+- Just double-click the `.bat` files for easy operation
+
+## Troubleshooting
+
+1. **Database Connection Issues**
+   - Check `.env` file settings
+   - Verify PostgreSQL server is accessible
+   - Test with `test.bat`
+
+2. **Permission Issues**
+   - Run as administrator if needed
+   - Check firewall settings for API mode
+
+3. **Log Files**
+   - Check `water_monitor.log` for detailed logs
+   - Logs are created in the same directory as the executable
+
+## Support
+
+For issues or questions, check the documentation files included.
+"""
+
+    with open("dist/DEPLOYMENT_README.txt", "w") as f:
+        f.write(readme_content)
+
+    print("✅ Created DEPLOYMENT_README.txt")
+
+
+def main():
+    """Main build process"""
+    print("Building Ping River Monitor Executable")
+    print("=" * 50)
+
+    # Check if we're in the right directory
+    if not os.path.exists("run.py"):
+        print(
+            "❌ Error: run.py not found. Please run this from the project root directory."
+        )
+        return False
+
+    # Install PyInstaller
+    install_pyinstaller()
+
+    # Create spec file
+    create_spec_file()
+
+    # Build executable
+    if not build_executable():
+        return False
+
+    # Create convenience files
+    create_batch_files()
+    create_readme()
+
+    print("\n" + "=" * 50)
+    print("🎉 BUILD COMPLETE!")
+    print("📁 Check the 'dist' folder for your executable")
+    print("💡 Edit the .env file before distributing")
+    print("🚀 Ready for deployment!")
+
+    return True
+
+
+if __name__ == "__main__":
+    success = main()
+    sys.exit(0 if success else 1)

build_simple.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+"""
+Simple build script for standalone executable
+"""
+
+import os
+import shutil
+import sys
+from pathlib import Path
+
+
+def main():
+    print("Building Ping River Monitor Executable")
+    print("=" * 50)
+
+    # Check if PyInstaller is installed
+    try:
+        import PyInstaller
+
+        print("[OK] PyInstaller available")
+    except ImportError:
+        print("[INFO] Installing PyInstaller...")
+        os.system("uv add --dev pyinstaller")
+
+    # Clean previous builds
+    if os.path.exists("dist"):
+        shutil.rmtree("dist")
+        print("[CLEAN] Removed old dist directory")
+    if os.path.exists("build"):
+        shutil.rmtree("build")
+        print("[CLEAN] Removed old build directory")
+
+    # Build command with all necessary options
+    cmd = [
+        "uv",
+        "run",
+        "pyinstaller",
+        "--onefile",
+        "--console",
+        "--name=ping-river-monitor",
+        "--add-data=.env;.",
+        "--add-data=sql;sql",
+        "--add-data=README.md;.",
+        "--add-data=POSTGRESQL_SETUP.md;.",
+        "--add-data=SQLITE_MIGRATION.md;.",
+        "--hidden-import=psycopg2",
+        "--hidden-import=sqlalchemy.dialects.postgresql",
+        "--hidden-import=sqlalchemy.dialects.sqlite",
+        "--hidden-import=dotenv",
+        "--hidden-import=pydantic",
+        "--hidden-import=fastapi",
+        "--hidden-import=uvicorn",
+        "--hidden-import=schedule",
+        "--hidden-import=pandas",
+        "--clean",
+        "--noconfirm",
+        "run.py",
+    ]
+
+    print("[BUILD] Running PyInstaller...")
+    print("[CMD] " + " ".join(cmd))
+
+    result = os.system(" ".join(cmd))
+
+    if result == 0:
+        print("[SUCCESS] Executable built successfully!")
+
+        # Copy .env file to dist if it exists
+        if os.path.exists(".env") and os.path.exists("dist"):
+            shutil.copy2(".env", "dist/.env")
+            print("[COPY] .env file copied to dist/")
+
+        # Create batch files for easy usage
+        batch_files = {
+            "start.bat": """@echo off
+echo Starting Ping River Monitor...
+ping-river-monitor.exe
+pause
+""",
+            "start-api.bat": """@echo off
+echo Starting Web API...
+ping-river-monitor.exe --web-api
+pause
+""",
+            "test.bat": """@echo off
+echo Running test...
+ping-river-monitor.exe --test
+pause
+""",
+        }
+
+        for filename, content in batch_files.items():
+            if os.path.exists("dist"):
+                with open(f"dist/{filename}", "w") as f:
+                    f.write(content)
+                print(f"[CREATE] {filename}")
+
+        print("\n" + "=" * 50)
+        print("BUILD COMPLETE!")
+        print(f"Executable: dist/ping-river-monitor.exe")
+        print("Batch files: start.bat, start-api.bat, test.bat")
+        print("Don't forget to edit .env file before using!")
+
+        return True
+    else:
+        print("[ERROR] Build failed!")
+        return False
+
+
+if __name__ == "__main__":
+    success = main()
+    sys.exit(0 if success else 1)

docs/DATA_SOURCES.md

@@ -0,0 +1,256 @@
+# Data Sources & External API Catalog
+
+Catalog of every data source available to the Ping River Monitor — what we ingest
+today, what the ThaiWater/HII ecosystem exposes, and vetted external feeds for
+future model inputs (rainfall, dam releases, forecasts).
+
+All "verified" claims below were empirically probed on **2026-08-11**. Endpoints
+marked *(catalog)* were recovered from the thaiwater.net frontend JS bundle but
+not exercised (auth required).
+
+---
+
+## 1. Currently ingested — RID hydrology telemetry
+
+The only source persisted to the database and used by the ML pipeline.
+
+| | |
+|---|---|
+| Endpoint | `POST https://hyd-app-db.rid.go.th/webservice/getGroupHourlyWaterLevelReportAllHL.ashx` |
+| Agency | Royal Irrigation Department (RID) |
+| Auth | None |
+| Cadence | Hourly (`hourlytime` 1.00–24.00; hour 24 = midnight next day) |
+| Params | `DW[UtokID]=1`, `DW[BasinID]=6` (Ping), `DW[TimeCurrent]=<Buddhist-calendar date>`, `rows=100` |
+| Variables | Water level (m, gauge datum), discharge (m³/s, `'***'` = malformed), discharge % of channel capacity |
+| Stations | 16 P-series gauges (P.1 anchor at Nawarat Bridge; see `src/data/stations.json`) |
+| Client | `src/water_scraper_v3.py` |
+
+Human-facing page: <https://hyd-app-db.rid.go.th/hydro1h.html>
+
+---
+
+## 2. ThaiWater / HII ecosystem
+
+ThaiWater (<https://twa.thaiwater.net>) is the National Hydroinformatics
+Institute (HII) portal. It sits on **two distinct API layers** with very
+different access rules.
+
+### 2.1 `api-v3.thaiwater.net` — open, no authentication ✅
+
+Base: `https://api-v3.thaiwater.net/api/v1/thaiwater30/public/`
+
+Undocumented backend of the portal. No key, no session. No published rate
+limits or terms of use — be a good citizen (hourly polls, cache, filter to
+Ping basin `basin_code == 6`).
+
+#### `waterlevel_load` — national water-level snapshot (verified)
+
+```
+GET https://api-v3.thaiwater.net/api/v1/thaiwater30/public/waterlevel_load
+```
+
+- ~2.4 MB, 1,426 stations nationwide; **125 in Ping Basin, 61 in Chiang Mai
+  province, 34 with discharge**.
+- Per station: `waterlevel_m`, `waterlevel_msl`, `waterlevel_msl_previous`,
+  `flow_rate`, `discharge`, `storage_percent`, `situation_level` (1–4 flood
+  severity), `diff_wl_bank`, bank/critical levels (`min_bank`,
+  `critical_level_msl`, `warning_level_m`, `critical_level_m`, `qmax`),
+  `river_name`, basin, geocode, agency, lat/long, `is_key_station`.
+- Ping key stations present: P.1, P.67, P.75, P.81, P.92, P.17, P.87, P.4A,
+  P.7A, P.77, P.2A — plus RID mirrors (`ridhydro_P.1`, …), Tak-reach `TP.`/
+  `TUP.` codes, and HII sensor clusters (`PIN001-011`, `CHM001-005`).
+- **P.1 = internal station id `3226`** (lat 18.786961, long 99.005089).
+
+#### `waterlevel_graph` — hourly historical time series (verified) ⭐
+
+```
+GET .../waterlevel_graph?station_type=tele_waterlevel&station_id=3226&start_date=2024-09-25&end_date=2024-10-08
+```
+
+- Returns `{data: {graph_data: [{datetime, value, value_out, discharge}]}}`,
+  hourly. `value` is **water level in m MSL** (not gauge datum).
+- **Respects arbitrary date ranges. Archive verified back to at least
+  2019-08** (P.1 returned data for 2019-08-01). 14-day windows tested OK;
+  maximum window size not probed.
+- Oct 2024 record flood fully present: peak 305.8 m MSL @ 2024-10-05 12:00,
+  discharge 656 m³/s.
+- Datum conversion at P.1: 305.8 MSL peak = 5.30 m gauge ⇒
+  **gauge ≈ MSL − 300.5 m** (verify per station before use; each station has
+  its own datum offset).
+- Value: independent second historical source for cross-validating / gap-filling
+  the RID feed (RID grid is only ~56% filled).
+
+#### `rain_24h` — national rainfall snapshot (verified)
+
+```
+GET https://api-v3.thaiwater.net/api/v1/thaiwater30/public/rain_24h
+```
+
+- ~4.5 MB, 4,445 stations; **321 in Chiang Mai province**; agencies include
+  HII, DWR, RID.
+- Per station: `rain_1h`, `rain_24h` (mm), `rainfall_datetime`,
+  `station.id` (small int — the graph key), `station.tele_station_oldcode`
+  (e.g. `CHM005`, `STN0410`, `ridtele_TUP.14`), `sub_basin_id`, lat/long,
+  basin, geocode, agency.
+- **This is the missing rainfall input** for the flood model — near-real-time
+  hourly gauge rain across the upper Ping catchment.
+
+#### `rain_24h_graph` — trailing-window rainfall series (verified, limited) ⚠️
+
+```
+GET .../rain_24h_graph?station_type=tele_rainfall&station_id=418&start_date=...&end_date=...
+```
+
+- `station_id` is the **small `station.id`** from `rain_24h` (e.g. 418 =
+  CHM005 "Chiang Mai 5", Mae Taeng), *not* the top-level record id.
+- Returns hourly `{rainfall_datetime, rainfall_value}` — **but the date range
+  is IGNORED**: every request returns the same trailing ~36-hour window
+  (39 rows). Requests for 2020/2024 return identical data to today.
+- Consequence: **no rainfall history via this API**. To build training data,
+  persist `rain_24h` from now on and backfill history from satellite QPE or an
+  HII data request (§4).
+
+#### Probed and NOT available on api-v3 (all HTTP 404)
+
+`dam_daily`, `dam`, `dam_json`, `big_dam`, `mainstream_dam`, `weather`,
+`rain_graph`, `rainfall_graph`, `rain24hr_graph`. Dam data is v2-only (§2.2).
+
+### 2.2 `twa-api-public.thaiwater.net` — auth-gated (x-api-key / session) 🔒
+
+The layer our existing `src/thaiwater.py` client uses
+(`GET /v2/waterlevel` with `x-api-key: $THAIWATER_API_KEY`; wired to
+`GET /sensors/thaiwater` in the web API, display-only, never persisted).
+Without a key: HTTP 401/500. No public key-registration page was found —
+obtain a sanctioned key from HII (<https://hii.or.th>).
+
+Full endpoint catalog *(catalog — recovered from frontend JS, not exercised)*:
+
+- **Water level / discharge**: `/v2/waterlevel`, `/v2/waterlevel/list`,
+  `/v2/waterlevel/{id}/detail`, `/v2/waterlevel/canal`,
+  `/v2/waterlevel/sea-waterlevel`, `/v2/waterlevel-discharge`,
+  `/v2/waterlevel-discharge/list`, `/v2/waterlevel-discharge/{id}/detail`,
+  `/v2/waterlevel-discharge/{id}/forecast-table`,
+  `/v2/waterlevel-discharge/forecast`, `/v2/waterlevel-discharge/forecast/list`,
+  `/v2/watergate`, `/v2/waterload-tide`
+- **Dams** (incl. Bhumibol): `/v2/large-dam/daily-geo-json`,
+  `/v2/large-dam/daily/list`, `/v2/large-dam/hourly/list`,
+  `/v2/large-dam/daily/{id}/detail`, `/v2/large-dam/hourly/{id}/detail`,
+  `/v2/medium-dam/daily-geo-json`, `/v2/medium-dam/daily/list`,
+  `/v2/medium-dam/{id}/detail`, `/v2/summary/summary4dam`,
+  `/v2/summary/dam-summary`, `/v2/summary/dam-crisis`
+- **Rainfall**: `/v2/rainfall/{type}`, `/v2/rainfall/{type}/list`,
+  `/v2/district-rain/actual-measure`, `/v2/district-rain/forecast`,
+  `/v2/district-rain/accumulate`, `/v2/summary/rainfall24h-ranking-province`,
+  `/v2/summary/rainfall-24hr-forecast`, `/v2/summary/rainfall-forecast`,
+  `/v2/summary/warning-rainfall-24h`, `/v2/summary/warning-rainfall-48h`
+- **Weather / hazards**: `/v2/weather`, `/v2/storm`, `/v2/wave`, `/v2/pm25`,
+  `/v2/pm10`, `/v2/flood/flash-flood`, `/v2/flood/flash-flood-alert`,
+  `/v2/drought/alert`, `/v2/drought/risk-area/list`,
+  `/v2/summary/weather-summary`, `/v2/summary/temperature-forecast`,
+  `/v2/summary-area/rainfall`
+- **Time-series / graph** (base `/data/platform/v1/public/`):
+  `tele_waterlevel/graph`, `flow/graph`, `latest_waterlevel/forecast/graph`,
+  `latest_watertide/forecast/graph`, `dam_pdaily_sum_by_date`,
+  `dam_pdaily_sum_by_region_graph`, `dam_rulecurve/graph`,
+  `medium_dam/graph_year`, `monthly_rainfall/stations`,
+  `monthly_rainfall/anomaly-stations`, `tele_watergate/graph`,
+  `salinity_forecast_cpy/graph`, `sea_waterlevel_forecast/graph`,
+  `latest_weather_area`, `latest_weather_area_daily`
+
+### 2.3 Other HII hosts
+
+| Host | What | Access |
+|---|---|---|
+| `https://standard.thaiwater.net` | **Official water-data standard** — canonical station/basin/province code registries, data-exchange formats, warning-level definitions (Thai) | Open, docs site |
+| `https://api.hii.or.th/tiservice/v1/ws/{token}/isohyet/daily/latest/province/{code}` | Daily isohyet rainfall by province | Token in path |
+| `https://live1.hii.or.th/product/latest/rain/one_map/data/*.tif` | Rainfall anomaly & 1–6-month forecast GeoTIFF rasters | Open |
+| `https://data.hii.or.th` | HII open-data catalog — 36 datasets (rainfall telemetry, water level, weather, climate) | Open browsing |
+| `https://tiwrm.hii.or.th` | Legacy reports | Open |
+
+Historical bulk telemetry: HII documents a request channel at
+**nhcsoc@hii.or.th**.
+
+---
+
+## 3. Dams & reservoirs
+
+> **Geography matters: Bhumibol Dam (Tak) is ~240 km DOWNSTREAM of P.1** and
+> cannot influence Chiang Mai water levels. Do not use it as a P.1 feature.
+
+The predictive upstream reservoir is **Mae Ngat Somboon Chon** (Mae Ngat
+tributary, joins the Ping above Chiang Mai; spilled 110 m³/s during the
+Oct 2024 flood). Mae Kuang Udom Thara is the second upstream reservoir.
+
+| Source | What | Access |
+|---|---|---|
+| `https://app.rid.go.th/reservoir/api/dams` | **INGESTED** — daily snapshot of all ~35 large dams (storage/inflow/outflow MCM, % of usable). `POST` with form field `date=YYYY-MM-DD` (empty = today); GET returns 404 "Unknown method." Archive ≥ 2009; `level_msl` (`DMD_Q`) populated in older years only. Mae Ngat = `DAM_ID 200103` — hit 113% usable capacity, ~19 MCM/day inflow, in Oct 2024. Collected daily by `src/rid_reservoir.py` into `rid_dams` + `rid_reservoir_daily`; backfill via `scripts/backfill_rid_reservoir.py` | Open, no auth |
+| `https://lsim.rid.go.th/ForeCast?reservoirid=22` | Mae Ngat daily status/forecast (RID) | Open, scrape — timed out from outside RID network when probed 2026-08-13 |
+| `https://water.egat.co.th` | EGAT dams (Bhumibol/Sirikit) hourly+daily inflow/outflow/level | Endpoint catalog not public; contact EGAT (0-2436-8186). Only relevant downstream of Bhumibol |
+| ThaiWater `/v2/large-dam/*`, `dam_rulecurve/graph` | All large/medium dams incl. hourly | Requires HII API key (§2.2) |
+
+---
+
+## 4. Rainfall & weather (external)
+
+### Near-real-time (usable in the live inference path)
+
+| Source | Cadence / latency | Access | Notes |
+|---|---|---|---|
+| HII `rain_24h` (§2.1) | Hourly, near-real-time | Open JSON | Primary rain-gauge feed; persist from now on |
+| GSMaP NRT (JAXA) | Hourly, ~4 h latency, 0.1° | Free JAXA registration (FTP); or Google Earth Engine `JAXA/GPM_L3/GSMaP/v8/operational` (no registration) | Gauge-corrected `hourlyPrecipRateGC`; catchment-average rain where gauges are sparse |
+| NASA IMERG **Early Run** | Half-hourly, ~4 h latency, 0.1° | Free Earthdata login | NASA-stack alternative to GSMaP |
+
+### Forecasts (the only way past the ~17 h physical lead-time cap)
+
+| Source | What | Access |
+|---|---|---|
+| **Open-Meteo** (<https://open-meteo.com>) — ✅ **INGESTED** since 2026-08-12 (`src/ml/rain.py`): 5 upper-Ping catchment points feed the hgb-v3 model's rain features (trailing sums + forward-24h forecast, archive 2021+); the leader worker also persists hourly rows to the `openmeteo_rain` table | Hourly precip forecast ≤16 days, any lat/lon; **Historical Forecast API archive from 2021** (train on forecast-as-seen, leakage-free); Previous Runs API (fixed 1–7-day leads from Jan 2024); ERA5 back to 1940 | Free, no key, 10k calls/day, non-commercial w/ attribution |
+| TMD NWP API (`https://data.tmd.go.th/nwpapi/v1/forecast/location/...`) | WRF 4.2 daily/hourly forecasts by place, processed ~06:00 daily | Free Bearer-token registration (`/nwpapi/doc/main/`) |
+| GFS / ECMWF IFS open data | 0.25° global, 4×/day | Free (NOMADS / AWS / data.ecmwf.int); Open-Meteo already wraps both |
+
+### Training-only (too slow for live)
+
+| Source | Cadence | Latency |
+|---|---|---|
+| CHIRPS (`data.chc.ucsb.edu/products/CHIRPS-2.0/`) | Daily, 0.05° | ~2 days prelim / 3+ weeks final |
+| IMERG Late / Final | Half-hourly | ~14 h / ~3.5 months |
+| TMD observation API (`data.tmd.go.th/api/index1.php`) | 3-hourly / daily station obs, XML | Free uid+key registration |
+
+---
+
+## 5. Historical / open-data portals
+
+| Portal | Content |
+|---|---|
+| `https://data.hii.or.th` | 36 HII datasets (rainfall telemetry the most viewed) |
+| `https://data.go.th/dataset?organization=rid` | 4 RID datasets (API + ZIP) |
+| `https://gdcatalog.go.th` | Nationwide daily rainfall-station catalogs |
+| `https://hydro-1.net` | RID Upper-Northern Hydrology Center — hourly/daily tables, hydrology yearbooks (rating curves) for P-series stations; scrape/download |
+| `https://water.rid.go.th/flood/flood/daily.pdf` | RID daily flood bulletin (PDF only) |
+
+---
+
+## 6. Integration status & recommended order
+
+| Source | Status | Action |
+|---|---|---|
+| RID hourly gauges | ✅ Ingested (hourly → PostgreSQL) | — |
+| ThaiWater `/v2/waterlevel` | 🟡 Display-only (`src/thaiwater.py`, needs `THAIWATER_API_KEY`, never persisted) | Optionally persist |
+| HII `rain_24h` | ✅ Ingested hourly via `src/hii_collector.py` → `hii_rain_stations` + `hii_rainfall` (Ping-filtered; ~300 stations) | — |
+| HII `waterlevel_load` | ✅ Ingested hourly via `src/hii_collector.py` → `hii_wl_stations` + `hii_waterlevel` (125 Ping stations, m MSL; `rid_code` column maps mirrors like `ridhydro_P.1` → `P.1`, `offset_msl` converts MSL → gauge datum) | — |
+| HII `waterlevel_graph` | ✅ Backfill via `scripts/backfill_hii_waterlevel.py` → `hii_waterlevel` (hourly MSL + discharge, archive ≥2019; full-year windows per request; upserts never overwrite live-snapshot columns) | Run once on the box: `python scripts/backfill_hii_waterlevel.py` (defaults: 2019-01-01 → today, RID-mirror + key stations; `--stations P.1,P.67`, `--all` for every Ping station) |
+| Mae Ngat reservoir (app.rid.go.th) | ✅ Ingested (2026-08-13) | Daily storage/inflow/outflow for all large dams → `rid_reservoir_daily`; candidate model features for next retrain |
+| Satellite QPE (GSMaP/IMERG) | ❌ | Backfill training rainfall (GEE) |
+| Open-Meteo forecasts | ❌ | Add forecast features (live + 2021 archive for training) |
+| HII API key (dams, forecasts) | ❌ | Contact HII for sanctioned access |
+
+**Collector configuration** (`src/hii_collector.py`): runs automatically every
+scraping cycle (hourly cadence, even while the RID scraper is in 1-minute retry
+mode) from both `--web-api` and continuous-monitoring modes; one-shot via
+`python -m src.main --collect-hii`. Env vars: `ENABLE_HII_COLLECTION`
+(default `true`), `HII_BASIN_CODE` (default `6` = Ping). Requires a SQL
+`DB_TYPE` (sqlite/postgresql/mysql); tables are created automatically.
+
+**Caveats**: `api-v3` is an undocumented backend — no SLA, no ToS, may change
+without notice. Poll hourly at most, cache aggressively, and pursue official
+HII access for anything production-critical.

docs/FLOOD_FORECASTING.md

@@ -0,0 +1,760 @@
+# Flood forecasting
+
+Short-range flood-risk forecasts for the Ping River gauge network, trained on the
+monitor's own PostgreSQL history. This document covers what the system predicts,
+what it is built from, how well it actually performs, how to run it on the server,
+and when to retrain.
+
+Code lives in `src/ml/` (`data.py`, `features.py`, `train.py`, `predict.py`), the
+training entry point is `scripts/train_flood_model.py`, tests are in
+`tests/test_flood_forecast.py`, and trained artifacts land in `models/`.
+
+## 1. Overview
+
+For every station the system answers three questions at three lead times (6, 12
+and 24 hours):
+
+- **`p_warning`** — probability the water level reaches or exceeds the warning
+  threshold (3.0 m) at any point within the horizon.
+- **`p_danger`** — same for the danger threshold (4.5 m).
+- **`predicted_max_level`** — the expected peak level within the horizon, in
+  metres on the station's own datum.
+
+The window is open-ended forward: `exceed_warn_6` at 09:00 asks whether the level
+touches 3.0 m anywhere in (09:00, 15:00], not what it will be at 15:00 — the
+question an operator actually has.
+
+Fifteen of the sixteen stations have trained models. P.4A (Ban Mae Taeng) is
+excluded by `features.NOT_TRAINABLE` (17.2% hourly fill, effectively dead
+2019–2024: 289 rows in 2019, 769 in 2024) and is served by the persistence
+heuristic instead. It still feeds downstream stations as an *input*, where
+HistGradientBoosting's native NaN handling copes with the gaps.
+
+Every forecast row carries `source` (`model` or `heuristic`), `model_version`
+and `trained_at`, so a stale or degraded forecast is visible in the payload
+rather than silently indistinguishable from a good one.
+
+## 2. Data
+
+**Source of record.** PostgreSQL table `water_measurements` joined to `stations`.
+As verified on 2026-08-10 (`inventory.json`, `db_cross_check`): **592,240 rows,
+16 stations, 2018-08-01 through 2026-08-10**, zero mismatches against the HTTP
+API, and a `status` column that is uniformly `active` (there are no quality flags
+to filter on — bad readings must be caught by the feature pipeline, not by the
+database).
+
+**Coverage is the dominant data constraint.** Readings are nominally hourly
+(modal interval 1 h, ~95% of gaps), but only about **56% of hours on the complete
+hourly grid have a reading**: P.1, P.67, P.76 and P.84 at 56.1%, P.103 at 55.8%,
+P.21 at 55.4%, P.20 at 53.7%, P.87 at 53.3%, P.5 at 50.7%, P.4A at 17.2%.
+
+The missingness is **systematic, not random**. Measured over the 587 days from
+2025-01-01 in `models/cache/P.1.csv.gz`, the fraction of days with a reading at
+each hour is roughly 0.80 for 01:00–12:00, 0.68–0.69 for 13:00–16:00, 0.46–0.49
+for 17:00–21:00, 0.40–0.42 for 22:00–23:00, and 0.32 at midnight. That is a
+scrape-schedule fingerprint, not hydrology — and it is why hour-of-day is
+deliberately *not* a feature (see section 3).
+
+Long outages would poison training if used naively: P.87 lost 3,961 hours
+(165 days) in 2023, P.20 lost 2,681 hours in 2021, P.77 lost 2,522 hours in early
+2022, P.5 lost 2,429 hours over the 2020–21 turn. `features.TRAIN_START` excludes
+P.5 before 2022-01-01; the rest are handled by the per-row coverage gates.
+
+**How `src/ml/data.py` loads it.** `resolve_db_url()` picks a connection string in
+priority order: an explicit `--db-url` argument, then the `FLOOD_ML_DB_URL`
+environment variable, then `Config.get_database_config()` when `DB_TYPE` is
+`postgresql`, else `None`. `load_measurements()` then tries three tiers:
+
+1. **PostgreSQL** (`_fetch_from_db`) — the primary path. NULL discharge stays
+   NULL, which matters because the models must learn from the real missingness
+   pattern.
+2. **HTTP API** (`_fetch_from_api`, default `http://100.81.167.42:8000`) — a
+   fallback for running off-server. **Caveat:** the public history endpoint
+   backfills missing discharge with a synthetic rating-curve estimate, so this
+   path is not equivalent to the DB path. It is flagged as
+   `discharge_maybe_synthetic: true` in the cache metadata.
+3. **On-disk cache** (`models/cache/{station}.csv.gz` plus `meta.json`) — last
+   resort only. A successful DB or API fetch refreshes the cache; the cache is
+   never treated as a source of fresh data.
+
+`load_latest()` (used by the API) pulls the trailing 336 hours and never writes
+the cache.
+
+## 3. Physics and features
+
+### Upstream routing
+
+The Ping mainstem gives real forecast skill for free: a flood wave takes hours to
+travel downstream, so an upstream gauge reading *now* is information about a
+downstream gauge *later*. `data-scout` measured these travel times by
+cross-correlating water-level anomalies against the basin anchor P.1. The peak
+correlation lags, which are hard-coded in `features.UPSTREAM_LEADS`:
+
+| Station | Lead vs P.1 | Peak anomaly correlation | Distance to P.1 (km) |
+|---|---|---|---|
+| P.20 (Ban Chiang Dao) | 17 h | 0.59 | 84.5 |
+| P.92 (Ban Muang Aut) | 15 h | 0.66 | 63.8 |
+| P.75 (Ban Chai Lat) | 12 h | 0.61 | 45.0 |
+| P.4A (Ban Mae Taeng) | 12 h | 0.73 | 37.9 |
+| P.67 (Ban Tae) | 7 h | 0.74 | 25.3 |
+| P.21 (Ban Rim Tai) | 9 h | 0.56 | 15.0 |
+| P.103 (Ring Bridge 3) | 1 h | 0.88 | 9.3 |
+
+(Distances are cumulative straight-line gauge-to-gauge, from the inventory's
+`spatial_order_north_to_south`, not channel length — the real river is longer.)
+
+The lags are broadly consistent with distance, with one exception worth knowing
+about: P.21 is 10 km closer to P.1 than P.67 yet lags by 9 h rather than 7 h, and
+it has the weakest correlation of the mainstem set (0.56). Whatever the cause,
+the table encodes the measured lag rather than the one distance would predict —
+which is the point of measuring instead of assuming.
+
+Two stations are *downstream* of P.1 (P.5 at −12 h, P.81 at −4 h). For those,
+`UPSTREAM_LEADS` routes P.1 and P.103 forward as their inputs, which is the same
+physics running the other direction.
+
+Six western-tributary stations — P.82, P.84, P.87, P.77, P.85, P.76 — have empty
+`UPSTREAM_LEADS` and are **un-routed**. Their anomaly correlations with P.1 are
+0.22–0.32, low enough that routing them would inject noise rather than signal.
+They are forecast from their own history plus the P.1 basin-state features. This
+is a known gap: those catchments have no upstream gauge of their own in this
+network.
+
+### Feature set (`features.build_features`)
+
+Everything is computed on an hourly grid built by `make_hourly_grid()`, which
+keeps three aligned frames: `observed` (raw, NaN where nothing was recorded),
+`filled` (forward-filled with `FFILL_LIMIT_H = 3`), and `mask` (True where a real
+reading exists). Features read `filled`; labels read `observed` only.
+
+Per target station:
+
+- **Self level**: current level, lags at 1/2/3/6/12/24/48/72 h.
+- **Rate of rise**: level minus its own value 1/3/6/12/24 h ago — a river at
+  2.5 m and falling is a different situation from one at 2.5 m rising 30 cm/h.
+- **Rolling statistics**: 6/24 h means, 6/24/72 h maxima, 24 h minimum.
+- **Discharge**: current, lags at 6/24 h, 6 h rise (read from `observed`, so NULL
+  discharge stays NULL).
+- **Observation health**: `obs_age_h` (hours since the last real reading, capped
+  at `FFILL_LIMIT_H`) and `cov_24h` (fraction of the last 24 h actually observed),
+  so the model can learn to hedge when a gauge is going quiet.
+- **Routed upstream**, per `(upstream, lead)` pair: the upstream level at
+  `lead−3`, `lead`, and `lead+3` hours ago, its 6 h rise at `lead`, and its 24 h
+  rolling max at `lead−3`. The three-point bracket absorbs error in the measured
+  travel time rather than depending on it being exact.
+- **Basin state** (non-P.1 stations only): P.1 level, its 24 h rolling max, and
+  its 24 h rise.
+- **Seasonality**: `doy_sin`, `doy_cos` and an `is_monsoon` flag for June–October.
+
+**Hour-of-day is deliberately excluded.** Given the availability profile in
+section 2, an hour-of-day feature would let the model learn "readings at 03:00
+are more likely to exist" and route that through to the label — an artefact of
+when the scraper runs, with no hydrological content, that would evaporate the
+moment the scrape schedule changed.
+
+### No-leakage guarantees
+
+- Only `shift()`, backward `rolling()` and forward-fill are used — nothing
+  interpolates, and no row can read a value timestamped after itself.
+- `test_no_future_leakage` enforces this empirically: it adds +50 m to every
+  reading after time *t*, rebuilds the features, and asserts the rows at or
+  before *t* are bit-identical.
+- Labels come from `observed`, never `filled`, so a forward-filled value can
+  never become its own target.
+- The split is strictly temporal, and `early_stopping` is disabled in
+  `HGB_PARAMS` specifically because scikit-learn's internal validation split is
+  random and would leak across time.
+
+### Coverage gating
+
+A label is only trusted if enough of its forward window was actually observed.
+`build_labels` requires `MIN_WINDOW_COVERAGE = 0.5` — at least half the horizon's
+hours present — otherwise the label is NaN and the row is dropped from that head's
+training set. The one exception is deliberate: **an observed exceedance always
+produces a positive label regardless of coverage**, because a confirmed 3.5 m
+reading inside a sparse window is not ambiguous. Rows whose own features are
+stale (`obs_age_h` is NaN, i.e. the last real reading is more than 3 h old) are
+dropped entirely in `build_matrix`.
+
+## 4. Models
+
+### Architecture
+
+One `HistGradientBoosting` model per **station × horizon × head**:
+
+| Head | Type | Target |
+|---|---|---|
+| `max_{h}` | `HistGradientBoostingRegressor` (squared error) | *rise*: max observed level in (t, t+h] minus level at t (v2; serving adds the level back) |
+| `warn_{h}` | `HistGradientBoostingClassifier` | level ≥ 3.0 m anywhere in (t, t+h] |
+| `danger_{h}` | `HistGradientBoostingClassifier` | level ≥ 4.5 m anywhere in (t, t+h] |
+
+Nine heads per station, three horizons (6/12/24 h), fifteen trained stations.
+Hyperparameters are fixed (`HGB_PARAMS`: 300 iterations, learning rate 0.06, 31
+leaf nodes, minimum 50 samples per leaf, L2 1.0, `random_state=42`), chosen in an
+earlier sweep and not re-searched per run — training is deterministic and
+repeatable.
+
+HistGradientBoosting was chosen for three concrete reasons: it handles NaN
+natively (essential given ~44% missing hours), it needs no feature scaling, and
+it trains on CPU alone — no GPU anywhere in this pipeline (measured cost in
+section 6).
+
+### Head gating and fallbacks
+
+The system degrades in tiers rather than failing:
+
+1. **Belt-and-braces probability** *(since 2026-08-11 — see the re-examination
+   note in section 7)*: the sigmoid-of-regression probability
+   `p = 1/(1 + exp(−(predicted_max − threshold)/σ))` is always computed (σ =
+   the regressor's test-residual std, floor `MIN_SIGMA = 0.15` m), and when a
+   classifier head exists — trained only if the span had at least
+   `MIN_POSITIVES_FOR_CLASSIFIER = 30` positives — the served probability is
+   `max(classifier, sigmoid)`. The classifier can raise the alarm but never
+   silence it: on the gap-filled data a trained classifier stayed near zero
+   through the 2024 record crossing while the regression tracked it.
+2. **Sigmoid only**, when the classifier head is absent or skipped
+   (recorded in `skipped_heads` with its reason).
+3. **Persistence heuristic** (`predict._heuristic_forecast`), when there is no
+   model file at all, or the station's newest reading is more than
+   `STALE_AFTER_H = 6` hours old. It extrapolates the last 3 h rate of rise
+   forward with a 0.7 damping factor and a fixed σ of 0.3 m. It is not skilful; it
+   exists so the endpoint always returns something structurally valid.
+
+A station is skipped entirely if it has fewer than `MIN_ROWS_TO_TRAIN = 200`
+usable rows; an individual head is skipped below `MIN_ROWS_FOR_HEAD = 50` labeled
+rows. `_safe_fit` converts any fit failure (typically HistGradientBoosting's
+binning step rejecting an all-NaN or constant column) into a recorded skip rather
+than a station-killing exception.
+
+### Training procedure
+
+`train_station` runs two passes. First it evaluates on the strict temporal
+holdout (train ≤ 2024-12-31, test 2025-01-01 → 2026-08-10) to produce the metrics
+and the σ calibration. Then it **refits every head on the entire record** for the
+deployed artifact, so the shipped model has seen the most recent data. Because
+the full record has more labeled rows than the training half, the head-gating
+decisions can differ between the two passes — `skipped_heads` is therefore
+re-derived during the refit so it always describes what is actually in the saved
+bundle, not what the evaluation pass decided.
+
+### Bundle format
+
+`models/flood_{station}.joblib` contains: `station_code`, `model_version`
+(`hgb-v1+<git short SHA>`), `trained_at`, `sklearn_version`, `feature_names`,
+`horizons`, `thresholds`, `heads`, `sigma`, `skipped_heads`, `train_span`, and
+`n_train_rows`.
+
+`feature_names` is the important one. At prediction time `_model_forecast`
+rebuilds the feature row from live data and checks it against the bundle's stored
+list; if any expected column is missing it logs an error and falls back to the
+heuristic rather than feeding scikit-learn silently misaligned columns.
+`test_feature_name_stability` guards the same invariant at build time. Bundles are
+cached in memory keyed by `(path, mtime)`, so dropping in a retrained file
+invalidates the cache without a restart.
+
+## 5. Measured performance
+
+### Holdout metrics (`models/metrics.json`)
+
+Two evaluations exist and they differ sharply — the re-examination note in the
+next section explains why (the hourly grid was gap-filled from ~56% to ~93%
+between them, roughly doubling the test rows and collapsing the warning base
+rates).
+
+**Current model** `hgb-v3` (rise target + Open-Meteo rain features),
+generated 2026-08-12 on the gap-filled DB (~976k rows). Train ≤ 2024-12-31,
+test 2025-01-01 → 2026-08-12. P.1:
+
+| Horizon | Warning PR-AUC | MAE | MAE above 2 m | Test rows | Base rate |
+|---|---|---|---|---|---|
+| 6 h | 0.783 | 4.9 cm | 5.0 cm | 14,034 | 0.12% |
+| 12 h | 0.508 | 7.2 cm | 10.7 cm | 14,028 | 0.16% |
+| 24 h | 0.288 | 8.7 cm | 18.1 cm | 14,034 | 0.25% |
+
+(Progression across the same day's runs — v1 absolute target:
+5.5/8.1/10.5 cm MAE; v2 rise: 5.0/7.2/9.4; v3 rise+rain: 4.9/7.2/8.7 —
+with above-2 m MAE falling 24.0 → 20.0 → 18.1 cm at 24 h. PR-AUC belongs to
+the unchanged classifier heads; serving is belt-and-braces so alerting uses
+the improved regression path regardless.)
+
+Level accuracy improved; standalone classifier discrimination did not survive
+the data change (which is why serving is now `max(classifier, sigmoid)` — see
+"Head gating"). Recall-at-FAR is null at all horizons on this run. Across
+stations the 6 h warning PR-AUC now spans 0.987 (P.77) / 0.982 (P.5) / 0.956
+(P.85) / 0.950 (P.67) down to 0.436 (P.84), and danger heads are now evaluable
+at nine stations — strongest P.5 (0.958/0.883/0.811 at 6/12/24 h) and P.77
+(0.942/0.863/0.786); P.103's danger metrics, previously the highlight, are null
+on this span.
+
+**Historical evaluation** (`hgb-v1+49a3de0`, 2026-08-10, pre-gap-fill DB —
+kept for the record; these numbers described the sparser 56%-filled grid and do
+not reproduce on today's data):
+
+| Horizon | Warning PR-AUC | Recall @1% FAR | MAE | Test rows | Base rate |
+|---|---|---|---|---|---|
+| 6 h | 0.974 | 98.3% | 6.1 cm | 8,536 | 1.36% |
+| 12 h | 0.904 | 93.8% | 9.0 cm | 7,932 | 1.61% |
+| 24 h | 0.900 | 90.1% | 11.3 cm | 8,572 | 1.77% |
+
+The dramatic PR-AUC difference is mostly the base rate: the filled grid adds
+~5,500 quiet test hours per horizon while the number of positive hours barely
+changes, so the same ranking quality scores far lower — and the classifier's
+genuine out-of-distribution weakness (see the backtest sections) does the rest.
+
+### 2026-08-11 re-examination: fuller data changes the backtest story
+
+> **Read this before the two backtest sections below.** On 2026-08-11 the
+> backtests were codified into `scripts/backtest_render.py` (previously they
+> were one-off runs) and re-run after the database grew from 592k to ~976k
+> rows (a `--fill-gaps all` pass repaired most of the missing 44% of the
+> hourly grid). Three things changed:
+>
+> 1. **The 2024 crossing was 8 hours earlier than documented.** The recovered
+>    hours show P.1 crossing 3.70 m at **17:00 on 24 September 2024**, not
+>    01:00 on 25 September — confirmed independently by the HII sensor at
+>    Nawarat Bridge (hii_waterlevel, station 3226: 3.73 m at 17:00). The
+>    originally celebrated "24-hour warning" was therefore ~16 hours measured
+>    against the real river.
+> 2. **Retraining on the fuller data improves level accuracy but degrades the
+>    warning classifiers.** P.1 24 h MAE improved (11.3 → 10.5 cm), but the
+>    warning-head PR-AUC collapsed (0.900 → 0.288 at 24 h): with the filled
+>    grid the classifier trains on many more dry-season rows and now stays
+>    silent through the September 2024 record crossing while the regression
+>    head tracks it. Serving was changed to belt-and-braces —
+>    `max(classifier, sigmoid(regression))` — so alerting can never be worse
+>    than the regression path.
+> 3. **Honest current lead times, from the regenerated charts below:** the
+>    retrained configuration first alerts ~18 h *after* the true 24 Sep 2024
+>    crossing and roughly *at* the 27 Sep 2025 crossing. The earlier, better
+>    numbers came from models trained and evaluated on the sparser data. The
+>    conclusion is not that the old system was better — it is that gauge-only
+>    features fundamentally lack lead time for fast rises, which is exactly
+>    the rainfall-input and rise-target work now queued (see "Honest limits").
+>
+> `scripts/backtest_render.py` regenerates all three charts and fails its
+> acceptance gate while the 2024 lead stays under 12 h — keeping this page
+> honest is now automatic.
+>
+> **2026-08-12 follow-up — hgb-v2 (rise target).** A rolling-origin,
+> event-aware evaluation (`scripts/evaluate_variants.py`, one fold per monsoon
+> 2021-2025) compared the absolute-level target against rise-target variants.
+> The rise target — regression predicts *future max minus current level*, the
+> level is added back at serving — won decisively and is now deployed as
+> `hgb-v2`: the regenerated charts below show the 2024 first alert moving from
+> 18 h late to **6 h early** (11:00 vs the 17:00 crossing), the 2025 alert
+> from at-crossing to **45 h early**, the record-peak underprediction
+> eliminated (the model now slightly overshoots 5.30 m rather than capping
+> ~0.4 m below it), and P.1 MAE improving ~11% at every horizon. Weighted and
+> quantile variants were evaluated and rejected (more false alarms, no
+> calibration gain by Brier score). The ≥12 h acceptance gate still fails at
+> +6 h for 2024 — genuine further lead needs rainfall inputs, not modelling.
+>
+> **2026-08-12 follow-up 2 — hgb-v3 (rain features): the gate passes.**
+> Open-Meteo catchment rainfall (five upper-Ping points, forecast-model
+> archive 2021+, `src/ml/rain.py`) added four features: trailing 6/24/72 h
+> rain sums and `rain_fc24`, the forward-24 h forecast sum — the first input
+> that can act before water reaches any gauge. On the rolling-origin harness
+> (`models/eval_rain.json`) rain roughly halved flood-year Brier scores, cut
+> flood-regime MAE 20–40%, and moved the hard 2024 leads from +6 h to +11 h
+> (P.1) and +10 to +19 h (P.103); the marginal 2025 double-crest event trades
+> its artifact +46 h "lead" for a calibrated +2 h with zero false alarms. The
+> regenerated backtest below now shows a **13-hour warning for the 2024
+> record flood (alert 04:00, crossing 17:00) — the ≥12 h acceptance gate
+> passes for the first time**. P.1 MAE improves again to 4.9/7.2/8.7 cm at
+> 6/12/24 h. Serving fetches live rain hourly and degrades to NaN features
+> (never a crash) if Open-Meteo is unreachable.
+
+### The September 2025 flood, as the deployed configuration saw it
+
+![Observed vs predicted through the September 2025 flood — model trained only
+through 2024](img/backtest-2025-p1.png)
+
+This uses the deployed configuration (train ≤ 2024-12-31) on an event it never
+saw. *(Chart regenerated 2026-08-12 with the hgb-v2 rise target on the
+gap-filled data — see the re-examination note above for the history of these
+numbers.)* The v3 model first alerts at **16:00 on 27 September 2025 — 2 hours before
+the river crosses 3.70 m** at 18:00. This is a shorter lead than v2's 45 h,
+and deliberately so: v2's long "lead" was an alarm that latched through the
+near-miss 3.51 m crest of the 26th; v3's rain-informed probabilities are far
+better calibrated on this marginal event (Brier halved, zero false-alarm
+episodes on the season) and fire when exceedance actually becomes likely. A
+barely-over-threshold crest is intrinsically a short-notice event.
+
+### Headline validation: the October 2024 record flood
+
+The holdout above never sees a true extreme, because the 2024 flood is in the
+training half. So the model was retrained on data **ending 2024-08-31** and asked
+to forecast September–November 2024 cold, with no knowledge of the event that
+followed. This is the closest thing to a real operational test available.
+
+![Observed P.1 level vs the model's 24 h-ahead predicted peak through the
+October 2024 flood, with the warning probability below](img/backtest-2024-p1.png)
+
+The render above shows the whole event hour by hour *(regenerated 2026-08-12
+with the hgb-v3 rise + rain configuration)*. Top: the observed level (blue)
+against the 24 h-ahead predicted peak the model issued at each hour (amber,
+dashed) — the amber line leads the blue one into both flood waves. Bottom: the
+belt-and-braces probability of flooding within 24 h; the **first alert comes
+at 04:00 on 24 September, 13 hours before the true 17:00 crossing**, while
+the river in town still read 2.9 m — the rain features react to upstream
+precipitation before any gauge rises. The rise target removed the
+cannot-exceed-training-max ceiling, so the record 5.30 m peak is tracked
+rather than capped. The same historic model track drives the dashboard's
+"Replay Oct 2024 flood" feature.
+
+![Hour-by-hour detail of the detection window, 22–28 September
+2024](img/backtest-2024-p1-detail.png)
+
+The hour-by-hour detail of the detection window *(regenerated 2026-08-12,
+hgb-v3)* shows the sequence: the river crosses 3.70 m at **17:00 on
+24 September** (the hours recovered by gap-filling; independently confirmed by
+the HII sensor at the same bridge), and the model's probability crosses 0.5 at
+**04:00 — a 13-hour warning** delivered while the river stood at 2.9 m. The
+same alert under the absolute-level target came 18 hours *after* the crossing
+(v1), and 6 hours before it with the rise target alone (v2); catchment
+rainfall closed the rest. This clears the ≥12 h acceptance gate in
+`scripts/backtest_render.py`.
+
+The event bullets below quote the original (pre-gap-fill) evaluation of the
+deployed model and are kept for the historical record — see the re-examination
+note above for why the lead times no longer reproduce:
+
+- **25 September cold start.** P.1's first warning crossing of the episode was
+  alerted **24–26 hours ahead**. This is the genuinely impressive case: the river
+  was in normal state, and the alert came from upstream routing alone.
+- **5 October record peak** (P.1 5.30 m, P.103 9.93 m — the highest levels in the
+  eight-year record). Alerted **48 hours ahead**. Read this one carefully: the
+  river was already in sustained flood by then, so "48 hours" is the
+  `_first_alert_at` lookback window (`lookback_h = 48`) saturating, not a
+  measurement of true lead time. The model was correctly alarmed throughout;
+  the metric simply cannot express how much earlier than 48 h that started.
+- **P.103 danger head** over the same window: PR-AUC 0.98–0.99, recall at 1% FAR
+  87–95%. It called the danger-level crossings, not just the warning ones.
+- **8 November re-flood.** Caught **26–31 hours ahead** by the 12 and 24 h models
+  — a second, independent event in the same test window.
+- **P.103's 1 September "miss"** is a test-boundary artefact: the event begins in
+  the first hours of the test span, before the feature window has enough test-side
+  history to have produced a sustained alert. It is not a model failure, but it is
+  also not evidence of skill.
+
+### Honest limits
+
+**Genuine lead time is capped by gauge-only physics.** The longest upstream travel
+time into P.1 is 17 h (P.20), and the strongest predictors are much closer:
+P.103 at 1 h, P.67 at 7 h, P.21 at 9 h. Once a 24 h forecast reaches past roughly
+17 h, there is no observation that has "already happened" to inform it — the model
+is extrapolating basin state and season — unless it has rainfall. That is no
+longer hypothetical: hgb-v3's Open-Meteo features (see the re-examination
+notes in section 7) took the 2024 record-flood lead from 18 h late (v1
+gauge-only, absolute target) to 13 h early, precisely because catchment rain
+acts before any gauge rises, and `rain_fc24` — a weather *forecast* — acts
+before the rain itself falls. **Remaining honest limits:** marginal
+just-over-threshold crests (2025: +2 h) are intrinsically short-notice; the
+rain series only exists from 2021-03, so older training rows are rain-blind;
+forecast-rain quality bounds what the feature can add; and Mae Ngat/Mae Kuang
+dam releases remain uningested (see `docs/DATA_SOURCES.md`).
+
+**Danger-level skill at P.1 is unproven.** P.1 never crossed 4.5 m in the
+2025-01-01 → 2026-08-10 test span (`base_rate_danger` is 0.0, so every danger
+metric is `null`). The danger head exists and is trained on the full record — the
+river has spent 57 hours above 4.5 m historically, 0.144% of all hours — but no
+out-of-sample number backs it. Treat `p_danger` at P.1 as indicative, not
+validated.
+
+**Thresholds are per-station as of 2026-08-10.** `THRESHOLDS` now carries
+calibrated (warning, danger) pairs for all 16 stations, derived from the DB's
+`discharge_percent` (RID % of channel capacity): warning = median level at
+75–85% capacity, danger = median level at 95–105%. P.1 instead uses the official
+Chiang Mai inundation map (`P1_FLOOD_STAGES`): warning 3.70 m (city flooding
+begins, stage 1) and danger 4.20 m (stage 5). The prior single default of
+(3.0, 4.5) m made P.103 badly over-alert (its bank-full level is ~6.75 m) and
+P.67 under-alert (overflow at ~2.9 m, 1.6 m below the old danger line).
+**A retrain is required after any threshold change** — classifier labels depend
+on them; until then, model rows report the thresholds baked into their bundle.
+P.1 additionally reports `stages`: exceedance probability for each of the seven
+official inundation stages (3.70–4.60 m), computed from the regression head and
+its calibration sigma, so they need no retrain and no per-stage classifiers.
+
+## 6. Deployment
+
+### API
+
+`GET /forecast` (`src/web_api.py`) returns one JSON row per station × horizon with
+the fields listed in section 1. Results are cached in-process for
+`FORECAST_TTL = 900` seconds (15 minutes), which matches the data cadence — the
+underlying readings do not update faster than hourly. Inference runs in a thread
+via `asyncio.to_thread` so it never blocks the event loop.
+
+Failure modes: **503** if the `src.ml` package cannot be imported (missing
+scikit-learn, say), **502** on any other exception.
+
+One behaviour worth knowing, because the code comments suggest otherwise: the
+endpoint's `FileNotFoundError` ("No trained flood models found") and `RuntimeError`
+handlers are unreachable — nothing in `src/ml/` raises either, and
+`predict._forecast_station` checks `bundle_path.exists()` and falls back to the
+heuristic instead. So **before the first training run `/forecast` returns 200 with
+an all-heuristic payload**, not a 503, provided there is recent gauge data; you
+get an empty `200 []` only when there is no recent data at all. Judge deployment
+state by the `source` field, not the status code.
+
+### Dashboard
+
+The "Flood risk outlook" panel (`src/static/dashboard.html`, `loadForecasts()`)
+loads non-blocking after the map renders and **stays hidden unless `/forecast`
+returns a non-empty array** — a non-OK response, an empty array, or a thrown
+fetch all just leave the panel hidden, and the rest of the dashboard is
+unaffected. Per the note above, this means the panel appears with heuristic-only
+content once data is flowing but before any model is trained; the per-chip
+tooltip is what tells you so. Stations are sorted worst-risk first, each showing
+three chips (6/12/24 h) coloured by risk band, with the tooltip carrying the exact
+warning and danger percentages, the predicted peak level, and a "heuristic
+fallback" note when the row did not come from a model. The panel is labelled
+*experimental*.
+
+### Training on the server
+
+The server already has the PostgreSQL connection configured, so no host override
+is needed:
+
+```bash
+cd /path/to/Northern-Thailand-Ping-River-Monitor
+python scripts/train_flood_model.py --stations all
+```
+
+`resolve_db_url()` picks up `Config.get_database_config()` automatically when
+`DB_TYPE=postgresql`. The run writes fifteen `models/flood_{station}.joblib`
+bundles plus `models/metrics.json`.
+
+### Artifacts and dependencies
+
+The fifteen bundles total **101.4 MB** — mean 6.76 MB, from 4.04 MB (P.20) to
+9.35 MB (P.103 and P.87, with P.5 next at 8.99 MB) — plus `metrics.json` at
+0.34 MB and a 2.1 MB `models/cache/`. **These are not in
+git**, and they should stay that way — artifacts are produced on the server, not
+shipped. `.gitignore` excludes `models/*.joblib`, `models/cache/` and
+`models/metrics.json` for exactly this reason.
+
+Two pins matter and are already in `requirements.txt` / `pyproject.toml`:
+`scikit-learn==1.9.0` and `numpy>=1.24,<2` (pandas 2.0.3 wheels are ABI
+incompatible with numpy 2.x). Bundles record `sklearn_version`; unpickling a
+bundle under a different scikit-learn version is not guaranteed to work, so
+retrain after any scikit-learn upgrade rather than assuming the artifacts carry
+over.
+
+### Measured resource use
+
+All figures below were measured on 2026-08-10 on a development workstation —
+**24 physical / 32 logical cores at 2.20 GHz, 32 GiB RAM** (Python 3.11.9,
+scikit-learn 1.9.0, joblib 1.5.3, numpy 1.26.4, pandas 2.0.3) — **not** on the
+production server. They come from two independent benchmark runs on that same
+machine, which is why a couple of figures below are quoted as narrow ranges.
+Treat the CPU times as a floor and the memory figures as representative, since
+RSS barely depends on core count. Training read the `models/cache/` csv.gz files
+(592,240 rows load in 0.5 s); loading the same history from PostgreSQL was not
+measured and will be slower.
+
+**Training** (`train_all`, all 15 stations, evaluation pass plus full refit):
+
+| Measurement | Value |
+|---|---|
+| Full 15-station run, unrestricted threads | **199 s (3.3 min)** |
+| Peak RSS during the full run | **209 MB** |
+| Single station, unrestricted (P.1 / P.103) | 16.5 s / 18.7 s |
+
+HistGradientBoosting threads through OpenMP, and it scales only modestly. Timing
+P.1 alone under `OMP_NUM_THREADS`:
+
+| Threads | 1 | 2 | 4 | unrestricted (32) |
+|---|---|---|---|---|
+| P.1 train time | 36.5 s | 23.5 s | 14.7 s | 16.5 s |
+
+Two things follow. **Four threads is the sweet spot** — 32 threads was marginally
+*slower* than 4, so oversubscription costs you a little. And **even one core is
+enough**: at 36.5 s per station, a single-core box retrains all fifteen in roughly
+9 minutes (extrapolated, not measured end-to-end).
+
+Per station the fit costs **8–17 s**, and P.1 is the worst case at 16.9 s — it is
+the basin anchor, so it carries 64 features against 32 for stations with fewer
+upstream inputs (P.85 9.6 s, P.20 8.2 s). Two things are *not* the cost driver.
+Evaluation isn't: P.1 with `skip_eval=True` took 17.3 s, no faster than the full
+path. Nor is feature engineering — `build_matrix` over P.1's whole 8-year history
+is 256 ms against 8–17 s of fitting. **The fit is the cost.**
+
+One honest caveat about the run that produced the current artifacts. By file
+mtime it wrote all fifteen models between 11:55:29 and 12:01:15 — **5 min 46 s**,
+averaging 25 s/station including joblib serialization, which lines up with the
+measured fits. But `models/cache/meta.json` records the data fetch finishing at
+11:45:49, so end to end that run spanned about 15.5 minutes, and the 9 min 40 s
+gap between fetch and first model could not be reconstructed from the surviving
+artifacts. Do not attribute it to per-station training cost. Either way the
+conclusion holds: **retraining is minutes, not tens of minutes.**
+
+**Inference** (15 bundles, 16 stations × 3 horizons = 48 rows):
+
+| Measurement | Value |
+|---|---|
+| Cold call — every bundle unpickled from disk | **6.7 s** |
+| Warm call — bundles in `_MODEL_CACHE` | **0.72 s** median (0.63–0.84 s) |
+| RSS after imports, before any model | 71 MB |
+| RSS with all 15 bundles resident | **288 MB** |
+
+The 101.4 MB of on-disk pickles expand to roughly **203–211 MB resident** — about
+2× — and they stay there: `_MODEL_CACHE` replaces an entry when the file's mtime
+changes but never drops one to reclaim memory. That is the single largest memory
+cost of the whole feature.
+
+Where the time goes: cold start is 5.30 s, of which 0.46 s is the import and
+4.84 s is unpickling, and 2.6 s of *that* is the first bundle alone paying a
+one-time lazy `sklearn.ensemble` import — the remaining fourteen average 159 ms.
+Of the ~640 ms warm compute, model prediction is ~525 ms, the hourly grid 47 ms,
+and feature building 71 ms across all sixteen stations.
+
+**Live-endpoint measurements** (a second, independent benchmark run against a real
+uvicorn instance of the app, same day, same workstation, RSS summed over the
+process tree):
+
+| Measurement | Value |
+|---|---|
+| `/forecast` cache hit (15-min TTL) | **2.4 ms** median |
+| `/forecast` cache miss, default threads | 15.2 s (≈7 s of that was the HTTP data fallback; a local DB replaces it) |
+| `/forecast` cache miss, `OMP_NUM_THREADS=1` | 10.9 s |
+| API process RSS, idle → models resident | 76 MB → **335 MB** |
+
+The endpoint-level RSS (335 MB) is higher than the models-only figure above
+because the live process also retains the pandas frames from the data pull and
+the HTTP/JSON machinery — use 335 MB as the sizing number.
+
+One threading subtlety cuts the other way in serving: inference is ~135
+single-row predicts, and at one row OpenMP thread dispatch costs more than the
+math — `OMP_NUM_THREADS=1` makes the warm compute 2.6× faster (642 ms → 252 ms).
+Training shows the opposite (2.3× slower single-threaded), so set the variable
+per process, never globally.
+
+**Server sizing, in plain terms:** this is a small workload and almost any server
+runs it. **RAM is the binding constraint, not CPU.** Budget about **1 GB for the
+API process** so the ~335 MB steady state has headroom on top of the rest of the
+app; training peaks at only ~210–315 MB and can share the same box. No GPU
+anywhere. Pin thread counts per process — `OMP_NUM_THREADS=1` in the serving
+unit, `OMP_NUM_THREADS=4` for retraining so it cannot monopolise every core while
+the API is serving. And since a cold call costs seconds against a 2.4 ms cache
+hit, consider warming `/forecast` once at startup rather than letting a user
+absorb it.
+
+## 7. Retraining policy
+
+**Why it matters here specifically.** This is not a generic "models go stale"
+argument:
+
+- **Channel geometry changes after every major flood.** Scour, deposition and
+  bank failure shift the level-to-discharge relationship at a gauge, and RID
+  revises rating curves after big events. A model trained on the pre-2024 channel
+  is predicting levels for a cross-section that no longer exists.
+- **Extreme events extend the label range.** The highest P.103 reading before the
+  2024 season was 7.54 m (October 2022); the 2024 event pushed it to 8.27 m on
+  26 September and 9.93 m on 5 October. Gradient boosting cannot extrapolate past
+  its training range — predictions saturate at the largest value it has seen — so
+  every new record is what makes the next one predictable.
+- **Station outages change feature availability.** P.87's 165-day gap in 2023 and
+  P.4A's five dead years mean the set of populated features drifts over time.
+  Retraining lets head gating and NaN handling re-adapt to the current sensors.
+
+**Recommended schedule:**
+
+| When | Why |
+|---|---|
+| **Every year, May–June (pre-monsoon)** | The minimum. Ensures the model entering the flood season has seen last season in full. |
+| **Monthly, July–November** | Cheap insurance during the season — a full retrain costs minutes, not hours (section 6), so `nice` it and forget it. |
+| **After any major flood event** | Non-negotiable. Channel geometry and rating curves have changed, and the new extreme extends the trainable label range. |
+
+Staleness is auditable without guesswork: `model_version` embeds the git short SHA
+of the code that trained the bundle (`hgb-v1+49a3de0`), and `trained_at` is a
+timestamp in every bundle. Both are echoed in every `/forecast` row, so you can
+tell from the API response alone which code produced a forecast and how old the
+model is.
+
+## 8. Operations runbook
+
+All commands assume the project virtualenv is active (`.venv` locally).
+
+**Train (all stations, with evaluation):**
+
+```bash
+python scripts/train_flood_model.py --stations all
+```
+
+**Train a subset, refit-only (skips the holdout evaluation — much faster, but
+produces no metrics and leaves σ at the `MIN_SIGMA` floor):**
+
+```bash
+python scripts/train_flood_model.py --stations P.1,P.103 --skip-eval
+```
+
+**Train from a workstation against the server's database:**
+
+```bash
+export FLOOD_ML_DB_URL='postgresql://user:pass@host:5432/dbname'
+python scripts/train_flood_model.py --stations all
+```
+
+Do not commit that URL anywhere. If the DB is unreachable the loader silently
+falls back to the HTTP API, whose discharge values are partly synthetic — check
+the log line `PostgreSQL fetch failed, falling back to HTTP API` before trusting a
+run.
+
+**Verify before promoting.** Training writes `models/metrics.json` alongside the
+bundles. Check it before treating a run as good:
+
+```bash
+python -c "import json; m=json.load(open('models/metrics.json')); \
+print(m['model_version'], m['split']); \
+print({s: v['status'] for s, v in m['stations'].items()}); \
+print({h: (d.get('pr_auc_warn'), d.get('mae')) for h, d in m['stations']['P.1']['per_horizon'].items()})"
+```
+
+Expect fifteen `trained` and one `heuristic` (P.4A). A station that reports
+`failed` names its reason in the same payload. Compare against the *previous
+run's* `metrics.json`, not an absolute bar: after the 2026-08-11 gap-fill the
+expected baseline is P.1 6 h warning PR-AUC ≈ 0.78 and MAE ≈ 5.5 cm (the
+historical ~0.97 figure belonged to the sparse pre-fill grid — see section 5).
+A *material drop from the previous run* usually means a data problem (a gauge
+that went quiet, or a bad backfill) rather than a modelling one.
+
+**Run the tests** (synthetic data only, no database or network required):
+
+```bash
+python -m pytest tests/test_flood_forecast.py -v
+```
+
+Seven tests covering leakage, label alignment, the coverage gate, forward-fill and
+staleness, a train/predict round trip, the heuristic fallback, and feature-name
+stability. The whole suite runs in about 8 seconds, so there is no excuse for
+skipping it before a deploy.
+
+**Understanding graceful degradation.** Three things can make a forecast row
+non-model-backed, and all of them are visible in the payload:
+
+- `source: "heuristic"`, `model_version: "heuristic-v1"` — either no bundle exists
+  for that station (P.4A always, every station before the first training run), or
+  the station's newest reading is more than 6 hours old.
+- A single horizon coming back heuristic while others are model-backed — that
+  horizon's head is in the bundle's `skipped_heads`, almost always because the
+  station had fewer than 30 positive examples for that threshold.
+- A whole station flipping to heuristic after a code change — the feature-name
+  check in `_model_forecast` caught a mismatch between the live feature builder
+  and the stored `feature_names`. The fix is to retrain; the log line names the
+  missing columns.
+
+A live example from the 2026-08-10 cache: of 48 forecast rows, 42 came from
+models and 6 were heuristic — three for P.4A, which has no bundle by design, and
+three for P.92, whose newest reading was 02:00 while the basin's newest was 09:00.
+That 7 hours of staleness crossed `STALE_AFTER_H = 6`, so P.92 correctly dropped
+to persistence. Both fallback triggers, working as intended, in one ordinary call.
+
+Inspect a bundle's skipped heads directly (`joblib.load` unpickles, so only ever
+point it at a bundle this pipeline's own `train.py` wrote — never a file from
+elsewhere):
+
+```bash
+python -c "import joblib; b=joblib.load('models/flood_P.1.joblib'); \
+print(b['model_version'], b['trained_at'], b['n_train_rows']); print(b['skipped_heads'])"
+```

docs/GRAFANA_MATRIX_ALERTING.md

@@ -0,0 +1,168 @@
+# Grafana Matrix Alerting Setup
+
+## Overview
+Configure Grafana to send water level alerts directly to Matrix channels when thresholds are exceeded.
+
+## Prerequisites
+- Grafana instance with your PostgreSQL data source
+- Matrix account and access token
+- Matrix room for alerts
+
+## Step 1: Configure Matrix Contact Point
+
+1. **In Grafana, go to Alerting → Contact Points**
+2. **Add new contact point:**
+   ```
+   Name: matrix-water-alerts
+   Integration: Webhook
+   URL: https://matrix.org/_matrix/client/v3/rooms/!ROOM_ID:matrix.org/send/m.room.message
+   HTTP Method: POST
+   ```
+
+3. **Add Headers:**
+   ```
+   Authorization: Bearer YOUR_MATRIX_ACCESS_TOKEN
+   Content-Type: application/json
+   ```
+
+4. **Message Template:**
+   ```json
+   {
+     "msgtype": "m.text",
+     "body": "🌊 WATER ALERT: {{ .CommonLabels.alertname }}\n\nStation: {{ .CommonLabels.station_code }}\nLevel: {{ .CommonAnnotations.water_level }}m\nStatus: {{ .CommonLabels.severity }}\n\nTime: {{ .CommonAnnotations.time }}"
+   }
+   ```
+
+## Step 2: Create Alert Rules
+
+### High Water Level Alert
+```yaml
+Rule Name: high-water-level
+Query: water_level > 6.0
+Condition: IS ABOVE 6.0 FOR 5m
+Labels:
+  - severity: critical
+  - station_code: {{ .station_code }}
+Annotations:
+  - water_level: {{ .water_level }}
+  - summary: "Critical water level at {{ .station_code }}"
+```
+
+### Low Water Level Alert
+```yaml
+Rule Name: low-water-level
+Query: water_level < 1.0
+Condition: IS BELOW 1.0 FOR 10m
+Labels:
+  - severity: warning
+  - station_code: {{ .station_code }}
+```
+
+### Data Gap Alert
+```yaml
+Rule Name: data-gap
+Query: increase(measurements_total[1h]) == 0
+Condition: IS EQUAL TO 0 FOR 30m
+Labels:
+  - severity: warning
+  - issue: data-gap
+```
+
+## Step 3: Matrix Setup
+
+### Get Matrix Access Token
+```bash
+curl -X POST https://matrix.org/_matrix/client/v3/login \
+  -H "Content-Type: application/json" \
+  -d '{
+    "type": "m.login.password",
+    "user": "your_username",
+    "password": "your_password"
+  }'
+```
+
+### Create Alert Room
+```bash
+curl -X POST "https://matrix.org/_matrix/client/v3/createRoom" \
+  -H "Authorization: Bearer YOUR_ACCESS_TOKEN" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "name": "Water Level Alerts - Northern Thailand",
+    "topic": "Automated alerts for Ping River water monitoring",
+    "preset": "trusted_private_chat"
+  }'
+```
+
+## Example Alert Queries
+
+### Critical Water Levels
+```promql
+# High water alert
+water_level{station_code=~"P.1|P.4A|P.20"} > 6.0
+
+# Dangerous discharge
+discharge{station_code=~".*"} > 500
+
+# Rapid level change
+increase(water_level[15m]) > 0.5
+```
+
+### System Health
+```promql
+# No data received
+up{job="water-monitor"} == 0
+
+# Old data
+(time() - timestamp) > 7200
+```
+
+## Alert Notification Format
+
+Your Matrix messages will look like:
+```
+🌊 WATER ALERT: High Water Level
+
+Station: P.1 (Chiang Mai)
+Level: 6.2m (CRITICAL)
+Discharge: 450 cms
+Status: DANGER
+
+Time: 2025-09-26 14:30:00
+Trend: Rising (+0.3m in 30min)
+
+📍 Location: 18.7883°N, 98.9853°E
+```
+
+## Advanced Features
+
+### Escalation Rules
+```yaml
+# Send to different rooms based on severity
+- if: severity == "critical"
+  receiver: matrix-emergency
+- if: severity == "warning"
+  receiver: matrix-alerts
+- if: time_of_day() outside "08:00-20:00"
+  receiver: matrix-night-duty
+```
+
+### Rate Limiting
+```yaml
+group_wait: 5m
+group_interval: 10m
+repeat_interval: 30m
+```
+
+## Testing Alerts
+
+1. **Test Contact Point** - Use Grafana's test button
+2. **Simulate Alert** - Manually trigger with test data
+3. **Verify Matrix** - Check message formatting and delivery
+
+## Troubleshooting
+
+### Common Issues
+- **403 Forbidden**: Check Matrix access token
+- **Room not found**: Verify room ID format
+- **No alerts**: Check query syntax and thresholds
+- **Spam**: Configure proper grouping and intervals

docs/GRAFANA_MATRIX_SETUP.md

@@ -0,0 +1,351 @@
+# Complete Grafana Matrix Alerting Setup Guide
+
+## Overview
+Configure Grafana to send water level alerts directly to Matrix channels when thresholds are exceeded.
+
+## Prerequisites
+- Grafana instance running (v8.0+)
+- PostgreSQL data source configured in Grafana
+- Matrix account
+- Matrix room for alerts
+
+## Step 1: Get Matrix Access Token
+
+### Method 1: Using curl
+```bash
+curl -X POST https://matrix.org/_matrix/client/v3/login \
+  -H "Content-Type: application/json" \
+  -d '{
+    "type": "m.login.password",
+    "user": "your_username",
+    "password": "your_password"
+  }'
+```
+
+### Method 2: Using Element Web Client
+1. Open Element in browser: https://app.element.io
+2. Login to your account
+3. Go to Settings → Help & About → Advanced
+4. Copy your Access Token
+
+### Method 3: Using Matrix Admin Panel
+- If you have admin access to your homeserver, generate token via admin API
+
+## Step 2: Create Alert Room
+
+```bash
+curl -X POST "https://matrix.org/_matrix/client/v3/createRoom" \
+  -H "Authorization: Bearer YOUR_ACCESS_TOKEN" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "name": "Water Level Alerts - Northern Thailand",
+    "topic": "Automated alerts for Ping River water monitoring",
+    "preset": "private_chat"
+  }'
+```
+
+Save the `room_id` from the response (format: !roomid:homeserver.com)
+
+## Step 3: Configure Grafana Contact Point
+
+### Navigate to Alerting
+1. In Grafana, go to **Alerting → Contact Points**
+2. Click **Add contact point**
+
+### Contact Point Settings
+```
+Name: matrix-water-alerts
+Integration: Webhook
+URL: https://matrix.org/_matrix/client/v3/rooms/!YOUR_ROOM_ID:matrix.org/send/m.room.message/{{ .GroupLabels.alertname }}_{{ .GroupLabels.severity }}_{{ now.Unix }}
+HTTP Method: POST
+```
+
+### Headers
+```
+Authorization: Bearer YOUR_MATRIX_ACCESS_TOKEN
+Content-Type: application/json
+```
+
+### Message Template (JSON Body)
+```json
+{
+  "msgtype": "m.text",
+  "body": "🌊 **PING RIVER WATER ALERT**\n\n**Alert:** {{ .GroupLabels.alertname }}\n**Severity:** {{ .GroupLabels.severity | toUpper }}\n**Station:** {{ .GroupLabels.station_code }} ({{ .GroupLabels.station_name }})\n\n{{ range .Alerts }}**Status:** {{ .Status | toUpper }}\n**Water Level:** {{ .Annotations.water_level }}m\n**Threshold:** {{ .Annotations.threshold }}m\n**Time:** {{ .StartsAt.Format \"2006-01-02 15:04:05\" }}\n{{ if .Annotations.discharge }}**Discharge:** {{ .Annotations.discharge }} cms\n{{ end }}{{ if .Annotations.message }}**Details:** {{ .Annotations.message }}\n{{ end }}{{ end }}\n📈 **Dashboard:** {{ .ExternalURL }}\n📍 **Location:** Northern Thailand Ping River"
+}
+```
+
+## Step 4: Create Alert Rules
+
+### High Water Level Alert
+```yaml
+# Rule Configuration
+Rule Name: high-water-level
+Evaluation Group: water-level-alerts
+Folder: Water Monitoring
+
+# Query A
+SELECT
+  station_code,
+  station_name_th as station_name,
+  water_level,
+  discharge,
+  timestamp
+FROM water_measurements
+WHERE
+  timestamp > now() - interval '5 minutes'
+  AND water_level > 6.0
+
+# Condition
+IS ABOVE 6.0 FOR 5 minutes
+
+# Labels
+severity: critical
+alertname: High Water Level
+station_code: {{ $labels.station_code }}
+station_name: {{ $labels.station_name }}
+
+# Annotations
+water_level: {{ $values.water_level }}
+threshold: 6.0
+discharge: {{ $values.discharge }}
+summary: Critical water level detected at {{ $labels.station_code }}
+```
+
+### Emergency Water Level Alert
+```yaml
+Rule Name: emergency-water-level
+Query: water_level > 8.0
+Condition: IS ABOVE 8.0 FOR 2 minutes
+Labels:
+  severity: emergency
+  alertname: Emergency Water Level
+Annotations:
+  threshold: 8.0
+  message: IMMEDIATE ACTION REQUIRED - Flood risk imminent
+```
+
+### Low Water Level Alert
+```yaml
+Rule Name: low-water-level
+Query: water_level < 1.0
+Condition: IS BELOW 1.0 FOR 15 minutes
+Labels:
+  severity: warning
+  alertname: Low Water Level
+Annotations:
+  threshold: 1.0
+  message: Drought conditions detected
+```
+
+### Data Gap Alert
+```yaml
+Rule Name: data-gap
+Query:
+  SELECT
+    station_code,
+    MAX(timestamp) as last_seen
+  FROM water_measurements
+  GROUP BY station_code
+  HAVING MAX(timestamp) < now() - interval '2 hours'
+
+Condition: HAS NO DATA FOR 30 minutes
+Labels:
+  severity: warning
+  alertname: Data Gap
+  issue: missing-data
+```
+
+### Rapid Level Change Alert
+```yaml
+Rule Name: rapid-level-change
+Query:
+  SELECT
+    station_code,
+    water_level,
+    LAG(water_level, 1) OVER (PARTITION BY station_code ORDER BY timestamp) as prev_level
+  FROM water_measurements
+  WHERE timestamp > now() - interval '15 minutes'
+  HAVING ABS(water_level - prev_level) > 0.5
+
+Condition: CHANGE > 0.5m FOR 1 minute
+Labels:
+  severity: warning
+  alertname: Rapid Water Level Change
+```
+
+## Step 5: Configure Notification Policy
+
+### Create Notification Policy
+```yaml
+# Policy Tree
+- receiver: matrix-water-alerts
+  match:
+    severity: emergency|critical
+  group_wait: 10s
+  group_interval: 5m
+  repeat_interval: 30m
+
+- receiver: matrix-water-alerts
+  match:
+    severity: warning
+  group_wait: 30s
+  group_interval: 10m
+  repeat_interval: 2h
+```
+
+### Grouping Rules
+```yaml
+group_by: [alertname, station_code]
+group_wait: 10s
+group_interval: 5m
+repeat_interval: 1h
+```
+
+## Step 6: Station-Specific Thresholds
+
+Create separate rules for each station with appropriate thresholds:
+
+```sql
+-- P.1 (Chiang Mai) - Urban area, higher thresholds
+SELECT * FROM water_measurements
+WHERE station_code = 'P.1' AND water_level > 6.5
+
+-- P.4A (Mae Ping) - Agricultural area
+SELECT * FROM water_measurements
+WHERE station_code = 'P.4A' AND water_level > 5.0
+
+-- P.20 (Downstream) - Lower threshold
+SELECT * FROM water_measurements
+WHERE station_code = 'P.20' AND water_level > 4.0
+```
+
+## Step 7: Advanced Features
+
+### Time-Based Routing
+```yaml
+# Different receivers for day/night
+time_intervals:
+  - name: working_hours
+    time_intervals:
+      - times:
+        - start_time: '08:00'
+          end_time: '20:00'
+        weekdays: ['monday:friday']
+
+routes:
+  - receiver: matrix-alerts-day
+    match:
+      severity: warning
+    active_time_intervals: [working_hours]
+
+  - receiver: matrix-alerts-night
+    match:
+      severity: warning
+    active_time_intervals: ['!working_hours']
+```
+
+### Multi-Channel Alerts
+```yaml
+# Send critical alerts to multiple rooms
+- receiver: matrix-emergency
+  webhook_configs:
+    - url: https://matrix.org/_matrix/client/v3/rooms/!emergency:matrix.org/send/m.room.message
+      http_config:
+        authorization:
+          credentials: "Bearer EMERGENCY_TOKEN"
+    - url: https://matrix.org/_matrix/client/v3/rooms/!general:matrix.org/send/m.room.message
+      http_config:
+        authorization:
+          credentials: "Bearer GENERAL_TOKEN"
+```
+
+## Step 8: Testing
+
+### Test Contact Point
+1. Go to Contact Points in Grafana
+2. Select your Matrix contact point
+3. Click "Test" button
+4. Check Matrix room for test message
+
+### Test Alert Rules
+1. Temporarily lower thresholds
+2. Wait for condition to trigger
+3. Verify alert appears in Grafana
+4. Verify Matrix message received
+5. Reset thresholds
+
+### Manual Alert Trigger
+```bash
+# Simulate high water level in database
+INSERT INTO water_measurements (station_code, water_level, timestamp)
+VALUES ('P.1', 7.5, NOW());
+```
+
+## Troubleshooting
+
+### Common Issues
+
+#### 403 Forbidden
+- **Cause**: Invalid Matrix access token
+- **Fix**: Regenerate token or check permissions
+
+#### Room Not Found
+- **Cause**: Incorrect room ID format
+- **Fix**: Ensure room ID starts with ! and includes homeserver
+
+#### No Alerts Firing
+- **Cause**: Query returns no results
+- **Fix**: Test queries in Grafana Explore, check data availability
+
+#### Alert Spam
+- **Cause**: No grouping configured
+- **Fix**: Configure proper group_by and intervals
+
+#### Messages Not Formatted
+- **Cause**: Template syntax errors
+- **Fix**: Validate JSON template, check Grafana template docs
+
+### Debug Steps
+1. Check Grafana alert rule status
+2. Verify contact point test succeeds
+3. Check Grafana logs: `/var/log/grafana/grafana.log`
+4. Test Matrix API directly with curl
+5. Verify database connectivity and query results
+
+## Environment Variables
+
+Add to your `.env`:
+```bash
+MATRIX_HOMESERVER=https://matrix.org
+MATRIX_ACCESS_TOKEN=your_access_token_here
+MATRIX_ROOM_ID=!your_room_id:matrix.org
+GRAFANA_URL=http://your-grafana-host:3000
+```
+
+## Example Alert Message
+Your Matrix messages will appear as:
+```
+🌊 **PING RIVER WATER ALERT**
+
+**Alert:** High Water Level
+**Severity:** CRITICAL
+**Station:** P.1 (สถานีเชียงใหม่)
+
+**Status:** FIRING
+**Water Level:** 6.75m
+**Threshold:** 6.0m
+**Time:** 2025-09-26 14:30:00
+**Discharge:** 450.2 cms
+
+📈 **Dashboard:** http://grafana:3000
+📍 **Location:** Northern Thailand Ping River
+```
+
+## Security Notes
+- Store Matrix tokens securely (environment variables)
+- Use room-specific tokens when possible
+- Enable rate limiting to prevent spam
+- Consider using dedicated alerting user account
+- Regularly rotate access tokens
+
+This setup provides comprehensive water level monitoring with immediate Matrix notifications when thresholds are exceeded.

docs/MATRIX_QUICK_START.md

@@ -0,0 +1,85 @@
+# Quick Matrix Alerting Setup
+
+## Step 1: Get Matrix Account
+1. Go to https://app.element.io or install Element app
+2. Create account or login with existing Matrix account
+
+## Step 2: Get Access Token
+
+### Method 1: Element Web (Recommended)
+1. Open Element in browser: https://app.element.io
+2. Login to your account
+3. Click Settings (gear icon) → Help & About → Advanced
+4. Copy your "Access Token" (starts with `syt_...` or similar)
+
+### Method 2: Command Line
+```bash
+curl -X POST https://matrix.org/_matrix/client/v3/login \
+  -H "Content-Type: application/json" \
+  -d '{
+    "type": "m.login.password",
+    "user": "your_username",
+    "password": "your_password"
+  }'
+```
+
+## Step 3: Create Alert Room
+1. In Element, click "+" to create new room
+2. Name: "Water Level Alerts"
+3. Set to Private
+4. Copy the room ID from room settings (format: `!roomid:matrix.org`)
+
+## Step 4: Configure .env File
+Add these to your `.env` file:
+```bash
+# Matrix Alerting Configuration
+MATRIX_HOMESERVER=https://matrix.org
+MATRIX_ACCESS_TOKEN=syt_your_access_token_here
+MATRIX_ROOM_ID=!your_room_id:matrix.org
+
+# Grafana Integration (optional)
+GRAFANA_URL=http://localhost:3000
+```
+
+## Step 5: Test Configuration
+```bash
+# Test Matrix connection
+uv run python run.py --alert-test
+
+# Check system status (shows Matrix config)
+uv run python run.py --status
+
+# Run alert check
+uv run python run.py --alert-check
+```
+
+## Example Alert Message
+When thresholds are exceeded, you'll receive messages like:
+```
+🌊 **WATER LEVEL ALERT**
+
+**Station:** P.1 (สถานีเชียงใหม่)
+**Alert Type:** Critical Water Level
+**Severity:** CRITICAL
+
+**Current Level:** 6.75m
+**Threshold:** 6.0m
+**Difference:** +0.75m
+**Discharge:** 450.2 cms
+
+**Time:** 2025-09-26 14:30:00
+
+📈 View dashboard: http://localhost:3000
+```
+
+## Cron Job Setup (Optional)
+Add to crontab for automatic alerting:
+```bash
+# Check water levels every 15 minutes
+*/15 * * * * cd /path/to/monitor && uv run python run.py --alert-check >> alerts.log 2>&1
+```
+
+## Troubleshooting
+- **403 Error**: Check Matrix access token is valid
+- **Room Not Found**: Verify room ID includes `!` prefix and `:homeserver.com` suffix
+- **No Alerts**: Check database has recent data with `uv run python run.py --status`

docs/references/NOTABLE_DOCUMENTS.md

@@ -34,6 +34,20 @@ This document contains important references and external resources related to th
 - **Usage**: Reference for individual station characteristics and historical data patterns
 - **Station**: P.76 - บ้านแม่อีไฮ (Banb Mae I Hai)
 
+### **ThaiWater / HII (Hydro-Informatics Institute) Resources**
+
+#### **4. ThaiWater Portal**
+- **URL**: https://twa.thaiwater.net
+- **Description**: National water situation portal (rainfall, water level, dams, warnings)
+- **Language**: Thai/English
+- **Usage**: Backed by open and auth-gated APIs — full endpoint catalog in [DATA_SOURCES.md](../DATA_SOURCES.md)
+
+#### **5. ThaiWater Data Standard**
+- **URL**: https://standard.thaiwater.net
+- **Description**: Official water-data standard for exchange and warning — canonical station/basin/province code registries, data formats, warning-level definitions
+- **Language**: Thai
+- **Usage**: Reference for station metadata and warning-level semantics
+
 ## 📊 **Data Sources and APIs**
 
 ### **Primary Data Source**

models/eval_rain.json

@@ -0,0 +1,439 @@
+[
+ {
+  "station": "P.1",
+  "warn_thr": 3.7,
+  "folds": [
+   {
+    "year": 2021,
+    "n_train": 20024,
+    "n_test": 4392,
+    "events": [],
+    "variants": {
+     "rise": {
+      "mae": 0.070970681677648,
+      "mae_above_2p5": null,
+      "brier_warn": 0.0,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain": {
+      "mae": 0.07247641391827915,
+      "mae_above_2p5": null,
+      "brier_warn": 0.0,
+      "events": [],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2022,
+    "n_train": 28782,
+    "n_test": 4392,
+    "events": [
+     {
+      "crossing": "2022-10-02T19:00:00",
+      "peak_ts": "2022-10-03T15:00:00",
+      "peak_level": 4.65
+     }
+    ],
+    "variants": {
+     "rise": {
+      "mae": 0.08489734638010188,
+      "mae_above_2p5": 0.24517428534843191,
+      "brier_warn": 0.004136576477574926,
+      "events": [
+       {
+        "crossing": "2022-10-02T19:00:00",
+        "lead_h": 6.0,
+        "peak_level": 4.65,
+        "peak_pred_24h_before": 3.870617057762467
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain": {
+      "mae": 0.08491621780630447,
+      "mae_above_2p5": 0.24539225527181602,
+      "brier_warn": 0.004209435091817237,
+      "events": [
+       {
+        "crossing": "2022-10-02T19:00:00",
+        "lead_h": 5.0,
+        "peak_level": 4.65,
+        "peak_pred_24h_before": 3.844081593978701
+       }
+      ],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2023,
+    "n_train": 37537,
+    "n_test": 4392,
+    "events": [],
+    "variants": {
+     "rise": {
+      "mae": 0.07647507344443248,
+      "mae_above_2p5": 0.12593127745781565,
+      "brier_warn": 3.19529506149809e-11,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain": {
+      "mae": 0.07491688056430452,
+      "mae_above_2p5": 0.09506457784884237,
+      "brier_warn": 6.639993512164624e-14,
+      "events": [],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2024,
+    "n_train": 46321,
+    "n_test": 4392,
+    "events": [
+     {
+      "crossing": "2024-09-24T17:00:00",
+      "peak_ts": "2024-09-26T02:00:00",
+      "peak_level": 4.93
+     },
+     {
+      "crossing": "2024-10-03T09:00:00",
+      "peak_ts": "2024-10-05T12:00:00",
+      "peak_level": 5.3
+     }
+    ],
+    "variants": {
+     "rise": {
+      "mae": 0.10075626719251345,
+      "mae_above_2p5": 0.26384939692286363,
+      "brier_warn": 0.011608221543810462,
+      "events": [
+       {
+        "crossing": "2024-09-24T17:00:00",
+        "lead_h": 6.0,
+        "peak_level": 4.93,
+        "peak_pred_24h_before": 4.534709676862131
+       },
+       {
+        "crossing": "2024-10-03T09:00:00",
+        "lead_h": 3.0,
+        "peak_level": 5.3,
+        "peak_pred_24h_before": 5.056619694027486
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain": {
+      "mae": 0.08874847709708966,
+      "mae_above_2p5": 0.20967242138010514,
+      "brier_warn": 0.005862181747890141,
+      "events": [
+       {
+        "crossing": "2024-09-24T17:00:00",
+        "lead_h": 11.0,
+        "peak_level": 4.93,
+        "peak_pred_24h_before": 4.863137825109792
+       },
+       {
+        "crossing": "2024-10-03T09:00:00",
+        "lead_h": 21.0,
+        "peak_level": 5.3,
+        "peak_pred_24h_before": 5.557547530221961
+       }
+      ],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2025,
+    "n_train": 55081,
+    "n_test": 4392,
+    "events": [
+     {
+      "crossing": "2025-09-27T18:00:00",
+      "peak_ts": "2025-09-27T22:00:00",
+      "peak_level": 3.93
+     }
+    ],
+    "variants": {
+     "rise": {
+      "mae": 0.13102686839694494,
+      "mae_above_2p5": 0.2729944665011028,
+      "brier_warn": 0.010889835530782944,
+      "events": [
+       {
+        "crossing": "2025-09-27T18:00:00",
+        "lead_h": 46.0,
+        "peak_level": 3.93,
+        "peak_pred_24h_before": 3.232029710676319
+       }
+      ],
+      "false_alarm_episodes": 1
+     },
+     "rise_rain": {
+      "mae": 0.11160288394028639,
+      "mae_above_2p5": 0.24838881498904114,
+      "brier_warn": 0.005150122823202273,
+      "events": [
+       {
+        "crossing": "2025-09-27T18:00:00",
+        "lead_h": 2.0,
+        "peak_level": 3.93,
+        "peak_pred_24h_before": 3.239754736790134
+       }
+      ],
+      "false_alarm_episodes": 0
+     }
+    }
+   }
+  ]
+ },
+ {
+  "station": "P.103",
+  "warn_thr": 5.95,
+  "folds": [
+   {
+    "year": 2021,
+    "n_train": 20009,
+    "n_test": 4392,
+    "events": [],
+    "variants": {
+     "rise": {
+      "mae": 0.15279228710793116,
+      "mae_above_2p5": null,
+      "brier_warn": 0.0,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain": {
+      "mae": 0.15114584578006643,
+      "mae_above_2p5": null,
+      "brier_warn": 0.0,
+      "events": [],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2022,
+    "n_train": 28769,
+    "n_test": 4392,
+    "events": [
+     {
+      "crossing": "2022-08-14T04:00:00",
+      "peak_ts": "2022-08-14T08:00:00",
+      "peak_level": 6.09
+     },
+     {
+      "crossing": "2022-10-02T16:00:00",
+      "peak_ts": "2022-10-03T16:00:00",
+      "peak_level": 7.54
+     }
+    ],
+    "variants": {
+     "rise": {
+      "mae": 0.15389446229226025,
+      "mae_above_2p5": 0.41887094569113953,
+      "brier_warn": 0.007740077230532647,
+      "events": [
+       {
+        "crossing": "2022-08-14T04:00:00",
+        "lead_h": 6.0,
+        "peak_level": 6.09,
+        "peak_pred_24h_before": 4.837975953559253
+       },
+       {
+        "crossing": "2022-10-02T16:00:00",
+        "lead_h": 9.0,
+        "peak_level": 7.54,
+        "peak_pred_24h_before": 7.168225721504108
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain": {
+      "mae": 0.15303667736579823,
+      "mae_above_2p5": 0.38833713105646916,
+      "brier_warn": 0.0074511589478895475,
+      "events": [
+       {
+        "crossing": "2022-08-14T04:00:00",
+        "lead_h": 6.0,
+        "peak_level": 6.09,
+        "peak_pred_24h_before": 4.8804746828604815
+       },
+       {
+        "crossing": "2022-10-02T16:00:00",
+        "lead_h": 10.0,
+        "peak_level": 7.54,
+        "peak_pred_24h_before": 7.226685294045882
+       }
+      ],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2023,
+    "n_train": 37524,
+    "n_test": 4392,
+    "events": [],
+    "variants": {
+     "rise": {
+      "mae": 0.16106550376855028,
+      "mae_above_2p5": null,
+      "brier_warn": 0.0001779564366636896,
+      "events": [],
+      "false_alarm_episodes": 1
+     },
+     "rise_rain": {
+      "mae": 0.15541158498896382,
+      "mae_above_2p5": null,
+      "brier_warn": 4.103077380009017e-09,
+      "events": [],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2024,
+    "n_train": 46308,
+    "n_test": 4058,
+    "events": [
+     {
+      "crossing": "2024-09-24T10:00:00",
+      "peak_ts": "2024-09-26T00:00:00",
+      "peak_level": 8.27
+     },
+     {
+      "crossing": "2024-09-30T03:00:00",
+      "peak_ts": "2024-09-30T06:00:00",
+      "peak_level": 5.99
+     },
+     {
+      "crossing": "2024-10-03T06:00:00",
+      "peak_ts": "2024-10-05T07:00:00",
+      "peak_level": 9.93
+     }
+    ],
+    "variants": {
+     "rise": {
+      "mae": 0.17314177863843333,
+      "mae_above_2p5": 0.48294427141283425,
+      "brier_warn": 0.016091510788709233,
+      "events": [
+       {
+        "crossing": "2024-09-24T10:00:00",
+        "lead_h": 10.0,
+        "peak_level": 8.27,
+        "peak_pred_24h_before": 7.167139790234238
+       },
+       {
+        "crossing": "2024-09-30T03:00:00",
+        "lead_h": 10.0,
+        "peak_level": 5.99,
+        "peak_pred_24h_before": 5.618725525893519
+       },
+       {
+        "crossing": "2024-10-03T06:00:00",
+        "lead_h": 69.0,
+        "peak_level": 9.93,
+        "peak_pred_24h_before": 7.889819144742388
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_rain": {
+      "mae": 0.1400555603272253,
+      "mae_above_2p5": 0.2920105854094151,
+      "brier_warn": 0.007294262727198402,
+      "events": [
+       {
+        "crossing": "2024-09-24T10:00:00",
+        "lead_h": 19.0,
+        "peak_level": 8.27,
+        "peak_pred_24h_before": 8.04142923647258
+       },
+       {
+        "crossing": "2024-09-30T03:00:00",
+        "lead_h": 10.0,
+        "peak_level": 5.99,
+        "peak_pred_24h_before": 5.42
+       },
+       {
+        "crossing": "2024-10-03T06:00:00",
+        "lead_h": 55.0,
+        "peak_level": 9.93,
+        "peak_pred_24h_before": 8.655375706947945
+       }
+      ],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2025,
+    "n_train": 54577,
+    "n_test": 4392,
+    "events": [
+     {
+      "crossing": "2025-09-26T06:00:00",
+      "peak_ts": "2025-09-27T21:00:00",
+      "peak_level": 6.64
+     },
+     {
+      "crossing": "2025-10-03T06:00:00",
+      "peak_ts": "2025-10-03T12:00:00",
+      "peak_level": 6.14
+     }
+    ],
+    "variants": {
+     "rise": {
+      "mae": 0.21617454799997243,
+      "mae_above_2p5": 0.406245211140301,
+      "brier_warn": 0.012587550711857222,
+      "events": [
+       {
+        "crossing": "2025-09-26T06:00:00",
+        "lead_h": 14.0,
+        "peak_level": 6.64,
+        "peak_pred_24h_before": 5.73
+       },
+       {
+        "crossing": "2025-10-03T06:00:00",
+        "lead_h": 47.0,
+        "peak_level": 6.14,
+        "peak_pred_24h_before": 5.953951787788988
+       }
+      ],
+      "false_alarm_episodes": 1
+     },
+     "rise_rain": {
+      "mae": 0.17651730883692462,
+      "mae_above_2p5": 0.38926658786456825,
+      "brier_warn": 0.015873258461498164,
+      "events": [
+       {
+        "crossing": "2025-09-26T06:00:00",
+        "lead_h": 6.0,
+        "peak_level": 6.64,
+        "peak_pred_24h_before": 5.760150202082536
+       },
+       {
+        "crossing": "2025-10-03T06:00:00",
+        "lead_h": 8.0,
+        "peak_level": 6.14,
+        "peak_pred_24h_before": 5.605274163405912
+       }
+      ],
+      "false_alarm_episodes": 2
+     }
+    }
+   }
+  ]
+ }
+]

models/eval_variants.json

@@ -0,0 +1,723 @@
+[
+ {
+  "station": "P.1",
+  "warn_thr": 3.7,
+  "folds": [
+   {
+    "year": 2021,
+    "n_train": 20024,
+    "n_test": 4392,
+    "events": [],
+    "variants": {
+     "baseline_abs": {
+      "mae": 0.073069548799388,
+      "mae_above_2p5": null,
+      "brier_warn": 0.0,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise": {
+      "mae": 0.070970681677648,
+      "mae_above_2p5": null,
+      "brier_warn": 0.0,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_weighted": {
+      "mae": 0.07394501467643348,
+      "mae_above_2p5": null,
+      "brier_warn": 0.0,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_quantile": {
+      "mae": 0.07289422294875283,
+      "mae_above_2p5": null,
+      "brier_warn": 6.870660393452143e-17,
+      "events": [],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2022,
+    "n_train": 28782,
+    "n_test": 4392,
+    "events": [
+     {
+      "crossing": "2022-10-02T19:00:00",
+      "peak_ts": "2022-10-03T15:00:00",
+      "peak_level": 4.65
+     }
+    ],
+    "variants": {
+     "baseline_abs": {
+      "mae": 0.09128484356553154,
+      "mae_above_2p5": 0.2856025611084259,
+      "brier_warn": 0.005440790049399641,
+      "events": [
+       {
+        "crossing": "2022-10-02T19:00:00",
+        "lead_h": 0.0,
+        "peak_level": 4.65,
+        "peak_pred_24h_before": 3.4
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise": {
+      "mae": 0.08489734638010188,
+      "mae_above_2p5": 0.24517428534843191,
+      "brier_warn": 0.004136576477574926,
+      "events": [
+       {
+        "crossing": "2022-10-02T19:00:00",
+        "lead_h": 6.0,
+        "peak_level": 4.65,
+        "peak_pred_24h_before": 3.870617057762467
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_weighted": {
+      "mae": 0.08451609397908402,
+      "mae_above_2p5": 0.24677443475763222,
+      "brier_warn": 0.0041231071254735656,
+      "events": [
+       {
+        "crossing": "2022-10-02T19:00:00",
+        "lead_h": 6.0,
+        "peak_level": 4.65,
+        "peak_pred_24h_before": 3.9060484870307466
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_quantile": {
+      "mae": 0.08412283956084679,
+      "mae_above_2p5": 0.2653552568120815,
+      "brier_warn": 0.004349073127074364,
+      "events": [
+       {
+        "crossing": "2022-10-02T19:00:00",
+        "lead_h": 3.0,
+        "peak_level": 4.65,
+        "peak_pred_24h_before": 3.641048749261635
+       }
+      ],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2023,
+    "n_train": 37537,
+    "n_test": 4392,
+    "events": [],
+    "variants": {
+     "baseline_abs": {
+      "mae": 0.08161389876263064,
+      "mae_above_2p5": 0.1335546690803318,
+      "brier_warn": 3.284466170848282e-09,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise": {
+      "mae": 0.07647507344443248,
+      "mae_above_2p5": 0.12593127745781565,
+      "brier_warn": 3.19529506149809e-11,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_weighted": {
+      "mae": 0.07633405851952908,
+      "mae_above_2p5": 0.12234064428755373,
+      "brier_warn": 9.093694748477163e-10,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_quantile": {
+      "mae": 0.07460579730428984,
+      "mae_above_2p5": 0.11243109915383911,
+      "brier_warn": 8.148092046865233e-08,
+      "events": [],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2024,
+    "n_train": 46321,
+    "n_test": 4392,
+    "events": [
+     {
+      "crossing": "2024-09-24T17:00:00",
+      "peak_ts": "2024-09-26T02:00:00",
+      "peak_level": 4.93
+     },
+     {
+      "crossing": "2024-10-03T09:00:00",
+      "peak_ts": "2024-10-05T12:00:00",
+      "peak_level": 5.3
+     }
+    ],
+    "variants": {
+     "baseline_abs": {
+      "mae": 0.11011679980584997,
+      "mae_above_2p5": 0.28849258735632527,
+      "brier_warn": 0.010963910189916666,
+      "events": [
+       {
+        "crossing": "2024-09-24T17:00:00",
+        "lead_h": 0.0,
+        "peak_level": 4.93,
+        "peak_pred_24h_before": 4.39
+       },
+       {
+        "crossing": "2024-10-03T09:00:00",
+        "lead_h": 0.0,
+        "peak_level": 5.3,
+        "peak_pred_24h_before": 4.9
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise": {
+      "mae": 0.10075626719251345,
+      "mae_above_2p5": 0.26384939692286363,
+      "brier_warn": 0.011608221543810462,
+      "events": [
+       {
+        "crossing": "2024-09-24T17:00:00",
+        "lead_h": 6.0,
+        "peak_level": 4.93,
+        "peak_pred_24h_before": 4.534709676862131
+       },
+       {
+        "crossing": "2024-10-03T09:00:00",
+        "lead_h": 3.0,
+        "peak_level": 5.3,
+        "peak_pred_24h_before": 5.056619694027486
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_weighted": {
+      "mae": 0.10644718990351541,
+      "mae_above_2p5": 0.2848978622398853,
+      "brier_warn": 0.013872775238121434,
+      "events": [
+       {
+        "crossing": "2024-09-24T17:00:00",
+        "lead_h": 9.0,
+        "peak_level": 4.93,
+        "peak_pred_24h_before": 4.502016075573465
+       },
+       {
+        "crossing": "2024-10-03T09:00:00",
+        "lead_h": 72.0,
+        "peak_level": 5.3,
+        "peak_pred_24h_before": 5.123356409745132
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_quantile": {
+      "mae": 0.09617759284936986,
+      "mae_above_2p5": 0.271753261978628,
+      "brier_warn": 0.011660822672241142,
+      "events": [
+       {
+        "crossing": "2024-09-24T17:00:00",
+        "lead_h": 4.0,
+        "peak_level": 4.93,
+        "peak_pred_24h_before": 4.532042588397363
+       },
+       {
+        "crossing": "2024-10-03T09:00:00",
+        "lead_h": 2.0,
+        "peak_level": 5.3,
+        "peak_pred_24h_before": 4.956335200519836
+       }
+      ],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2025,
+    "n_train": 55081,
+    "n_test": 4392,
+    "events": [
+     {
+      "crossing": "2025-09-27T18:00:00",
+      "peak_ts": "2025-09-27T22:00:00",
+      "peak_level": 3.93
+     }
+    ],
+    "variants": {
+     "baseline_abs": {
+      "mae": 0.15985415338980086,
+      "mae_above_2p5": 0.277881996645181,
+      "brier_warn": 0.009824391159106445,
+      "events": [
+       {
+        "crossing": "2025-09-27T18:00:00",
+        "lead_h": 0.0,
+        "peak_level": 3.93,
+        "peak_pred_24h_before": 3.2561560254201525
+       }
+      ],
+      "false_alarm_episodes": 2
+     },
+     "rise": {
+      "mae": 0.13102686839694494,
+      "mae_above_2p5": 0.2729944665011028,
+      "brier_warn": 0.010889835530782944,
+      "events": [
+       {
+        "crossing": "2025-09-27T18:00:00",
+        "lead_h": 46.0,
+        "peak_level": 3.93,
+        "peak_pred_24h_before": 3.232029710676319
+       }
+      ],
+      "false_alarm_episodes": 1
+     },
+     "rise_weighted": {
+      "mae": 0.13121120517634968,
+      "mae_above_2p5": 0.27796411411759725,
+      "brier_warn": 0.009393099541680463,
+      "events": [
+       {
+        "crossing": "2025-09-27T18:00:00",
+        "lead_h": 46.0,
+        "peak_level": 3.93,
+        "peak_pred_24h_before": 3.23
+       }
+      ],
+      "false_alarm_episodes": 3
+     },
+     "rise_quantile": {
+      "mae": 0.11678707793162903,
+      "mae_above_2p5": 0.28528124941530186,
+      "brier_warn": 0.013009605253899563,
+      "events": [
+       {
+        "crossing": "2025-09-27T18:00:00",
+        "lead_h": 45.0,
+        "peak_level": 3.93,
+        "peak_pred_24h_before": 3.23
+       }
+      ],
+      "false_alarm_episodes": 3
+     }
+    }
+   }
+  ]
+ },
+ {
+  "station": "P.103",
+  "warn_thr": 5.95,
+  "folds": [
+   {
+    "year": 2021,
+    "n_train": 20009,
+    "n_test": 4392,
+    "events": [],
+    "variants": {
+     "baseline_abs": {
+      "mae": 0.1517830652111977,
+      "mae_above_2p5": null,
+      "brier_warn": 0.0,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise": {
+      "mae": 0.15279228710793116,
+      "mae_above_2p5": null,
+      "brier_warn": 0.0,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_weighted": {
+      "mae": 0.17632375724441426,
+      "mae_above_2p5": null,
+      "brier_warn": 0.0,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise_quantile": {
+      "mae": 0.13593873552532,
+      "mae_above_2p5": null,
+      "brier_warn": 1.6481640122270367e-09,
+      "events": [],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2022,
+    "n_train": 28769,
+    "n_test": 4392,
+    "events": [
+     {
+      "crossing": "2022-08-14T04:00:00",
+      "peak_ts": "2022-08-14T08:00:00",
+      "peak_level": 6.09
+     },
+     {
+      "crossing": "2022-10-02T16:00:00",
+      "peak_ts": "2022-10-03T16:00:00",
+      "peak_level": 7.54
+     }
+    ],
+    "variants": {
+     "baseline_abs": {
+      "mae": 0.15960776318568,
+      "mae_above_2p5": 0.5240325509357986,
+      "brier_warn": 0.010134410144044625,
+      "events": [
+       {
+        "crossing": "2022-08-14T04:00:00",
+        "lead_h": 0.0,
+        "peak_level": 6.09,
+        "peak_pred_24h_before": 5.01930531142127
+       },
+       {
+        "crossing": "2022-10-02T16:00:00",
+        "lead_h": 0.0,
+        "peak_level": 7.54,
+        "peak_pred_24h_before": 6.01
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise": {
+      "mae": 0.15389446229226025,
+      "mae_above_2p5": 0.41887094569113953,
+      "brier_warn": 0.007740077230532647,
+      "events": [
+       {
+        "crossing": "2022-08-14T04:00:00",
+        "lead_h": 6.0,
+        "peak_level": 6.09,
+        "peak_pred_24h_before": 4.837975953559253
+       },
+       {
+        "crossing": "2022-10-02T16:00:00",
+        "lead_h": 9.0,
+        "peak_level": 7.54,
+        "peak_pred_24h_before": 7.168225721504108
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_weighted": {
+      "mae": 0.1498671765711928,
+      "mae_above_2p5": 0.40088103598066704,
+      "brier_warn": 0.00801963186023152,
+      "events": [
+       {
+        "crossing": "2022-08-14T04:00:00",
+        "lead_h": 6.0,
+        "peak_level": 6.09,
+        "peak_pred_24h_before": 4.81
+       },
+       {
+        "crossing": "2022-10-02T16:00:00",
+        "lead_h": 8.0,
+        "peak_level": 7.54,
+        "peak_pred_24h_before": 7.002000385945215
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_quantile": {
+      "mae": 0.13833442255794418,
+      "mae_above_2p5": 0.4529809369770209,
+      "brier_warn": 0.009090606171021184,
+      "events": [
+       {
+        "crossing": "2022-08-14T04:00:00",
+        "lead_h": 1.0,
+        "peak_level": 6.09,
+        "peak_pred_24h_before": 4.8698313890426705
+       },
+       {
+        "crossing": "2022-10-02T16:00:00",
+        "lead_h": 5.0,
+        "peak_level": 7.54,
+        "peak_pred_24h_before": 6.650326949981244
+       }
+      ],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2023,
+    "n_train": 37524,
+    "n_test": 4392,
+    "events": [],
+    "variants": {
+     "baseline_abs": {
+      "mae": 0.1489843074454942,
+      "mae_above_2p5": null,
+      "brier_warn": 1.882373500761472e-08,
+      "events": [],
+      "false_alarm_episodes": 0
+     },
+     "rise": {
+      "mae": 0.16106550376855028,
+      "mae_above_2p5": null,
+      "brier_warn": 0.0001779564366636896,
+      "events": [],
+      "false_alarm_episodes": 1
+     },
+     "rise_weighted": {
+      "mae": 0.1688589840007776,
+      "mae_above_2p5": null,
+      "brier_warn": 0.0012753245446952602,
+      "events": [],
+      "false_alarm_episodes": 1
+     },
+     "rise_quantile": {
+      "mae": 0.1510624979748974,
+      "mae_above_2p5": null,
+      "brier_warn": 0.0005959001115900152,
+      "events": [],
+      "false_alarm_episodes": 1
+     }
+    }
+   },
+   {
+    "year": 2024,
+    "n_train": 46308,
+    "n_test": 4058,
+    "events": [
+     {
+      "crossing": "2024-09-24T10:00:00",
+      "peak_ts": "2024-09-26T00:00:00",
+      "peak_level": 8.27
+     },
+     {
+      "crossing": "2024-09-30T03:00:00",
+      "peak_ts": "2024-09-30T06:00:00",
+      "peak_level": 5.99
+     },
+     {
+      "crossing": "2024-10-03T06:00:00",
+      "peak_ts": "2024-10-05T07:00:00",
+      "peak_level": 9.93
+     }
+    ],
+    "variants": {
+     "baseline_abs": {
+      "mae": 0.193584781859895,
+      "mae_above_2p5": 0.5381754200484403,
+      "brier_warn": 0.018292133172974026,
+      "events": [
+       {
+        "crossing": "2024-09-24T10:00:00",
+        "lead_h": 5.0,
+        "peak_level": 8.27,
+        "peak_pred_24h_before": 7.11
+       },
+       {
+        "crossing": "2024-09-30T03:00:00",
+        "lead_h": 0.0,
+        "peak_level": 5.99,
+        "peak_pred_24h_before": 5.42
+       },
+       {
+        "crossing": "2024-10-03T06:00:00",
+        "lead_h": 0.0,
+        "peak_level": 9.93,
+        "peak_pred_24h_before": 7.86
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise": {
+      "mae": 0.17314177863843333,
+      "mae_above_2p5": 0.48294427141283425,
+      "brier_warn": 0.016091510788709233,
+      "events": [
+       {
+        "crossing": "2024-09-24T10:00:00",
+        "lead_h": 10.0,
+        "peak_level": 8.27,
+        "peak_pred_24h_before": 7.167139790234238
+       },
+       {
+        "crossing": "2024-09-30T03:00:00",
+        "lead_h": 10.0,
+        "peak_level": 5.99,
+        "peak_pred_24h_before": 5.618725525893519
+       },
+       {
+        "crossing": "2024-10-03T06:00:00",
+        "lead_h": 69.0,
+        "peak_level": 9.93,
+        "peak_pred_24h_before": 7.889819144742388
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_weighted": {
+      "mae": 0.18833560689452924,
+      "mae_above_2p5": 0.4948752445774323,
+      "brier_warn": 0.01622076553986666,
+      "events": [
+       {
+        "crossing": "2024-09-24T10:00:00",
+        "lead_h": 11.0,
+        "peak_level": 8.27,
+        "peak_pred_24h_before": 7.283961881370446
+       },
+       {
+        "crossing": "2024-09-30T03:00:00",
+        "lead_h": 10.0,
+        "peak_level": 5.99,
+        "peak_pred_24h_before": 5.783590096006894
+       },
+       {
+        "crossing": "2024-10-03T06:00:00",
+        "lead_h": 69.0,
+        "peak_level": 9.93,
+        "peak_pred_24h_before": 7.922098265471745
+       }
+      ],
+      "false_alarm_episodes": 0
+     },
+     "rise_quantile": {
+      "mae": 0.1644888402154109,
+      "mae_above_2p5": 0.46461119464587947,
+      "brier_warn": 0.013172046828124688,
+      "events": [
+       {
+        "crossing": "2024-09-24T10:00:00",
+        "lead_h": 11.0,
+        "peak_level": 8.27,
+        "peak_pred_24h_before": 7.348489352370365
+       },
+       {
+        "crossing": "2024-09-30T03:00:00",
+        "lead_h": 8.0,
+        "peak_level": 5.99,
+        "peak_pred_24h_before": 5.42
+       },
+       {
+        "crossing": "2024-10-03T06:00:00",
+        "lead_h": 2.0,
+        "peak_level": 9.93,
+        "peak_pred_24h_before": 7.91343423984052
+       }
+      ],
+      "false_alarm_episodes": 0
+     }
+    }
+   },
+   {
+    "year": 2025,
+    "n_train": 54577,
+    "n_test": 4392,
+    "events": [
+     {
+      "crossing": "2025-09-26T06:00:00",
+      "peak_ts": "2025-09-27T21:00:00",
+      "peak_level": 6.64
+     },
+     {
+      "crossing": "2025-10-03T06:00:00",
+      "peak_ts": "2025-10-03T12:00:00",
+      "peak_level": 6.14
+     }
+    ],
+    "variants": {
+     "baseline_abs": {
+      "mae": 0.2036944006891808,
+      "mae_above_2p5": 0.40787978637525985,
+      "brier_warn": 0.014130122065259989,
+      "events": [
+       {
+        "crossing": "2025-09-26T06:00:00",
+        "lead_h": 14.0,
+        "peak_level": 6.64,
+        "peak_pred_24h_before": 5.73
+       },
+       {
+        "crossing": "2025-10-03T06:00:00",
+        "lead_h": 41.0,
+        "peak_level": 6.14,
+        "peak_pred_24h_before": 5.703778873833753
+       }
+      ],
+      "false_alarm_episodes": 2
+     },
+     "rise": {
+      "mae": 0.21617454799997243,
+      "mae_above_2p5": 0.406245211140301,
+      "brier_warn": 0.012587550711857222,
+      "events": [
+       {
+        "crossing": "2025-09-26T06:00:00",
+        "lead_h": 14.0,
+        "peak_level": 6.64,
+        "peak_pred_24h_before": 5.73
+       },
+       {
+        "crossing": "2025-10-03T06:00:00",
+        "lead_h": 47.0,
+        "peak_level": 6.14,
+        "peak_pred_24h_before": 5.953951787788988
+       }
+      ],
+      "false_alarm_episodes": 1
+     },
+     "rise_weighted": {
+      "mae": 0.21768864574559124,
+      "mae_above_2p5": 0.41071503357135775,
+      "brier_warn": 0.014605838473436519,
+      "events": [
+       {
+        "crossing": "2025-09-26T06:00:00",
+        "lead_h": 13.0,
+        "peak_level": 6.64,
+        "peak_pred_24h_before": 5.73
+       },
+       {
+        "crossing": "2025-10-03T06:00:00",
+        "lead_h": 49.0,
+        "peak_level": 6.14,
+        "peak_pred_24h_before": 5.606011827978066
+       }
+      ],
+      "false_alarm_episodes": 1
+     },
+     "rise_quantile": {
+      "mae": 0.18196250028633115,
+      "mae_above_2p5": 0.3987262421172889,
+      "brier_warn": 0.011560937993250782,
+      "events": [
+       {
+        "crossing": "2025-09-26T06:00:00",
+        "lead_h": 13.0,
+        "peak_level": 6.64,
+        "peak_pred_24h_before": 5.73
+       },
+       {
+        "crossing": "2025-10-03T06:00:00",
+        "lead_h": 42.0,
+        "peak_level": 6.14,
+        "peak_pred_24h_before": 6.00597561680636
+       }
+      ],
+      "false_alarm_episodes": 1
+     }
+    }
+   }
+  ]
+ }
+]

ping-river-monitor.spec

@@ -0,0 +1,38 @@
+# -*- mode: python ; coding: utf-8 -*-
+
+
+a = Analysis(
+    ['run.py'],
+    pathex=[],
+    binaries=[],
+    datas=[('.env', '.'), ('sql', 'sql'), ('README.md', '.'), ('POSTGRESQL_SETUP.md', '.'), ('SQLITE_MIGRATION.md', '.')],
+    hiddenimports=['psycopg2', 'sqlalchemy.dialects.postgresql', 'sqlalchemy.dialects.sqlite', 'dotenv', 'pydantic', 'fastapi', 'uvicorn', 'schedule', 'pandas'],
+    hookspath=[],
+    hooksconfig={},
+    runtime_hooks=[],
+    excludes=[],
+    noarchive=False,
+    optimize=0,
+)
+pyz = PYZ(a.pure)
+
+exe = EXE(
+    pyz,
+    a.scripts,
+    a.binaries,
+    a.datas,
+    [],
+    name='ping-river-monitor',
+    debug=False,
+    bootloader_ignore_signals=False,
+    strip=False,
+    upx=True,
+    upx_exclude=[],
+    runtime_tmpdir=None,
+    console=True,
+    disable_windowed_traceback=False,
+    argv_emulation=False,
+    target_arch=None,
+    codesign_identity=None,
+    entitlements_file=None,
+)

pyproject.toml

@@ -0,0 +1,130 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "northern-thailand-ping-river-monitor"
+version = "3.1.3"
+description = "Real-time water level monitoring system for the Ping River Basin in Northern Thailand"
+readme = "README.md"
+license = {text = "MIT"}
+authors = [
+    {name = "Ping River Monitor Team", email = "contact@example.com"}
+]
+keywords = [
+    "water monitoring",
+    "hydrology",
+    "thailand",
+    "ping river",
+    "environmental monitoring",
+    "time series",
+    "fastapi",
+    "real-time data"
+]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Science/Research",
+    "Intended Audience :: System Administrators",
+    "Topic :: Scientific/Engineering :: Hydrology",
+    "Topic :: System :: Monitoring",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Operating System :: OS Independent",
+    "Environment :: Web Environment",
+    "Framework :: FastAPI"
+]
+requires-python = ">=3.11"
+dependencies = [
+    # Core dependencies
+    "requests==2.31.0",
+    "schedule==1.2.0",
+    "pandas==2.0.3",
+    "numpy>=1.24,<2",
+    # Flood forecasting (ML)
+    "scikit-learn==1.9.0",
+    # Web API framework
+    "fastapi==0.104.1",
+    "uvicorn[standard]==0.24.0",
+    "pydantic==2.5.0",
+    # Database adapters
+    "sqlalchemy==2.0.23",
+    "influxdb==5.3.1",
+    "pymysql==1.1.0",
+    "psycopg2-binary==2.9.9",
+    # Monitoring and metrics
+    "psutil==5.9.6"
+]
+
+[project.optional-dependencies]
+dev = [
+    # Testing
+    "pytest==7.4.3",
+    "pytest-cov==4.1.0",
+    "pytest-asyncio==0.21.1",
+    # Code formatting and linting
+    "black==23.11.0",
+    "flake8==6.1.0",
+    "isort==5.12.0",
+    "mypy==1.7.1",
+    # Pre-commit hooks
+    "pre-commit==3.5.0",
+    # Development tools
+    "ipython==8.17.2",
+    "jupyter==1.0.0",
+    # Type stubs
+    "types-requests==2.31.0.10",
+    "types-python-dateutil==2.8.19.14"
+]
+docs = [
+    "sphinx==7.2.6",
+    "sphinx-rtd-theme==1.3.0",
+    "sphinx-autodoc-typehints==1.25.2"
+]
+all = [
+    "influxdb==5.3.1",
+    "pymysql==1.1.0",
+    "psycopg2-binary==2.9.9"
+]
+
+[project.scripts]
+ping-river-monitor = "src.main:main"
+ping-river-api = "src.web_api:main"
+
+[project.urls]
+Homepage = "https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor"
+Repository = "https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor"
+Issues = "https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/issues"
+Documentation = "https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/wiki"
+
+[dependency-groups]
+dev = [
+    # Testing
+    "pytest==7.4.3",
+    "pytest-cov==4.1.0",
+    "pytest-asyncio==0.21.1",
+    # Code formatting and linting
+    "black==23.11.0",
+    "flake8==6.1.0",
+    "isort==5.12.0",
+    "mypy==1.7.1",
+    # Pre-commit hooks
+    "pre-commit==3.5.0",
+    # Development tools
+    "ipython==8.17.2",
+    "jupyter==1.0.0",
+    # Type stubs
+    "types-requests==2.31.0.10",
+    "types-python-dateutil==2.8.19.14",
+    # Documentation
+    "sphinx==7.2.6",
+    "sphinx-rtd-theme==1.3.0",
+    "sphinx-autodoc-typehints==1.25.2",
+    "pyinstaller>=6.16.0",
+]
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-dir]
+"" = "src"

requirements.txt

@@ -2,6 +2,10 @@
 requests==2.31.0
 schedule==1.2.0
 pandas==2.0.3
+numpy>=1.24,<2  # pandas 2.0.3 wheels are ABI-incompatible with numpy 2.x
+
+# Flood forecasting (ML)
+scikit-learn==1.9.0
 
 # Web API framework
 fastapi==0.104.1

run.py

@@ -3,12 +3,13 @@
 Simple startup script for Thailand Water Monitor
 """
 
-import sys
 import os
+import sys
 
 # Add src directory to Python path
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src'))
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "src"))
 
 if __name__ == "__main__":
     from src.main import main
-    main()
+
+    main()

scripts/backfill_hii_waterlevel.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+"""CLI entry point for backfilling hii_waterlevel from the HII archive.
+
+Usage:
+    python scripts/backfill_hii_waterlevel.py                      # key stations, 2019..today
+    python scripts/backfill_hii_waterlevel.py --stations P.1,P.67
+    python scripts/backfill_hii_waterlevel.py --start 2024-09-01 --end 2024-11-01 --all
+"""
+
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+from src.hii_backfill import main
+
+if __name__ == "__main__":
+    sys.exit(0 if main() else 1)

scripts/backfill_rain_db.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+"""Backfill the openmeteo_rain table with the full 2021+ catchment history.
+
+Usage:
+    uv run scripts/backfill_rain_db.py                 # DB from Config/.env
+    uv run scripts/backfill_rain_db.py --db-url postgresql://...
+"""
+
+import argparse
+import logging
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+from src.config import Config
+from src.ml.rain import backfill_db
+
+
+def main(argv=None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--db-url", default=None)
+    args = parser.parse_args(argv)
+
+    logging.basicConfig(
+        level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s"
+    )
+    if args.db_url:
+        # postgresql+psycopg2://... -> postgresql (driver suffix is not a dialect)
+        connection_string = args.db_url
+        db_type = args.db_url.split(":", 1)[0].split("+", 1)[0]
+    else:
+        cfg = Config.get_database_config()
+        if cfg["type"] not in ("sqlite", "postgresql", "mysql"):
+            print(f"requires a SQL database, got {cfg['type']}", file=sys.stderr)
+            return 1
+        connection_string, db_type = cfg["connection_string"], cfg["type"]
+
+    from sqlalchemy import create_engine
+
+    engine = create_engine(connection_string, pool_pre_ping=True)
+    saved = backfill_db(engine, db_type)
+    print(f"backfilled {saved} hourly rows into openmeteo_rain")
+    return 0 if saved else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/backfill_rid_reservoir.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+"""Backfill rid_reservoir_daily with RID large-dam history (Mae Ngat et al.).
+
+One request per day against app.rid.go.th/reservoir/api/dams (archive reaches
+back to at least 2009). Days already stored are skipped, so reruns only fetch
+what is missing — safe alongside the hourly live collector, and a rerun
+repairs holes left by transient failures.
+
+Usage:
+    uv run scripts/backfill_rid_reservoir.py                    # missing days since 2018-08-01
+    uv run scripts/backfill_rid_reservoir.py --start 2015-01-01
+    uv run scripts/backfill_rid_reservoir.py --db-url postgresql://...
+"""
+
+import argparse
+import datetime
+import logging
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+from src.config import Config
+from src.rid_reservoir import RidReservoirStore, backfill
+
+DEFAULT_START = datetime.date(2018, 8, 1)  # start of the water_measurements grid
+
+
+def main(argv=None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--start", type=datetime.date.fromisoformat, default=DEFAULT_START
+    )
+    parser.add_argument("--end", type=datetime.date.fromisoformat, default=None)
+    parser.add_argument("--db-url", default=None)
+    parser.add_argument("--throttle", type=float, default=0.4)
+    args = parser.parse_args(argv)
+
+    logging.basicConfig(
+        level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s"
+    )
+    if args.db_url:
+        # postgresql+psycopg2://... -> postgresql (driver suffix is not a dialect)
+        connection_string = args.db_url
+        db_type = args.db_url.split(":", 1)[0].split("+", 1)[0]
+    else:
+        cfg = Config.get_database_config()
+        if cfg["type"] not in ("sqlite", "postgresql", "mysql"):
+            print(f"requires a SQL database, got {cfg['type']}", file=sys.stderr)
+            return 1
+        connection_string, db_type = cfg["connection_string"], cfg["type"]
+
+    store = RidReservoirStore(connection_string, db_type)
+    if not store.connect():
+        print(
+            "database connection failed — check the connection string",
+            file=sys.stderr,
+        )
+        return 1
+    end = args.end or datetime.date.today()
+    span_days = (end - args.start).days + 1
+    missing = span_days - len(store.present_dates(args.start, end))
+    saved = backfill(store, args.start, end, throttle_seconds=args.throttle)
+    print(f"backfilled {saved} dam-day rows ({missing} days were missing)")
+    return 0 if saved or missing == 0 else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/backtest_render.py

@@ -0,0 +1,266 @@
+#!/usr/bin/env python3
+"""Regenerate the documented P.1 flood-backtest charts in docs/img/.
+
+For each chart an eval-only model (regression 24 h peak + warning classifier)
+is trained on data STRICTLY BEFORE the event, then the event window is walked
+hour by hour exactly as the live system would have seen it:
+
+  backtest-2024-p1.png         Oct 2024 record flood, trained < 1 Sep 2024
+  backtest-2024-p1-detail.png  22-28 Sep 2024 zoom of the first crossing
+  backtest-2025-p1.png         Sep 2025 flood, deployed config (trained <= 2024)
+
+This codifies the previously prose-only acceptance test: the run fails with a
+non-zero exit if the model gives less than 12 h of warning before the first
+3.70 m crossing of the 2024 event.
+
+Usage:
+    python scripts/backtest_render.py                 # uses FLOOD_ML_DB_URL/Config
+    python scripts/backtest_render.py --db-url postgresql://...
+"""
+
+import argparse
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+import matplotlib
+
+matplotlib.use("Agg")
+import matplotlib.dates as mdates
+import matplotlib.pyplot as plt
+import pandas as pd
+
+from src.ml import data, features
+from src.ml.train import _make_classifier, _make_regressor
+
+STATION = "P.1"
+STAGE1 = 3.70  # official Chiang Mai stage 1 - city flooding begins
+STAGE7 = 4.60  # stage 7 - widespread
+HORIZON = 24
+
+INK = "#132b35"
+BLUE = "#1c6ea4"
+AMBER = "#c07d10"
+RED = "#d9534f"
+
+
+def fit_backtest_model(df_long: pd.DataFrame, train_end: str):
+    """Train the 24 h regression + warning heads on rows <= train_end only.
+
+    Mirrors the deployed hgb-v3 pipeline: the regression head learns the RISE
+    over the current level, with Open-Meteo catchment-rain features (trailing
+    sums + the forward-24h forecast sum); label statistics are bounded to the
+    training cutoff.
+    """
+    from src.ml import rain as rain_mod
+
+    rain_series = rain_mod.catchment_mean(rain_mod.load_history())
+    X, Y, _meta = features.build_matrix(
+        df_long, STATION, (HORIZON,), stats_end=train_end, rain=rain_series
+    )
+    train_mask = X.index <= pd.Timestamp(train_end)
+    X_train, Y_train = X.loc[train_mask], Y.loc[train_mask]
+
+    max_col, warn_col = f"max_level_{HORIZON}", f"exceed_warn_{HORIZON}"
+    reg_rows = Y_train[max_col].notna()
+    rise = Y_train.loc[reg_rows, max_col] - X_train.loc[reg_rows, "level"]
+    reg = _make_regressor().fit(X_train.loc[reg_rows], rise)
+    warn_rows = Y_train[warn_col].notna()
+    clf = _make_classifier().fit(
+        X_train.loc[warn_rows], Y_train.loc[warn_rows, warn_col].astype(int)
+    )
+    return X, reg, clf
+
+
+def event_series(df_long, X, reg, clf, window_start: str, window_end: str):
+    """Observed level plus the forecasts the model would have issued hourly."""
+    grid = features.make_hourly_grid(df_long)
+    # observed has MultiIndex columns (station_code, field)
+    observed = grid.observed[(STATION, "water_level")]
+    observed = observed.loc[window_start:window_end].dropna().astype(float)
+
+    Xw = X.loc[window_start:window_end]
+    forecasts = pd.DataFrame(index=Xw.index)
+    # reg predicts the rise; add the current level back (as serving does)
+    forecasts["pred_max"] = reg.predict(Xw) + Xw["level"].to_numpy()
+    # Belt-and-braces probability: the classifier OR the regression-sigmoid,
+    # whichever is more alarmed. The classifier alone proved unreliable on
+    # out-of-distribution extremes (silent on the 2024 record flood).
+    import numpy as np
+    p_clf = clf.predict_proba(Xw)[:, 1]
+    p_sig = 1.0 / (1.0 + np.exp(-(forecasts["pred_max"] - STAGE1) / 0.15))
+    forecasts["p_flood"] = np.maximum(p_clf, p_sig)
+
+    flood_start = observed[observed >= STAGE1].index.min()
+    alerts = forecasts[forecasts["p_flood"] >= 0.5].index
+    first_alert = alerts.min() if len(alerts) else None
+    return observed, forecasts, flood_start, first_alert
+
+
+def _style_axes(ax):
+    ax.spines[["top", "right"]].set_visible(False)
+    ax.tick_params(colors=INK, labelsize=11)
+    ax.grid(axis="y", color="#dfe9e7", linewidth=0.8)
+    ax.set_axisbelow(True)
+
+
+def render(observed, forecasts, flood_start, first_alert, out_path, *,
+           title, subtitle, detail=False, show_stage7=False, peak_note=None):
+    fig, (ax, axp) = plt.subplots(
+        2, 1, figsize=(12.6, 7.6), sharex=True,
+        gridspec_kw={"height_ratios": [2.2, 1], "hspace": 0.12},
+    )
+    fig.patch.set_facecolor("white")
+
+    marker = dict(marker="o", markersize=3) if detail else {}
+    ax.plot(observed.index, observed.values, color=BLUE, linewidth=2.2,
+            label="Observed level" + (" (hourly)" if detail else ""), **marker)
+    marker = dict(marker="s", markersize=3) if detail else {}
+    ax.plot(forecasts.index, forecasts["pred_max"], color=AMBER, linewidth=2,
+            linestyle="--", label="Predicted 24 h peak (issued at that hour)", **marker)
+
+    ax.axhline(STAGE1, color=RED, linewidth=1, alpha=0.65)
+    ax.annotate(f"{STAGE1:.2f} m · stage 1 · flooding begins", xy=(0.06, STAGE1),
+                xycoords=("axes fraction", "data"), xytext=(0, 5),
+                textcoords="offset points", color=RED, fontsize=10.5)
+    if show_stage7:
+        ax.axhline(STAGE7, color=RED, linewidth=1, alpha=0.65)
+        ax.annotate(f"{STAGE7:.2f} m · stage 7 · widespread", xy=(0.06, STAGE7),
+                    xycoords=("axes fraction", "data"), xytext=(0, 5),
+                    textcoords="offset points", color=RED, fontsize=10.5)
+
+    if peak_note:
+        peak_ts = observed.idxmax()
+        ax.annotate(peak_note, xy=(peak_ts, observed.max()),
+                    xytext=(12, 10), textcoords="offset points",
+                    color=BLUE, fontsize=11.5, fontweight="bold")
+
+    ax.set_ylabel("P.1 water level (m)", color=INK, fontsize=11.5)
+    ax.legend(loc="upper left", frameon=False, fontsize=10.5)
+    _style_axes(ax)
+
+    axp.plot(forecasts.index, forecasts["p_flood"], color=AMBER, linewidth=1.8)
+    axp.fill_between(forecasts.index, 0, forecasts["p_flood"],
+                     color=AMBER, alpha=0.28)
+    axp.axhline(0.5, color=INK, linewidth=0.9, linestyle=":", alpha=0.6)
+    axp.set_ylim(-0.02, 1.1)
+    axp.set_ylabel(f"P(flooding within {HORIZON} h)", color=INK, fontsize=11.5)
+    _style_axes(axp)
+
+    if first_alert is not None:
+        lead_h = None if flood_start is None else \
+            int((flood_start - first_alert).total_seconds() // 3600)
+        lead_txt = "" if lead_h is None else (
+            f"\n({lead_h} h before flooding began)" if lead_h >= 0
+            else f"\n({-lead_h} h after flooding began)"
+        )
+        if detail and flood_start is not None:
+            for a in (ax, axp):
+                a.axvline(first_alert, color=AMBER, linewidth=1.4, alpha=0.85)
+                a.axvline(flood_start, color=BLUE, linewidth=1.4, alpha=0.85)
+            # Anchor labels away from each other in chronological order so a
+            # late alert (alert AFTER crossing) cannot overprint the labels.
+            events = sorted(
+                [(first_alert, "model alert", AMBER), (flood_start, "flooding begins", BLUE)]
+            )
+            for (ts, label, color), (offset, align) in zip(events, ((-8, "right"), (8, "left"))):
+                ax.annotate(f"{label}\n{ts:%d %b %H:%M}",
+                            xy=(ts, observed.min()), xytext=(offset, 18),
+                            textcoords="offset points", ha=align,
+                            color=color, fontsize=11, fontweight="bold")
+            mid_y = observed.min() + (observed.max() - observed.min()) * 0.28
+            ax.annotate("", xy=(flood_start, mid_y), xytext=(first_alert, mid_y),
+                        arrowprops=dict(arrowstyle="<->", color=INK, lw=1.3))
+            arrow_label = (
+                f"{lead_h} h warning" if lead_h >= 0 else f"alert {-lead_h} h late"
+            )
+            ax.annotate(arrow_label,
+                        xy=(first_alert + (flood_start - first_alert) / 2, mid_y),
+                        xytext=(0, 8), textcoords="offset points", ha="center",
+                        color=INK, fontsize=11.5, fontweight="bold")
+        else:
+            axp.annotate(f"first alert · {first_alert:%d %b %H:%M}{lead_txt}",
+                         xy=(first_alert, 0.62), xytext=(10, 0),
+                         textcoords="offset points", color=RED, fontsize=10.5,
+                         bbox=dict(facecolor="white", alpha=0.75, edgecolor="none"))
+
+    locator = mdates.DayLocator(interval=1 if detail else 3)
+    axp.xaxis.set_major_locator(locator)
+    axp.xaxis.set_major_formatter(mdates.DateFormatter("%d %b"))
+    fig.suptitle(f"{title}\n{subtitle}", x=0.07, y=0.985, ha="left",
+                 fontsize=15, color=INK)
+    fig.subplots_adjust(top=0.885, left=0.07, right=0.97, bottom=0.07)
+    fig.savefig(out_path, dpi=110)
+    plt.close(fig)
+    print(f"wrote {out_path}")
+
+
+def main(argv=None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--db-url", default=None)
+    parser.add_argument("--out-dir", default=os.path.join("docs", "img"))
+    args = parser.parse_args(argv)
+
+    df = data.load_measurements(db_url=args.db_url)
+    if df.empty:
+        print("no measurement data available", file=sys.stderr)
+        return 1
+    os.makedirs(args.out_dir, exist_ok=True)
+
+    # --- October 2024 record flood: trained only on data before 1 Sep 2024 ---
+    X, reg, clf = fit_backtest_model(df, "2024-08-31")
+    obs, fc, flood_start, first_alert = event_series(
+        df, X, reg, clf, "2024-09-10", "2024-10-14 23:00")
+    peak = float(obs.max())
+    render(obs, fc, flood_start, first_alert,
+           os.path.join(args.out_dir, "backtest-2024-p1.png"),
+           title="October 2024 flood: what the model saw coming",
+           subtitle="P.1 Nawarat Bridge — model trained only on data before 1 Sep 2024",
+           show_stage7=True, peak_note=f"record peak {peak:.2f} m")
+
+    obs_d, fc_d, flood_d, alert_d = event_series(
+        df, X, reg, clf, "2024-09-21 18:00", "2024-09-28 06:00")
+    lead_h = None
+    if alert_d is not None and flood_d is not None:
+        lead_h = int((flood_d - alert_d).total_seconds() // 3600)
+    render(obs_d, fc_d, flood_d, alert_d,
+           os.path.join(args.out_dir, "backtest-2024-p1-detail.png"),
+           title="Detection in detail: 22–28 September 2024, hour by hour",
+           subtitle=(
+               f"the model alerts {lead_h} h before the river crosses the flooding line"
+               if lead_h is not None and lead_h > 0
+               else "model alert vs the river crossing the flooding line"
+           ),
+           detail=True)
+
+    # --- September 2025 flood: the deployed configuration (trained <= 2024) ---
+    X25, reg25, clf25 = fit_backtest_model(df, "2024-12-31")
+    obs25, fc25, flood25, alert25 = event_series(
+        df, X25, reg25, clf25, "2025-09-22", "2025-10-02 12:00")
+    pred_at_alert = float(fc25.loc[alert25:, "pred_max"].iloc[:24].max()) if alert25 is not None else None
+    note = f"peak {float(obs25.max()):.2f} m" + (
+        f" (predicted {pred_at_alert:.2f} m)" if pred_at_alert is not None else "")
+    render(obs25, fc25, flood25, alert25,
+           os.path.join(args.out_dir, "backtest-2025-p1.png"),
+           title="The September 2025 flood — as forecast by the deployed configuration",
+           subtitle="model trained only on data through 2024; this event was never seen in training",
+           detail=True, peak_note=note)
+
+    print(f"2024: flooding began {flood_start}, first alert {first_alert}")
+    print(f"2025: flooding began {flood25}, first alert {alert25}")
+
+    # Acceptance gate: the flagship 2024 event must keep a >= 12 h warning
+    if first_alert is None or flood_start is None:
+        print("FAIL: 2024 event alert or crossing not found", file=sys.stderr)
+        return 1
+    lead = (flood_start - first_alert).total_seconds() / 3600
+    if lead < 12:
+        print(f"FAIL: 2024 first-alert lead {lead:.0f} h < 12 h", file=sys.stderr)
+        return 1
+    print(f"PASS: 2024 first-alert lead {lead:.0f} h")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/encode_password.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+"""
+Password URL encoder for PostgreSQL connection strings
+"""
+
+import urllib.parse
+import sys
+
+def encode_password(password: str) -> str:
+    """URL encode a password for use in connection strings"""
+    return urllib.parse.quote(password, safe='')
+
+def build_connection_string(username: str, password: str, host: str, port: int, database: str) -> str:
+    """Build a properly encoded PostgreSQL connection string"""
+    encoded_password = encode_password(password)
+    return f"postgresql://{username}:{encoded_password}@{host}:{port}/{database}"
+
+def main():
+    print("PostgreSQL Password URL Encoder")
+    print("=" * 40)
+
+    if len(sys.argv) > 1:
+        # Password provided as argument
+        password = sys.argv[1]
+    else:
+        # Interactive mode
+        password = input("Enter your password: ")
+
+    encoded = encode_password(password)
+
+    print(f"\nOriginal password: {password}")
+    print(f"URL encoded: {encoded}")
+
+    # Optional: build full connection string
+    try:
+        build_full = input("\nBuild full connection string? (y/N): ").strip().lower() == 'y'
+    except (EOFError, KeyboardInterrupt):
+        print("\nDone!")
+        return
+
+    if build_full:
+        username = input("Username: ").strip()
+        host = input("Host: ").strip()
+        port = input("Port [5432]: ").strip() or "5432"
+        database = input("Database [water_monitoring]: ").strip() or "water_monitoring"
+
+        connection_string = build_connection_string(username, password, host, int(port), database)
+
+        print(f"\nComplete connection string:")
+        print(f"POSTGRES_CONNECTION_STRING={connection_string}")
+
+        print(f"\nAdd this to your .env file:")
+        print(f"DB_TYPE=postgresql")
+        print(f"POSTGRES_CONNECTION_STRING={connection_string}")
+
+if __name__ == "__main__":
+    main()

scripts/evaluate_variants.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+"""CLI for the rolling-origin model-variant evaluation.
+
+Usage:
+    uv run scripts/evaluate_variants.py                        # P.1, all variants
+    uv run scripts/evaluate_variants.py --stations P.1,P.103
+    uv run scripts/evaluate_variants.py --variants baseline_abs,rise_quantile
+"""
+
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+from src.ml.evaluate import main
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/install.sh

@@ -0,0 +1,114 @@
+#!/usr/bin/env bash
+#
+# Install the Thailand Water Level Monitor as a hardened systemd service.
+#
+# Creates a dedicated system user, deploys the code to /opt, builds a uv-managed
+# virtualenv, installs the systemd unit, and enables the service. Idempotent:
+# safe to re-run to update an existing install.
+#
+# Usage (as root, from a checkout of the repo):
+#     sudo bash scripts/install.sh
+#
+# Override defaults via environment variables:
+#     APP_DIR=/opt/thailand-water-monitor SERVICE_USER=water-monitor sudo -E bash scripts/install.sh
+#
+set -euo pipefail
+
+APP_DIR="${APP_DIR:-/opt/thailand-water-monitor}"
+SERVICE_USER="${SERVICE_USER:-water-monitor}"
+SERVICE_GROUP="${SERVICE_GROUP:-${SERVICE_USER}}"
+SERVICE_NAME="water-monitor.service"
+
+# Resolve the repo root (parent of this scripts/ directory).
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
+
+log() { printf '\033[1;32m==>\033[0m %s\n' "$*"; }
+warn() { printf '\033[1;33m[warn]\033[0m %s\n' "$*"; }
+die() { printf '\033[1;31m[error]\033[0m %s\n' "$*" >&2; exit 1; }
+
+[ "$(id -u)" -eq 0 ] || die "This script must be run as root (use sudo)."
+
+# 1. Dedicated system user/group (no login, no home) --------------------------
+if ! getent group "${SERVICE_GROUP}" >/dev/null; then
+    log "Creating group ${SERVICE_GROUP}"
+    groupadd --system "${SERVICE_GROUP}"
+fi
+if ! id "${SERVICE_USER}" >/dev/null 2>&1; then
+    log "Creating system user ${SERVICE_USER}"
+    useradd --system --no-create-home --shell /usr/sbin/nologin \
+        --gid "${SERVICE_GROUP}" "${SERVICE_USER}"
+fi
+
+# 2. Deploy code to APP_DIR ----------------------------------------------------
+log "Deploying code to ${APP_DIR}"
+mkdir -p "${APP_DIR}"
+if command -v rsync >/dev/null 2>&1; then
+    rsync -a --delete \
+        --exclude '.git' --exclude '.venv' --exclude 'venv' \
+        --exclude '__pycache__' --exclude '*.pyc' \
+        --exclude '*.db' --exclude '.env' --exclude 'stations.json' \
+        "${REPO_DIR}/" "${APP_DIR}/"
+else
+    warn "rsync not found; falling back to cp (will not prune deleted files)"
+    cp -r "${REPO_DIR}/." "${APP_DIR}/"
+fi
+
+# 3. Build the uv-managed virtualenv ------------------------------------------
+# Prefer an already-installed uv. For stricter supply-chain control install uv
+# ahead of time via your distro / package manager; this script only fetches the
+# upstream installer (piped to a root shell) when AUTO_INSTALL_UV=1 is set, and
+# pins the version so the fetched script is reproducible.
+UV_VERSION="${UV_VERSION:-0.5.11}"
+if ! command -v uv >/dev/null 2>&1; then
+    if [ "${AUTO_INSTALL_UV:-0}" = "1" ]; then
+        warn "uv not found; installing pinned uv ${UV_VERSION} from astral.sh (runs as root)"
+        curl -LsSf "https://astral.sh/uv/${UV_VERSION}/install.sh" \
+            | env UV_INSTALL_DIR=/usr/local/bin sh
+    else
+        die "uv not found. Install it (e.g. your package manager, or 'pipx install uv'),
+     or re-run with AUTO_INSTALL_UV=1 to fetch the pinned upstream installer."
+    fi
+fi
+UV="$(command -v uv)"
+
+log "Creating virtualenv at ${APP_DIR}/venv"
+cd "${APP_DIR}"
+# Named 'venv' (not uv's default .venv) to match the systemd unit's ExecStart.
+"${UV}" venv venv
+"${UV}" pip install --python venv/bin/python -r requirements.txt
+
+# 4. Environment file ----------------------------------------------------------
+if [ ! -f "${APP_DIR}/.env" ]; then
+    if [ -f "${REPO_DIR}/.env" ]; then
+        log "Copying .env from checkout"
+        cp "${REPO_DIR}/.env" "${APP_DIR}/.env"
+    else
+        warn "No .env found. Copy .env.example to ${APP_DIR}/.env and fill in"
+        warn "MATRIX_ACCESS_TOKEN / MATRIX_ROOM_ID and DB settings before starting."
+    fi
+fi
+
+# 5. Ownership and permissions -------------------------------------------------
+# Service user needs write access for logs / stations.json.
+log "Setting ownership to ${SERVICE_USER}:${SERVICE_GROUP}"
+chown -R "${SERVICE_USER}:${SERVICE_GROUP}" "${APP_DIR}"
+# Restrict traversal to root + the service user, and lock down the secrets file
+# (contains the Matrix token and DB credentials).
+chmod 0750 "${APP_DIR}"
+if [ -f "${APP_DIR}/.env" ]; then
+    chmod 0600 "${APP_DIR}/.env"
+fi
+
+# 6. Install and enable the systemd unit --------------------------------------
+log "Installing systemd unit"
+install -m 0644 "${SCRIPT_DIR}/${SERVICE_NAME}" "/etc/systemd/system/${SERVICE_NAME}"
+systemctl daemon-reload
+systemctl enable "${SERVICE_NAME}"
+
+log "Done."
+echo
+echo "Next steps:"
+echo "  sudo systemctl start ${SERVICE_NAME}"
+echo "  systemctl status ${SERVICE_NAME}"
+echo "  sudo journalctl -u ${SERVICE_NAME} -f"

scripts/load_test.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+"""Staged load / client-stress test for the Ping River Monitor API + dashboard.
+
+Simulates a realistic traffic mix (dashboard page loads, the API calls the
+dashboard itself makes, heavy history queries, external API consumers) at
+increasing concurrency stages, and reports throughput, latency percentiles,
+and errors per stage plus the slowest endpoints.
+
+Run against a LOCAL instance for full stress (never full-stress production —
+it hosts live flood monitoring):
+
+    python -m uvicorn src.web_api:app --port 8125   # separate shell
+    python scripts/load_test.py http://localhost:8125
+
+A gentle production baseline (low, fixed concurrency):
+
+    python scripts/load_test.py https://water.buildfor.life --gentle
+"""
+
+import argparse
+import random
+import statistics
+import threading
+import time
+from collections import Counter
+
+import requests
+
+# Weighted endpoint mix: dashboard session + API consumers
+ENDPOINTS = [
+    ("/", 10),
+    ("/measurements/latest?limit=500", 20),
+    ("/stations", 10),
+    ("/api/hii/rainfall/latest", 15),
+    ("/api/hii/waterlevel/latest", 15),
+    ("/forecast", 10),
+    ("/api/stats", 5),
+    ("/measurements/history/P.1?hours=168", 10),
+    ("/measurements/history/P.67?hours=720", 5),
+    ("/health", 5),
+]
+POOL = [endpoint for endpoint, weight in ENDPOINTS for _ in range(weight)]
+
+FULL_STAGES = [(10, 20), (50, 20), (200, 25)]  # (clients, seconds)
+GENTLE_STAGES = [(3, 15), (8, 15)]
+
+
+def _worker(base, stop_at, results, errors):
+    session = requests.Session()
+    while time.time() < stop_at:
+        path = random.choice(POOL)
+        start = time.perf_counter()
+        try:
+            response = session.get(f"{base}{path}", timeout=30)
+            elapsed = time.perf_counter() - start
+            if response.status_code == 200:
+                results.append((path, elapsed))
+            else:
+                errors.append((path, response.status_code))
+        except Exception as error:
+            errors.append((path, type(error).__name__))
+
+
+def _pct(values, p):
+    if len(values) >= 100:
+        return statistics.quantiles(values, n=100)[p - 1]
+    return max(values)
+
+
+def run_stage(base, clients, seconds):
+    results, errors = [], []
+    stop_at = time.time() + seconds
+    threads = [
+        threading.Thread(
+            target=_worker, args=(base, stop_at, results, errors), daemon=True
+        )
+        for _ in range(clients)
+    ]
+    for thread in threads:
+        thread.start()
+    for thread in threads:
+        thread.join(timeout=seconds + 35)
+
+    latencies = [elapsed for _, elapsed in results]
+    total = len(results) + len(errors)
+    print(f"\n== {clients} clients x {seconds}s ==")
+    print(
+        f"requests: {total}  ok: {len(results)}  errors: {len(errors)}  "
+        f"rps: {total / seconds:.1f}"
+    )
+    if latencies:
+        print(
+            f"latency ms  p50: {statistics.median(latencies) * 1000:.0f}  "
+            f"p95: {_pct(latencies, 95) * 1000:.0f}  "
+            f"p99: {_pct(latencies, 99) * 1000:.0f}  "
+            f"max: {max(latencies) * 1000:.0f}"
+        )
+    by_endpoint = {}
+    for path, elapsed in results:
+        by_endpoint.setdefault(path, []).append(elapsed)
+    slowest = sorted(
+        by_endpoint.items(), key=lambda kv: -statistics.median(kv[1])
+    )[:4]
+    for path, values in slowest:
+        print(
+            f"  slow: {path:45} n={len(values):5} "
+            f"p50={statistics.median(values) * 1000:6.0f}ms "
+            f"max={max(values) * 1000:7.0f}ms"
+        )
+    if errors:
+        top = Counter(f"{path} {code}" for path, code in errors).most_common(5)
+        print(f"  errors: {top}")
+    return {"clients": clients, "total": total, "errors": len(errors)}
+
+
+def main(argv=None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("base", nargs="?", default="http://localhost:8125")
+    parser.add_argument(
+        "--gentle",
+        action="store_true",
+        help="low fixed concurrency (safe for the production instance)",
+    )
+    args = parser.parse_args(argv)
+    base = args.base.rstrip("/")
+
+    # Warm caches first so stage 1 doesn't measure cold-start work
+    for path in ("/forecast", "/api/stats", "/measurements/latest?limit=500"):
+        try:
+            requests.get(f"{base}{path}", timeout=60)
+        except Exception:
+            pass
+
+    print(f"target: {base}  mode: {'gentle' if args.gentle else 'full'}")
+    stages = GENTLE_STAGES if args.gentle else FULL_STAGES
+    summary = [run_stage(base, clients, seconds) for clients, seconds in stages]
+    worst = max(
+        (stage["errors"] / stage["total"] for stage in summary if stage["total"]),
+        default=1.0,
+    )
+    print(f"\nworst-stage error rate: {worst:.1%}")
+    return 0 if worst < 0.05 else 1
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(main())

scripts/locustfile.py

@@ -0,0 +1,98 @@
+"""Locust load profile for the Ping River Monitor API + dashboard.
+
+Two user types mirror real traffic: dashboard visitors (page + the API calls
+the page makes, polling like the auto-refresh does) and API consumers
+(direct endpoint hits, including heavy history queries).
+
+Full stress against a LOCAL instance (never full-stress production — it hosts
+live flood monitoring):
+
+    # separate shell: python -m uvicorn src.web_api:app --port 8125
+    .venv/Scripts/python.exe -m locust -f scripts/locustfile.py \
+        --host http://localhost:8125 --headless \
+        --users 200 --spawn-rate 20 --run-time 2m \
+        --html load-report.html
+
+Interactive UI instead: drop --headless and open http://localhost:8089.
+"""
+
+import random
+
+from locust import FastHttpUser, between, task
+
+# Explicit so measurements reflect compressed transfer (browsers always send this)
+GZIP = {"Accept-Encoding": "gzip, deflate"}
+
+
+class DashboardVisitor(FastHttpUser):
+    """A browser session: initial page load, then periodic refresh polling."""
+
+    weight = 3
+    wait_time = between(2, 6)
+
+    def on_start(self):
+        # What one real page load requests
+        self.client.get("/", headers=GZIP)
+        self.client.get("/stations", headers=GZIP)
+        self.client.get("/measurements/latest?limit=500", headers=GZIP)
+        self.client.get("/api/hii/waterlevel/latest", headers=GZIP)
+        self.client.get("/api/hii/rainfall/latest", headers=GZIP)
+
+    @task(4)
+    def poll_latest(self):
+        self.client.get("/measurements/latest?limit=500", headers=GZIP)
+
+    @task(2)
+    def poll_forecast(self):
+        self.client.get("/forecast", headers=GZIP)
+
+    @task(2)
+    def poll_rain(self):
+        self.client.get("/api/hii/rainfall/latest", headers=GZIP)
+
+    @task(1)
+    def view_history(self):
+        station = random.choice(["P.1", "P.67", "P.103", "P.75", "P.20"])
+        hours = random.choice([24, 168, 720])
+        self.client.get(
+            f"/measurements/history/{station}?hours={hours}",
+            headers=GZIP,
+            name="/measurements/history/[station]",
+        )
+
+    @task(1)
+    def stats(self):
+        self.client.get("/api/stats", headers=GZIP)
+
+
+class ApiConsumer(FastHttpUser):
+    """A script/integration hitting the JSON API directly, no think time."""
+
+    weight = 1
+    wait_time = between(0.1, 1)
+
+    @task(3)
+    def latest(self):
+        self.client.get("/measurements/latest?limit=100", headers=GZIP)
+
+    @task(3)
+    def hii_feeds(self):
+        self.client.get(random.choice(
+            ["/api/hii/waterlevel/latest", "/api/hii/rainfall/latest"]
+        ), headers=GZIP, name="/api/hii/[feed]/latest")
+
+    @task(2)
+    def forecast(self):
+        self.client.get("/forecast", headers=GZIP)
+
+    @task(2)
+    def heavy_history(self):
+        self.client.get(
+            "/measurements/history/P.1?hours=8760",
+            headers=GZIP,
+            name="/measurements/history/P.1 [heavy]",
+        )
+
+    @task(1)
+    def health(self):
+        self.client.get("/health", headers=GZIP)

scripts/migrate_sqlite_to_postgres.py

@@ -0,0 +1,619 @@
+#!/usr/bin/env python3
+"""
+SQLite to PostgreSQL Migration Tool
+Migrates all data from SQLite database to PostgreSQL
+"""
+
+import os
+import sys
+import logging
+import sqlite3
+from datetime import datetime, timezone
+from typing import Dict, List, Optional, Tuple, Any
+from dataclasses import dataclass
+
+# Add src to path for imports
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src'))
+
+@dataclass
+class MigrationStats:
+    stations_migrated: int = 0
+    measurements_migrated: int = 0
+    errors: List[str] = None
+    start_time: Optional[datetime] = None
+    end_time: Optional[datetime] = None
+
+    def __post_init__(self):
+        if self.errors is None:
+            self.errors = []
+
+class SQLiteToPostgresMigrator:
+    def __init__(self, sqlite_path: str, postgres_config: Dict[str, Any]):
+        self.sqlite_path = sqlite_path
+        self.postgres_config = postgres_config
+        self.sqlite_conn = None
+        self.postgres_adapter = None
+        self.stats = MigrationStats()
+
+        # Setup logging with UTF-8 encoding
+        logging.basicConfig(
+            level=logging.INFO,
+            format='%(asctime)s - %(levelname)s - %(message)s',
+            handlers=[
+                logging.StreamHandler(),
+                logging.FileHandler('migration.log', encoding='utf-8')
+            ]
+        )
+        self.logger = logging.getLogger(__name__)
+
+    def connect_databases(self) -> bool:
+        """Connect to both SQLite and PostgreSQL databases"""
+        try:
+            # Connect to SQLite
+            if not os.path.exists(self.sqlite_path):
+                self.logger.error(f"SQLite database not found: {self.sqlite_path}")
+                return False
+
+            self.sqlite_conn = sqlite3.connect(self.sqlite_path)
+            self.sqlite_conn.row_factory = sqlite3.Row  # For dict-like access
+            self.logger.info(f"Connected to SQLite database: {self.sqlite_path}")
+
+            # Connect to PostgreSQL
+            from database_adapters import create_database_adapter
+            self.postgres_adapter = create_database_adapter(
+                self.postgres_config['type'],
+                connection_string=self.postgres_config['connection_string']
+            )
+
+            if not self.postgres_adapter.connect():
+                self.logger.error("Failed to connect to PostgreSQL")
+                return False
+
+            self.logger.info("Connected to PostgreSQL database")
+            return True
+
+        except Exception as e:
+            self.logger.error(f"Database connection error: {e}")
+            return False
+
+    def analyze_sqlite_schema(self) -> Dict[str, List[str]]:
+        """Analyze SQLite database structure"""
+        try:
+            cursor = self.sqlite_conn.cursor()
+
+            # Get all tables
+            cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'")
+            tables = [row[0] for row in cursor.fetchall()]
+
+            schema_info = {}
+            for table in tables:
+                cursor.execute(f"PRAGMA table_info({table})")
+                columns = [row[1] for row in cursor.fetchall()]
+                schema_info[table] = columns
+
+                # Get row count
+                cursor.execute(f"SELECT COUNT(*) FROM {table}")
+                count = cursor.fetchone()[0]
+                self.logger.info(f"Table '{table}': {len(columns)} columns, {count} rows")
+
+            return schema_info
+
+        except Exception as e:
+            self.logger.error(f"Schema analysis error: {e}")
+            return {}
+
+    def migrate_stations(self) -> bool:
+        """Migrate station data"""
+        try:
+            cursor = self.sqlite_conn.cursor()
+
+            # Try different possible table names and structures
+            station_queries = [
+                # Modern structure
+                """SELECT id, station_code, station_name_th as thai_name, station_name_en as english_name,
+                         latitude, longitude, geohash, created_at, updated_at
+                  FROM stations""",
+
+                # Alternative structure 1
+                """SELECT id, station_code, thai_name, english_name,
+                         latitude, longitude, geohash, created_at, updated_at
+                  FROM stations""",
+
+                # Legacy structure
+                """SELECT station_id as id, station_code, station_name as thai_name,
+                         station_name as english_name, lat as latitude, lon as longitude,
+                         NULL as geohash, datetime('now') as created_at, datetime('now') as updated_at
+                  FROM water_stations""",
+
+                # Simple structure
+                """SELECT rowid as id, station_code, name as thai_name, name as english_name,
+                         NULL as latitude, NULL as longitude, NULL as geohash,
+                         datetime('now') as created_at, datetime('now') as updated_at
+                  FROM stations""",
+            ]
+
+            stations_data = []
+
+            for query in station_queries:
+                try:
+                    cursor.execute(query)
+                    rows = cursor.fetchall()
+                    if rows:
+                        self.logger.info(f"Found {len(rows)} stations using query variant")
+
+                        for row in rows:
+                            station = {
+                                'station_id': row[0],
+                                'station_code': row[1] or f"STATION_{row[0]}",
+                                'station_name_th': row[2] or f"Station {row[0]}",
+                                'station_name_en': row[3] or f"Station {row[0]}",
+                                'latitude': row[4],
+                                'longitude': row[5],
+                                'geohash': row[6],
+                                'status': 'active'
+                            }
+                            stations_data.append(station)
+                        break
+
+                except sqlite3.OperationalError as e:
+                    if "no such table" in str(e).lower() or "no such column" in str(e).lower():
+                        continue
+                    else:
+                        raise
+
+            if not stations_data:
+                self.logger.warning("No stations found in SQLite database")
+                return True
+
+            # Insert stations into PostgreSQL using raw SQL
+            # Since the adapter is designed for measurements, we'll use direct SQL
+            try:
+                from sqlalchemy import create_engine, text
+                engine = create_engine(self.postgres_config['connection_string'])
+
+                # Process stations individually to avoid transaction rollback issues
+                for station in stations_data:
+                    try:
+                        with engine.begin() as conn:
+                            # Use PostgreSQL UPSERT syntax with correct column names
+                            station_sql = """
+                            INSERT INTO stations (id, station_code, thai_name, english_name, latitude, longitude, geohash)
+                            VALUES (:station_id, :station_code, :thai_name, :english_name, :latitude, :longitude, :geohash)
+                            ON CONFLICT (id) DO UPDATE SET
+                                thai_name = EXCLUDED.thai_name,
+                                english_name = EXCLUDED.english_name,
+                                latitude = EXCLUDED.latitude,
+                                longitude = EXCLUDED.longitude,
+                                geohash = EXCLUDED.geohash,
+                                updated_at = CURRENT_TIMESTAMP
+                            """
+
+                            conn.execute(text(station_sql), {
+                                'station_id': station['station_id'],
+                                'station_code': station['station_code'],
+                                'thai_name': station['station_name_th'],
+                                'english_name': station['station_name_en'],
+                                'latitude': station.get('latitude'),
+                                'longitude': station.get('longitude'),
+                                'geohash': station.get('geohash')
+                            })
+
+                            self.stats.stations_migrated += 1
+
+                    except Exception as e:
+                        error_msg = f"Error migrating station {station.get('station_code', 'unknown')}: {str(e)[:100]}..."
+                        self.logger.warning(error_msg)
+                        self.stats.errors.append(error_msg)
+
+                self.logger.info(f"Migrated {self.stats.stations_migrated} stations")
+
+            except Exception as e:
+                self.logger.error(f"Station migration failed: {e}")
+                return False
+
+            self.logger.info(f"Migrated {self.stats.stations_migrated} stations")
+            return True
+
+        except Exception as e:
+            self.logger.error(f"Station migration error: {e}")
+            return False
+
+    def migrate_measurements(self, batch_size: int = 5000) -> bool:
+        """Migrate measurement data in batches"""
+        try:
+            cursor = self.sqlite_conn.cursor()
+
+            # Try different possible measurement table structures
+            measurement_queries = [
+                # Modern structure
+                """SELECT w.timestamp, w.station_id, s.station_code, s.station_name_th, s.station_name_en,
+                         w.water_level, w.discharge, w.discharge_percent, w.status
+                  FROM water_measurements w
+                  JOIN stations s ON w.station_id = s.id
+                  ORDER BY w.timestamp""",
+
+                # Alternative with different join
+                """SELECT w.timestamp, w.station_id, s.station_code, s.thai_name, s.english_name,
+                         w.water_level, w.discharge, w.discharge_percent, 'active' as status
+                  FROM water_measurements w
+                  JOIN stations s ON w.station_id = s.id
+                  ORDER BY w.timestamp""",
+
+                # Legacy structure
+                """SELECT timestamp, station_id, station_code, station_name, station_name,
+                         water_level, discharge, discharge_percent, 'active' as status
+                  FROM measurements
+                  ORDER BY timestamp""",
+
+                # Simple structure without joins
+                """SELECT timestamp, station_id, 'UNKNOWN' as station_code, 'Unknown' as station_name_th, 'Unknown' as station_name_en,
+                         water_level, discharge, discharge_percent, 'active' as status
+                  FROM water_measurements
+                  ORDER BY timestamp""",
+            ]
+
+            measurements_processed = 0
+
+            for query in measurement_queries:
+                try:
+                    # Get total count first
+                    count_query = query.replace("SELECT", "SELECT COUNT(*) FROM (SELECT").replace("ORDER BY w.timestamp", "") + ")"
+                    cursor.execute(count_query)
+                    total_measurements = cursor.fetchone()[0]
+
+                    if total_measurements == 0:
+                        continue
+
+                    self.logger.info(f"Found {total_measurements} measurements to migrate")
+
+                    # Process in batches
+                    offset = 0
+                    while True:
+                        batch_query = f"{query} LIMIT {batch_size} OFFSET {offset}"
+                        cursor.execute(batch_query)
+                        rows = cursor.fetchall()
+
+                        if not rows:
+                            break
+
+                        # Convert to measurement format
+                        measurements = []
+                        for row in rows:
+                            try:
+                                # Parse timestamp
+                                timestamp_str = row[0]
+                                if isinstance(timestamp_str, str):
+                                    try:
+                                        timestamp = datetime.fromisoformat(timestamp_str.replace('Z', '+00:00'))
+                                    except:
+                                        # Try other common formats
+                                        for fmt in ['%Y-%m-%d %H:%M:%S', '%Y-%m-%d %H:%M:%S.%f', '%Y-%m-%dT%H:%M:%S']:
+                                            try:
+                                                timestamp = datetime.strptime(timestamp_str, fmt)
+                                                break
+                                            except:
+                                                continue
+                                        else:
+                                            timestamp = datetime.now()
+                                else:
+                                    timestamp = timestamp_str
+
+                                measurement = {
+                                    'timestamp': timestamp,
+                                    'station_id': row[1] or 999,
+                                    'station_code': row[2] or 'UNKNOWN',
+                                    'station_name_th': row[3] or 'Unknown',
+                                    'station_name_en': row[4] or 'Unknown',
+                                    'water_level': float(row[5]) if row[5] is not None else None,
+                                    'discharge': float(row[6]) if row[6] is not None else None,
+                                    'discharge_percent': float(row[7]) if row[7] is not None else None,
+                                    'status': row[8] or 'active'
+                                }
+                                measurements.append(measurement)
+
+                            except Exception as e:
+                                error_msg = f"Error processing measurement row: {e}"
+                                self.logger.warning(error_msg)
+                                continue
+
+                        # Save batch to PostgreSQL using fast bulk insert
+                        if measurements:
+                            try:
+                                self._fast_bulk_insert(measurements)
+                                measurements_processed += len(measurements)
+                                self.stats.measurements_migrated += len(measurements)
+                                self.logger.info(f"Migrated {measurements_processed}/{total_measurements} measurements")
+                            except Exception as e:
+                                error_msg = f"Error saving measurement batch: {e}"
+                                self.logger.error(error_msg)
+                                self.stats.errors.append(error_msg)
+
+                        offset += batch_size
+
+                    # If we processed measurements, we're done
+                    if measurements_processed > 0:
+                        break
+
+                except sqlite3.OperationalError as e:
+                    if "no such table" in str(e).lower() or "no such column" in str(e).lower():
+                        continue
+                    else:
+                        raise
+
+            if measurements_processed == 0:
+                self.logger.warning("No measurements found in SQLite database")
+            else:
+                self.logger.info(f"Successfully migrated {measurements_processed} measurements")
+
+            return True
+
+        except Exception as e:
+            self.logger.error(f"Measurement migration error: {e}")
+            return False
+
+    def _fast_bulk_insert(self, measurements: List[Dict]) -> bool:
+        """Super fast bulk insert using PostgreSQL COPY or VALUES clause"""
+        try:
+            import psycopg2
+            from urllib.parse import urlparse
+            import io
+
+            # Parse connection string for direct psycopg2 connection
+            parsed = urlparse(self.postgres_config['connection_string'])
+
+            # Try super fast COPY method first
+            try:
+                conn = psycopg2.connect(
+                    host=parsed.hostname,
+                    port=parsed.port or 5432,
+                    database=parsed.path[1:],
+                    user=parsed.username,
+                    password=parsed.password
+                )
+
+                with conn:
+                    with conn.cursor() as cur:
+                        # Prepare data for COPY
+                        data_buffer = io.StringIO()
+                        null_val = '\\N'
+                        for m in measurements:
+                            data_buffer.write(f"{m['timestamp']}\t{m['station_id']}\t{m['water_level'] or null_val}\t{m['discharge'] or null_val}\t{m['discharge_percent'] or null_val}\t{m['status']}\n")
+
+                        data_buffer.seek(0)
+
+                        # Use COPY for maximum speed
+                        cur.copy_from(
+                            data_buffer,
+                            'water_measurements',
+                            columns=('timestamp', 'station_id', 'water_level', 'discharge', 'discharge_percent', 'status'),
+                            sep='\t'
+                        )
+
+                conn.close()
+                return True
+
+            except Exception as copy_error:
+                # Fallback to SQLAlchemy bulk insert
+                self.logger.debug(f"COPY failed, using bulk VALUES: {copy_error}")
+
+                from sqlalchemy import create_engine, text
+                engine = create_engine(self.postgres_config['connection_string'])
+
+                with engine.begin() as conn:
+                    # Use PostgreSQL's fast bulk insert with ON CONFLICT
+                    values_list = []
+                    for m in measurements:
+                        timestamp = m['timestamp'].isoformat() if hasattr(m['timestamp'], 'isoformat') else str(m['timestamp'])
+                        values_list.append(
+                            f"('{timestamp}', {m['station_id']}, {m['water_level'] or 'NULL'}, "
+                            f"{m['discharge'] or 'NULL'}, {m['discharge_percent'] or 'NULL'}, '{m['status']}')"
+                        )
+
+                    # Build bulk insert query with ON CONFLICT handling
+                    bulk_sql = f"""
+                    INSERT INTO water_measurements (timestamp, station_id, water_level, discharge, discharge_percent, status)
+                    VALUES {','.join(values_list)}
+                    ON CONFLICT (timestamp, station_id) DO UPDATE SET
+                        water_level = EXCLUDED.water_level,
+                        discharge = EXCLUDED.discharge,
+                        discharge_percent = EXCLUDED.discharge_percent,
+                        status = EXCLUDED.status
+                    """
+
+                    conn.execute(text(bulk_sql))
+                    return True
+
+        except Exception as e:
+            self.logger.warning(f"Fast bulk insert failed: {e}")
+            # Final fallback to original method
+            try:
+                success = self.postgres_adapter.save_measurements(measurements)
+                return success
+            except Exception as fallback_e:
+                self.logger.error(f"All insert methods failed: {fallback_e}")
+                return False
+
+    def verify_migration(self) -> bool:
+        """Verify the migration by comparing counts"""
+        try:
+            # Get SQLite counts
+            cursor = self.sqlite_conn.cursor()
+
+            sqlite_stations = 0
+            sqlite_measurements = 0
+
+            # Try to get station count
+            for table in ['stations', 'water_stations']:
+                try:
+                    cursor.execute(f"SELECT COUNT(*) FROM {table}")
+                    sqlite_stations = cursor.fetchone()[0]
+                    break
+                except:
+                    continue
+
+            # Try to get measurement count
+            for table in ['water_measurements', 'measurements']:
+                try:
+                    cursor.execute(f"SELECT COUNT(*) FROM {table}")
+                    sqlite_measurements = cursor.fetchone()[0]
+                    break
+                except:
+                    continue
+
+            # Get PostgreSQL counts
+            postgres_measurements = self.postgres_adapter.get_latest_measurements(limit=999999)
+            postgres_count = len(postgres_measurements)
+
+            self.logger.info("Migration Verification:")
+            self.logger.info(f"SQLite stations: {sqlite_stations}")
+            self.logger.info(f"SQLite measurements: {sqlite_measurements}")
+            self.logger.info(f"PostgreSQL measurements retrieved: {postgres_count}")
+            self.logger.info(f"Migrated stations: {self.stats.stations_migrated}")
+            self.logger.info(f"Migrated measurements: {self.stats.measurements_migrated}")
+
+            return True
+
+        except Exception as e:
+            self.logger.error(f"Verification error: {e}")
+            return False
+
+    def run_migration(self, sqlite_path: str = None) -> bool:
+        """Run the complete migration process"""
+        self.stats.start_time = datetime.now()
+
+        if sqlite_path:
+            self.sqlite_path = sqlite_path
+
+        self.logger.info("=" * 60)
+        self.logger.info("SQLite to PostgreSQL Migration Tool")
+        self.logger.info("=" * 60)
+        self.logger.info(f"SQLite database: {self.sqlite_path}")
+        self.logger.info(f"PostgreSQL: {self.postgres_config['type']}")
+
+        try:
+            # Step 1: Connect to databases
+            self.logger.info("Step 1: Connecting to databases...")
+            if not self.connect_databases():
+                return False
+
+            # Step 2: Analyze SQLite schema
+            self.logger.info("Step 2: Analyzing SQLite database structure...")
+            schema_info = self.analyze_sqlite_schema()
+            if not schema_info:
+                self.logger.error("Could not analyze SQLite database structure")
+                return False
+
+            # Step 3: Migrate stations
+            self.logger.info("Step 3: Migrating station data...")
+            if not self.migrate_stations():
+                self.logger.error("Station migration failed")
+                return False
+
+            # Step 4: Migrate measurements
+            self.logger.info("Step 4: Migrating measurement data...")
+            if not self.migrate_measurements():
+                self.logger.error("Measurement migration failed")
+                return False
+
+            # Step 5: Verify migration
+            self.logger.info("Step 5: Verifying migration...")
+            self.verify_migration()
+
+            self.stats.end_time = datetime.now()
+            duration = self.stats.end_time - self.stats.start_time
+
+            # Final report
+            self.logger.info("=" * 60)
+            self.logger.info("MIGRATION COMPLETED")
+            self.logger.info("=" * 60)
+            self.logger.info(f"Duration: {duration}")
+            self.logger.info(f"Stations migrated: {self.stats.stations_migrated}")
+            self.logger.info(f"Measurements migrated: {self.stats.measurements_migrated}")
+
+            if self.stats.errors:
+                self.logger.warning(f"Errors encountered: {len(self.stats.errors)}")
+                for error in self.stats.errors[:10]:  # Show first 10 errors
+                    self.logger.warning(f"  - {error}")
+                if len(self.stats.errors) > 10:
+                    self.logger.warning(f"  ... and {len(self.stats.errors) - 10} more errors")
+            else:
+                self.logger.info("No errors encountered")
+
+            return True
+
+        except Exception as e:
+            self.logger.error(f"Migration failed: {e}")
+            return False
+
+        finally:
+            # Cleanup
+            if self.sqlite_conn:
+                self.sqlite_conn.close()
+
+def main():
+    """Main entry point"""
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Migrate SQLite data to PostgreSQL")
+    parser.add_argument("sqlite_path", nargs="?", help="Path to SQLite database file")
+    parser.add_argument("--batch-size", type=int, default=5000, help="Batch size for processing measurements")
+    parser.add_argument("--fast", action="store_true", help="Use maximum speed mode (batch-size 10000)")
+    parser.add_argument("--dry-run", action="store_true", help="Analyze only, don't migrate")
+
+    args = parser.parse_args()
+
+    # Set fast mode
+    if args.fast:
+        args.batch_size = 10000
+
+    # Get SQLite path
+    sqlite_path = args.sqlite_path
+    if not sqlite_path:
+        # Try to find common SQLite database files
+        possible_paths = [
+            "water_levels.db",
+            "water_monitoring.db",
+            "database.db",
+            "../water_levels.db"
+        ]
+
+        for path in possible_paths:
+            if os.path.exists(path):
+                sqlite_path = path
+                break
+
+        if not sqlite_path:
+            print("SQLite database file not found. Please specify the path:")
+            print("  python migrate_sqlite_to_postgres.py /path/to/database.db")
+            return False
+
+    # Get PostgreSQL configuration
+    try:
+        from config import Config
+        postgres_config = Config.get_database_config()
+
+        if postgres_config['type'] != 'postgresql':
+            print("Error: PostgreSQL not configured. Set DB_TYPE=postgresql in your .env file")
+            return False
+
+    except Exception as e:
+        print(f"Error loading PostgreSQL configuration: {e}")
+        return False
+
+    # Run migration
+    migrator = SQLiteToPostgresMigrator(sqlite_path, postgres_config)
+
+    if args.dry_run:
+        print("DRY RUN MODE - Analyzing SQLite database structure only")
+        if migrator.connect_databases():
+            schema_info = migrator.analyze_sqlite_schema()
+            print("\nSQLite database structure analysis complete.")
+            print("Run without --dry-run to perform the actual migration.")
+        return True
+
+    success = migrator.run_migration()
+    return success
+
+if __name__ == "__main__":
+    success = main()
+    sys.exit(0 if success else 1)

scripts/setup_postgres.py

@@ -0,0 +1,175 @@
+#!/usr/bin/env python3
+"""
+PostgreSQL setup script for Northern Thailand Ping River Monitor
+This script helps you configure and test your PostgreSQL connection
+"""
+
+import os
+import sys
+import logging
+from typing import Optional
+from urllib.parse import urlparse
+
+def setup_logging():
+    logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
+
+def test_postgres_connection(connection_string: str) -> bool:
+    """Test connection to PostgreSQL database"""
+    try:
+        from sqlalchemy import create_engine, text
+
+        # Test connection
+        engine = create_engine(connection_string, pool_pre_ping=True)
+        with engine.connect() as conn:
+            result = conn.execute(text("SELECT version()"))
+            version = result.fetchone()[0]
+            logging.info(f"✅ Connected to PostgreSQL successfully!")
+            logging.info(f"Database version: {version}")
+            return True
+
+    except ImportError:
+        logging.error("❌ psycopg2-binary not installed. Run: uv add psycopg2-binary")
+        return False
+    except Exception as e:
+        logging.error(f"❌ Connection failed: {e}")
+        return False
+
+def parse_connection_string(connection_string: str) -> dict:
+    """Parse PostgreSQL connection string into components"""
+    try:
+        parsed = urlparse(connection_string)
+        return {
+            'host': parsed.hostname,
+            'port': parsed.port or 5432,
+            'database': parsed.path[1:] if parsed.path else None,
+            'username': parsed.username,
+            'password': parsed.password,
+        }
+    except Exception as e:
+        logging.error(f"Failed to parse connection string: {e}")
+        return {}
+
+def create_database_if_not_exists(connection_string: str, database_name: str) -> bool:
+    """Create database if it doesn't exist"""
+    try:
+        from sqlalchemy import create_engine, text
+
+        # Connect to default postgres database to create our database
+        parsed = urlparse(connection_string)
+        admin_connection = connection_string.replace(f"/{parsed.path[1:]}", "/postgres")
+
+        engine = create_engine(admin_connection, pool_pre_ping=True)
+
+        with engine.connect() as conn:
+            # Check if database exists
+            result = conn.execute(text(
+                "SELECT 1 FROM pg_database WHERE datname = :db_name"
+            ), {"db_name": database_name})
+
+            if result.fetchone():
+                logging.info(f"✅ Database '{database_name}' already exists")
+                return True
+            else:
+                # Create database
+                conn.execute(text("COMMIT"))  # End transaction
+                conn.execute(text(f'CREATE DATABASE "{database_name}"'))
+                logging.info(f"✅ Created database '{database_name}'")
+                return True
+
+    except Exception as e:
+        logging.error(f"❌ Failed to create database: {e}")
+        return False
+
+def initialize_tables(connection_string: str) -> bool:
+    """Initialize database tables"""
+    try:
+        # Import the database adapter to create tables
+        sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src'))
+        from database_adapters import SQLAdapter
+
+        adapter = SQLAdapter(connection_string=connection_string, db_type='postgresql')
+        if adapter.connect():
+            logging.info("✅ Database tables initialized successfully")
+            return True
+        else:
+            logging.error("❌ Failed to initialize tables")
+            return False
+
+    except Exception as e:
+        logging.error(f"❌ Failed to initialize tables: {e}")
+        return False
+
+def interactive_setup():
+    """Interactive setup wizard"""
+    print("🐘 PostgreSQL Setup Wizard for Ping River Monitor")
+    print("=" * 50)
+
+    # Get connection details
+    host = input("PostgreSQL host (e.g., 192.168.1.100): ").strip()
+    port = input("PostgreSQL port [5432]: ").strip() or "5432"
+    database = input("Database name [water_monitoring]: ").strip() or "water_monitoring"
+    username = input("Username: ").strip()
+    password = input("Password: ").strip()
+
+    # Optional SSL
+    use_ssl = input("Use SSL connection? (y/N): ").strip().lower() == 'y'
+    ssl_params = "?sslmode=require" if use_ssl else ""
+
+    connection_string = f"postgresql://{username}:{password}@{host}:{port}/{database}{ssl_params}"
+
+    print(f"\nGenerated connection string:")
+    print(f"POSTGRES_CONNECTION_STRING={connection_string}")
+
+    return connection_string
+
+def main():
+    setup_logging()
+
+    print("🚀 Northern Thailand Ping River Monitor - PostgreSQL Setup")
+    print("=" * 60)
+
+    # Check if connection string is provided via environment
+    connection_string = os.getenv('POSTGRES_CONNECTION_STRING')
+
+    if not connection_string:
+        print("No POSTGRES_CONNECTION_STRING found in environment.")
+        print("Starting interactive setup...\n")
+        connection_string = interactive_setup()
+
+        # Suggest adding to .env file
+        print(f"\n💡 Add this to your .env file:")
+        print(f"DB_TYPE=postgresql")
+        print(f"POSTGRES_CONNECTION_STRING={connection_string}")
+
+    # Parse connection details
+    config = parse_connection_string(connection_string)
+    if not config.get('host'):
+        logging.error("Invalid connection string format")
+        return False
+
+    print(f"\n🔗 Connecting to PostgreSQL at {config['host']}:{config['port']}")
+
+    # Test connection
+    if not test_postgres_connection(connection_string):
+        return False
+
+    # Try to create database
+    database_name = config.get('database', 'water_monitoring')
+    if database_name:
+        create_database_if_not_exists(connection_string, database_name)
+
+    # Initialize tables
+    if not initialize_tables(connection_string):
+        return False
+
+    print("\n🎉 PostgreSQL setup completed successfully!")
+    print("\nNext steps:")
+    print("1. Update your .env file with the connection string")
+    print("2. Run: make run-test")
+    print("3. Run: make run-api")
+
+    return True
+
+if __name__ == "__main__":
+    success = main()
+    sys.exit(0 if success else 1)

scripts/setup_uv.bat

@@ -0,0 +1,48 @@
+@echo off
+REM Setup script for uv-based development environment on Windows
+
+echo 🚀 Setting up Northern Thailand Ping River Monitor with uv...
+
+REM Check if uv is installed
+uv --version >nul 2>&1
+if %errorlevel% neq 0 (
+    echo ❌ uv is not installed. Please install it first:
+    echo    powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"
+    exit /b 1
+)
+
+echo ✅ uv found
+uv --version
+
+REM Initialize uv project if not already initialized
+if not exist "uv.lock" (
+    echo 🔧 Initializing uv project...
+    uv sync
+) else (
+    echo 📦 Syncing dependencies with uv...
+    uv sync
+)
+
+REM Install pre-commit hooks
+echo 🎣 Installing pre-commit hooks...
+uv run pre-commit install
+
+REM Create .env file if it doesn't exist
+if not exist ".env" (
+    if exist ".env.example" (
+        echo 📝 Creating .env file from template...
+        copy .env.example .env
+        echo ⚠️  Please edit .env file with your configuration
+    )
+)
+
+echo ✅ Setup complete!
+echo.
+echo 📚 Quick start commands:
+echo   make install-dev  # Install all dependencies
+echo   make run-test     # Run a test cycle
+echo   make run-api      # Start the web API
+echo   make test         # Run tests
+echo   make lint         # Check code quality
+echo.
+echo 🎉 Happy monitoring!

scripts/setup_uv.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+# Setup script for uv-based development environment
+
+set -e
+
+echo "🚀 Setting up Northern Thailand Ping River Monitor with uv..."
+
+# Check if uv is installed
+if ! command -v uv &> /dev/null; then
+    echo "❌ uv is not installed. Please install it first:"
+    echo "   curl -LsSf https://astral.sh/uv/install.sh | sh"
+    exit 1
+fi
+
+echo "✅ uv found: $(uv --version)"
+
+# Initialize uv project if not already initialized
+if [ ! -f "uv.lock" ]; then
+    echo "🔧 Initializing uv project..."
+    uv sync
+else
+    echo "📦 Syncing dependencies with uv..."
+    uv sync
+fi
+
+# Install pre-commit hooks
+echo "🎣 Installing pre-commit hooks..."
+uv run pre-commit install
+
+# Create .env file if it doesn't exist
+if [ ! -f ".env" ] && [ -f ".env.example" ]; then
+    echo "📝 Creating .env file from template..."
+    cp .env.example .env
+    echo "⚠️  Please edit .env file with your configuration"
+fi
+
+echo "✅ Setup complete!"
+echo ""
+echo "📚 Quick start commands:"
+echo "  make install-dev  # Install all dependencies"
+echo "  make run-test     # Run a test cycle"
+echo "  make run-api      # Start the web API"
+echo "  make test         # Run tests"
+echo "  make lint         # Check code quality"
+echo ""
+echo "🎉 Happy monitoring!"

scripts/train_flood_model.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+"""CLI entry point for training the Ping River flood forecast models.
+
+Usage:
+    python scripts/train_flood_model.py --stations all
+    python scripts/train_flood_model.py --stations P.1,P.103 --skip-eval
+"""
+
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+from src.ml.train import main
+
+if __name__ == "__main__":
+    main()

scripts/water-monitor.service

@@ -1,6 +1,6 @@
 [Unit]
 Description=Thailand Water Level Monitor
-Documentation=https://github.com/your-username/thailand-water-monitor
+Documentation=https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor
 After=network.target
 Wants=network-online.target
 

sql/init_postgres.sql

@@ -0,0 +1,162 @@
+-- Northern Thailand Ping River Monitor - PostgreSQL Database Schema
+-- This script initializes the database tables for water monitoring data
+
+-- Enable required extensions
+CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
+
+-- Create schema for better organization
+CREATE SCHEMA IF NOT EXISTS water_monitor;
+SET search_path TO water_monitor, public;
+
+-- Stations table - stores monitoring station information
+CREATE TABLE IF NOT EXISTS stations (
+    id SERIAL PRIMARY KEY,
+    station_code VARCHAR(10) UNIQUE NOT NULL,
+    thai_name VARCHAR(255) NOT NULL,
+    english_name VARCHAR(255) NOT NULL,
+    latitude DECIMAL(10,8),
+    longitude DECIMAL(11,8),
+    geohash VARCHAR(20),
+    elevation DECIMAL(8,2), -- meters above sea level
+    river_basin VARCHAR(100),
+    province VARCHAR(100),
+    district VARCHAR(100),
+    is_active BOOLEAN DEFAULT true,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+);
+
+-- Water measurements table - stores time series data
+CREATE TABLE IF NOT EXISTS water_measurements (
+    id BIGSERIAL PRIMARY KEY,
+    timestamp TIMESTAMP NOT NULL,
+    station_id INTEGER NOT NULL,
+    water_level NUMERIC(10,3), -- meters
+    discharge NUMERIC(10,2), -- cubic meters per second
+    discharge_percent NUMERIC(5,2), -- percentage of normal discharge
+    status VARCHAR(20) DEFAULT 'active',
+    data_quality VARCHAR(20) DEFAULT 'good', -- good, fair, poor, missing
+    remarks TEXT,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    FOREIGN KEY (station_id) REFERENCES stations(id) ON DELETE CASCADE,
+    UNIQUE(timestamp, station_id)
+);
+
+-- Alert thresholds table - stores warning/danger levels for each station
+CREATE TABLE IF NOT EXISTS alert_thresholds (
+    id SERIAL PRIMARY KEY,
+    station_id INTEGER NOT NULL,
+    threshold_type VARCHAR(20) NOT NULL, -- 'warning', 'danger', 'critical'
+    water_level_min NUMERIC(10,3),
+    water_level_max NUMERIC(10,3),
+    discharge_min NUMERIC(10,2),
+    discharge_max NUMERIC(10,2),
+    is_active BOOLEAN DEFAULT true,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    FOREIGN KEY (station_id) REFERENCES stations(id) ON DELETE CASCADE
+);
+
+-- Data quality log - tracks data collection issues
+CREATE TABLE IF NOT EXISTS data_quality_log (
+    id BIGSERIAL PRIMARY KEY,
+    timestamp TIMESTAMP NOT NULL,
+    station_id INTEGER,
+    issue_type VARCHAR(50) NOT NULL, -- 'connection_failed', 'invalid_data', 'missing_data'
+    description TEXT,
+    severity VARCHAR(20) DEFAULT 'info', -- 'info', 'warning', 'error', 'critical'
+    resolved_at TIMESTAMP,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    FOREIGN KEY (station_id) REFERENCES stations(id) ON DELETE SET NULL
+);
+
+-- Create indexes for better query performance
+CREATE INDEX IF NOT EXISTS idx_water_measurements_timestamp ON water_measurements(timestamp DESC);
+CREATE INDEX IF NOT EXISTS idx_water_measurements_station_id ON water_measurements(station_id);
+CREATE INDEX IF NOT EXISTS idx_water_measurements_station_timestamp ON water_measurements(station_id, timestamp DESC);
+CREATE INDEX IF NOT EXISTS idx_water_measurements_status ON water_measurements(status);
+CREATE INDEX IF NOT EXISTS idx_stations_code ON stations(station_code);
+CREATE INDEX IF NOT EXISTS idx_stations_active ON stations(is_active);
+CREATE INDEX IF NOT EXISTS idx_data_quality_timestamp ON data_quality_log(timestamp DESC);
+CREATE INDEX IF NOT EXISTS idx_data_quality_station ON data_quality_log(station_id);
+
+-- Create a view for latest measurements per station
+CREATE OR REPLACE VIEW latest_measurements AS
+SELECT
+    s.id as station_id,
+    s.station_code,
+    s.english_name,
+    s.thai_name,
+    s.latitude,
+    s.longitude,
+    s.province,
+    s.river_basin,
+    m.timestamp,
+    m.water_level,
+    m.discharge,
+    m.discharge_percent,
+    m.status,
+    m.data_quality,
+    CASE
+        WHEN m.timestamp > CURRENT_TIMESTAMP - INTERVAL '2 hours' THEN 'online'
+        WHEN m.timestamp > CURRENT_TIMESTAMP - INTERVAL '24 hours' THEN 'delayed'
+        ELSE 'offline'
+    END as station_status
+FROM stations s
+LEFT JOIN LATERAL (
+    SELECT * FROM water_measurements
+    WHERE station_id = s.id
+    ORDER BY timestamp DESC
+    LIMIT 1
+) m ON true
+WHERE s.is_active = true
+ORDER BY s.station_code;
+
+-- Create a function to update the updated_at timestamp
+CREATE OR REPLACE FUNCTION update_modified_column()
+RETURNS TRIGGER AS $$
+BEGIN
+    NEW.updated_at = CURRENT_TIMESTAMP;
+    RETURN NEW;
+END;
+$$ language 'plpgsql';
+
+-- Create triggers to automatically update updated_at
+DROP TRIGGER IF EXISTS update_stations_modtime ON stations;
+CREATE TRIGGER update_stations_modtime
+    BEFORE UPDATE ON stations
+    FOR EACH ROW
+    EXECUTE FUNCTION update_modified_column();
+
+-- Insert sample stations (Northern Thailand Ping River stations)
+INSERT INTO stations (id, station_code, thai_name, english_name, latitude, longitude, province, river_basin) VALUES
+(1, 'P.1', 'เชียงใหม่', 'Chiang Mai', 18.7883, 98.9853, 'Chiang Mai', 'Ping River'),
+(2, 'P.4A', 'ท่าแพ', 'Tha Phae', 18.7875, 99.0045, 'Chiang Mai', 'Ping River'),
+(3, 'P.12', 'สันป่าตอง', 'San Pa Tong', 18.6167, 98.9500, 'Chiang Mai', 'Ping River'),
+(4, 'P.20', 'ลำพูน', 'Lamphun', 18.5737, 99.0081, 'Lamphun', 'Ping River'),
+(5, 'P.30', 'ลี้', 'Li', 17.4833, 99.3000, 'Lamphun', 'Ping River'),
+(6, 'P.35', 'ป่าซาง', 'Pa Sang', 18.5444, 98.9397, 'Lamphun', 'Ping River'),
+(7, 'P.67', 'ตาก', 'Tak', 16.8839, 99.1267, 'Tak', 'Ping River'),
+(8, 'P.75', 'สามเงา', 'Sam Ngao', 17.1019, 99.4644, 'Tak', 'Ping River')
+ON CONFLICT (id) DO NOTHING;
+
+-- Insert sample alert thresholds
+INSERT INTO alert_thresholds (station_id, threshold_type, water_level_min, water_level_max) VALUES
+(1, 'warning', 4.5, NULL),
+(1, 'danger', 6.0, NULL),
+(1, 'critical', 7.5, NULL),
+(2, 'warning', 4.0, NULL),
+(2, 'danger', 5.5, NULL),
+(2, 'critical', 7.0, NULL)
+ON CONFLICT DO NOTHING;
+
+-- Grant permissions (adjust as needed for your setup)
+GRANT USAGE ON SCHEMA water_monitor TO postgres;
+GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA water_monitor TO postgres;
+GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA water_monitor TO postgres;
+
+-- Optional: Create a read-only user for reporting
+-- CREATE USER water_monitor_readonly WITH PASSWORD 'readonly_password';
+-- GRANT USAGE ON SCHEMA water_monitor TO water_monitor_readonly;
+-- GRANT SELECT ON ALL TABLES IN SCHEMA water_monitor TO water_monitor_readonly;
+
+COMMIT;

src/__init__.py

@@ -10,29 +10,25 @@ __version__ = "3.1.3"
 __author__ = "Ping River Monitor Team"
 __description__ = "Northern Thailand Ping River Monitoring System"
 
-from .water_scraper_v3 import EnhancedWaterMonitorScraper
-from .database_adapters import create_database_adapter, DatabaseAdapter
 from .config import Config
-from .models import WaterMeasurement, StationInfo, DatabaseConfig
-from .exceptions import (
-    WaterMonitorException,
-    DatabaseConnectionError,
-    APIConnectionError,
-    DataValidationError,
-    ConfigurationError
-)
+from .database_adapters import DatabaseAdapter, create_database_adapter
+from .exceptions import (APIConnectionError, ConfigurationError,
+                         DatabaseConnectionError, DataValidationError,
+                         WaterMonitorException)
+from .models import DatabaseConfig, StationInfo, WaterMeasurement
+from .water_scraper_v3 import EnhancedWaterMonitorScraper
 
 __all__ = [
-    'EnhancedWaterMonitorScraper',
-    'create_database_adapter',
-    'DatabaseAdapter',
-    'Config',
-    'WaterMeasurement',
-    'StationInfo',
-    'DatabaseConfig',
-    'WaterMonitorException',
-    'DatabaseConnectionError',
-    'APIConnectionError',
-    'DataValidationError',
-    'ConfigurationError'
-]
+    "EnhancedWaterMonitorScraper",
+    "create_database_adapter",
+    "DatabaseAdapter",
+    "Config",
+    "WaterMeasurement",
+    "StationInfo",
+    "DatabaseConfig",
+    "WaterMonitorException",
+    "DatabaseConnectionError",
+    "APIConnectionError",
+    "DataValidationError",
+    "ConfigurationError",
+]

src/alerting.py

@@ -0,0 +1,602 @@
+#!/usr/bin/env python3
+"""
+Water Level Alerting System with Matrix Integration
+"""
+
+import datetime
+import html
+import os
+import re
+from dataclasses import dataclass
+from enum import Enum
+from typing import Dict, List, Optional
+
+import requests
+
+try:
+    from .config import Config
+    from .database_adapters import create_database_adapter
+    from .logging_config import get_logger
+except ImportError:
+    import logging
+
+    from config import Config
+    from database_adapters import create_database_adapter
+
+    def get_logger(name):
+        return logging.getLogger(name)
+
+
+logger = get_logger(__name__)
+
+
+_BOLD_RE = re.compile(r"\*\*(.+?)\*\*")
+_URL_RE = re.compile(r"(https?://[^\s<]+)")
+
+
+def markdown_to_matrix_html(text: str) -> str:
+    """Convert the small Markdown subset we emit into Matrix-compatible HTML.
+
+    Matrix clients do NOT render Markdown in the plain ``body`` field; formatting
+    only shows when an HTML ``formatted_body`` is sent alongside it. We only use
+    ``**bold**``, bare URLs and newlines, so a minimal converter is sufficient and
+    avoids adding a Markdown dependency.
+    """
+    # Escape HTML special chars first so station/message data can't inject markup.
+    result = html.escape(text, quote=False)
+    result = _BOLD_RE.sub(r"<strong>\1</strong>", result)
+    result = _URL_RE.sub(r'<a href="\1">\1</a>', result)
+    result = result.replace("\n", "<br/>")
+    return result
+
+
+def strip_markdown(text: str) -> str:
+    """Produce a clean plain-text fallback for the Matrix ``body`` field."""
+    return _BOLD_RE.sub(r"\1", text)
+
+
+class AlertLevel(Enum):
+    INFO = "info"
+    WARNING = "warning"
+    CRITICAL = "critical"
+    EMERGENCY = "emergency"
+
+
+@dataclass
+class WaterAlert:
+    station_code: str
+    station_name: str
+    alert_type: str
+    level: AlertLevel
+    water_level: float
+    threshold: float
+    discharge: Optional[float] = None
+    timestamp: Optional[datetime.datetime] = None
+    message: Optional[str] = None
+
+
+class MatrixNotifier:
+    def __init__(self, homeserver: str, access_token: str, room_id: str):
+        self.homeserver = homeserver.rstrip("/")
+        self.access_token = access_token
+        self.room_id = room_id
+        self.session = requests.Session()
+
+    def send_message(
+        self, message: str, msgtype: str = "m.text", markdown: bool = True
+    ) -> bool:
+        """Send a message to the Matrix room.
+
+        When ``markdown`` is True (default) the ``message`` is treated as Markdown:
+        a rendered HTML ``formatted_body`` is sent so clients show real formatting,
+        with a plain-text ``body`` fallback for clients that ignore HTML.
+        """
+        try:
+            # Add transaction ID to prevent duplicates
+            txn_id = datetime.datetime.now().strftime("%Y%m%d_%H%M%S_%f")
+            url = f"{self.homeserver}/_matrix/client/v3/rooms/{self.room_id}/send/m.room.message/{txn_id}"
+
+            headers = {
+                "Authorization": f"Bearer {self.access_token}",
+                "Content-Type": "application/json",
+            }
+
+            if markdown:
+                data = {
+                    "msgtype": msgtype,
+                    "body": strip_markdown(message),
+                    "format": "org.matrix.custom.html",
+                    "formatted_body": markdown_to_matrix_html(message),
+                }
+            else:
+                data = {"msgtype": msgtype, "body": message}
+
+            # Matrix API requires PUT when transaction ID is in the URL path
+            response = self.session.put(url, headers=headers, json=data, timeout=10)
+            response.raise_for_status()
+
+            logger.info(
+                f"Matrix message sent successfully: {response.json().get('event_id')}"
+            )
+            return True
+
+        except Exception as e:
+            logger.error(f"Failed to send Matrix message: {e}")
+            return False
+
+    def send_alert(self, alert: WaterAlert) -> bool:
+        """Send formatted water alert to Matrix"""
+        emoji_map = {
+            AlertLevel.INFO: "ℹ️",
+            AlertLevel.WARNING: "⚠️",
+            AlertLevel.CRITICAL: "🚨",
+            AlertLevel.EMERGENCY: "🆘",
+        }
+
+        emoji = emoji_map.get(alert.level, "📊")
+
+        message = f"""{emoji} **WATER LEVEL ALERT**
+
+**Station:** {alert.station_code} ({alert.station_name})
+**Alert Type:** {alert.alert_type}
+**Severity:** {alert.level.value.upper()}
+
+**Current Level:** {alert.water_level:.2f}m
+**Threshold:** {alert.threshold:.2f}m
+**Difference:** {(alert.water_level - alert.threshold):+.2f}m
+"""
+
+        if alert.discharge:
+            message += f"**Discharge:** {alert.discharge:.1f} cms\n"
+
+        if alert.timestamp:
+            message += f"**Time:** {alert.timestamp.strftime('%Y-%m-%d %H:%M:%S')}\n"
+
+        if alert.message:
+            message += f"\n**Details:** {alert.message}\n"
+
+        # Add live dashboard link
+        dashboard_url = os.getenv("ALERT_DASHBOARD_URL", "https://water.buildfor.life/")
+        message += f"\n📈 **View Dashboard:** {dashboard_url}"
+
+        return self.send_message(message)
+
+
+class WaterLevelAlertSystem:
+    # Stations upstream of Chiang Mai (and CNX itself) to monitor
+    UPSTREAM_STATIONS = {
+        "P.20",  # Ban Chiang Dao
+        "P.75",  # Ban Chai Lat
+        "P.92",  # Ban Muang Aut
+        "P.4A",  # Ban Mae Taeng
+        "P.67",  # Ban Tae
+        "P.21",  # Ban Rim Tai
+        "P.103",  # Ring Bridge 3
+        "P.1",  # Nawarat Bridge (Chiang Mai)
+    }
+
+    def __init__(self):
+        self.db_adapter = None
+        self.matrix_notifier = None
+        self.thresholds = self._load_thresholds()
+
+        # Matrix configuration from environment
+        matrix_homeserver = os.getenv("MATRIX_HOMESERVER", "https://matrix.org")
+        matrix_token = os.getenv("MATRIX_ACCESS_TOKEN")
+        matrix_room = os.getenv("MATRIX_ROOM_ID")
+
+        if matrix_token and matrix_room:
+            self.matrix_notifier = MatrixNotifier(
+                matrix_homeserver, matrix_token, matrix_room
+            )
+            logger.info("Matrix notifications enabled")
+        else:
+            logger.warning("Matrix configuration missing - notifications disabled")
+
+    def _load_thresholds(self) -> Dict[str, Dict[str, float]]:
+        """Load alert thresholds from config or database"""
+        # Default thresholds for Northern Thailand stations
+        return {
+            "P.1": {
+                # Zone-based thresholds for Nawarat Bridge (P.1)
+                "zone_1": 3.7,
+                "zone_2": 3.9,
+                "zone_3": 4.0,
+                "zone_4": 4.1,
+                "zone_5": 4.2,
+                "zone_6": 4.3,
+                "zone_7": 4.6,
+                "zone_8": 4.8,
+                "newedge": 4.8,  # Same as zone 8 or adjust as needed
+                # Keep legacy thresholds for compatibility
+                "warning": 3.7,
+                "critical": 4.3,
+                "emergency": 4.8,
+            },
+            "P.4A": {"warning": 4.5, "critical": 6.0, "emergency": 7.5},
+            "P.20": {"warning": 3.0, "critical": 4.5, "emergency": 6.0},
+            "P.21": {"warning": 4.0, "critical": 5.5, "emergency": 7.0},
+            "P.67": {"warning": 6.0, "critical": 8.0, "emergency": 10.0},
+            "P.75": {"warning": 5.5, "critical": 7.5, "emergency": 9.5},
+            "P.103": {"warning": 7.0, "critical": 9.0, "emergency": 11.0},
+            # Default for unknown stations
+            "default": {"warning": 4.0, "critical": 6.0, "emergency": 8.0},
+        }
+
+    def connect_database(self):
+        """Initialize database connection"""
+        try:
+            db_config = Config.get_database_config()
+            self.db_adapter = create_database_adapter(
+                db_config["type"], connection_string=db_config["connection_string"]
+            )
+
+            if self.db_adapter.connect():
+                logger.info("Database connection established for alerting")
+                return True
+            else:
+                logger.error("Failed to connect to database")
+                return False
+
+        except Exception as e:
+            logger.error(f"Database connection error: {e}")
+            return False
+
+    def check_water_levels(self) -> List[WaterAlert]:
+        """Check current water levels against thresholds"""
+        alerts = []
+
+        if not self.db_adapter:
+            logger.error("Database not connected")
+            return alerts
+
+        try:
+            # Get latest measurements
+            measurements = self.db_adapter.get_latest_measurements(limit=50)
+
+            for measurement in measurements:
+                station_code = measurement.get("station_code", "UNKNOWN")
+                water_level = measurement.get("water_level")
+
+                if not water_level:
+                    continue
+
+                # Only alert for upstream stations and Chiang Mai
+                if station_code not in self.UPSTREAM_STATIONS:
+                    continue
+
+                # Get thresholds for this station
+                station_thresholds = self.thresholds.get(
+                    station_code, self.thresholds["default"]
+                )
+
+                # Check each threshold level
+                alert_level = None
+                threshold_value = None
+                alert_type = None
+
+                # Special handling for P.1 with zone-based thresholds
+                if station_code == "P.1" and "zone_1" in station_thresholds:
+                    # Check all zones in reverse order (highest to lowest)
+                    zones = [
+                        ("zone_8", 4.8, AlertLevel.EMERGENCY, "Zone 8 - Emergency"),
+                        ("newedge", 4.8, AlertLevel.EMERGENCY, "NewEdge Alert Level"),
+                        ("zone_7", 4.6, AlertLevel.CRITICAL, "Zone 7 - Critical"),
+                        ("zone_6", 4.3, AlertLevel.CRITICAL, "Zone 6 - Critical"),
+                        ("zone_5", 4.2, AlertLevel.WARNING, "Zone 5 - Warning"),
+                        ("zone_4", 4.1, AlertLevel.WARNING, "Zone 4 - Warning"),
+                        ("zone_3", 4.0, AlertLevel.WARNING, "Zone 3 - Warning"),
+                        ("zone_2", 3.9, AlertLevel.INFO, "Zone 2 - Info"),
+                        ("zone_1", 3.7, AlertLevel.INFO, "Zone 1 - Info"),
+                    ]
+
+                    for (
+                        zone_name,
+                        zone_threshold,
+                        zone_alert_level,
+                        zone_description,
+                    ) in zones:
+                        if water_level >= zone_threshold:
+                            alert_level = zone_alert_level
+                            threshold_value = zone_threshold
+                            alert_type = zone_description
+                            break
+
+                else:
+                    # Standard threshold checking for other stations
+                    if water_level >= station_thresholds.get("emergency", float("inf")):
+                        alert_level = AlertLevel.EMERGENCY
+                        threshold_value = station_thresholds["emergency"]
+                        alert_type = "Emergency Water Level"
+                    elif water_level >= station_thresholds.get(
+                        "critical", float("inf")
+                    ):
+                        alert_level = AlertLevel.CRITICAL
+                        threshold_value = station_thresholds["critical"]
+                        alert_type = "Critical Water Level"
+                    elif water_level >= station_thresholds.get("warning", float("inf")):
+                        alert_level = AlertLevel.WARNING
+                        threshold_value = station_thresholds["warning"]
+                        alert_type = "High Water Level"
+
+                if alert_level:
+                    alert = WaterAlert(
+                        station_code=station_code,
+                        station_name=measurement.get(
+                            "station_name_th", f"Station {station_code}"
+                        ),
+                        alert_type=alert_type,
+                        level=alert_level,
+                        water_level=water_level,
+                        threshold=threshold_value,
+                        discharge=measurement.get("discharge"),
+                        timestamp=measurement.get("timestamp"),
+                    )
+                    alerts.append(alert)
+
+        except Exception as e:
+            logger.error(f"Error checking water levels: {e}")
+
+        return alerts
+
+    def check_data_freshness(self, max_age_hours: int = 12) -> List[WaterAlert]:
+        """Check if data is fresh enough"""
+        alerts = []
+
+        if not self.db_adapter:
+            return alerts
+
+        try:
+            measurements = self.db_adapter.get_latest_measurements(limit=20)
+            cutoff_time = datetime.datetime.now() - datetime.timedelta(
+                hours=max_age_hours
+            )
+
+            for measurement in measurements:
+                timestamp = measurement.get("timestamp")
+                if timestamp and timestamp < cutoff_time:
+                    station_code = measurement.get("station_code", "UNKNOWN")
+
+                    age_hours = (
+                        datetime.datetime.now() - timestamp
+                    ).total_seconds() / 3600
+
+                    alert = WaterAlert(
+                        station_code=station_code,
+                        station_name=measurement.get(
+                            "station_name_th", f"Station {station_code}"
+                        ),
+                        alert_type="Stale Data",
+                        level=AlertLevel.WARNING,
+                        water_level=measurement.get("water_level", 0),
+                        threshold=max_age_hours,
+                        timestamp=timestamp,
+                        message=f"No fresh data for {age_hours:.1f} hours",
+                    )
+                    alerts.append(alert)
+
+        except Exception as e:
+            logger.error(f"Error checking data freshness: {e}")
+
+        return alerts
+
+    def check_rate_of_change(self, lookback_hours: int = 3) -> List[WaterAlert]:
+        """Check for rapid water level changes over recent hours"""
+        alerts = []
+
+        if not self.db_adapter:
+            return alerts
+
+        try:
+            # Define rate-of-change thresholds (meters per hour)
+            rate_thresholds = {
+                "P.1": {
+                    "warning": 0.15,  # 15cm/hour - moderate rise
+                    "critical": 0.25,  # 25cm/hour - rapid rise
+                    "emergency": 0.40,  # 40cm/hour - very rapid rise
+                },
+                "default": {"warning": 0.20, "critical": 0.35, "emergency": 0.50},
+            }
+
+            # Get recent measurements for each station
+            cutoff_time = datetime.datetime.now() - datetime.timedelta(
+                hours=lookback_hours
+            )
+
+            # Get unique stations from latest data
+            latest = self.db_adapter.get_latest_measurements(limit=20)
+            station_codes = set(
+                m.get("station_code") for m in latest if m.get("station_code")
+            )
+
+            for station_code in station_codes:
+                try:
+                    # Only alert for upstream stations and Chiang Mai
+                    if station_code not in self.UPSTREAM_STATIONS:
+                        continue
+
+                    # Get measurements for this station in the time window
+                    current_time = datetime.datetime.now()
+                    measurements = self.db_adapter.get_measurements_by_timerange(
+                        start_time=cutoff_time,
+                        end_time=current_time,
+                        station_codes=[station_code],
+                    )
+
+                    if len(measurements) < 2:
+                        continue  # Need at least 2 points to calculate rate
+
+                    # Sort by timestamp
+                    measurements = sorted(
+                        measurements, key=lambda m: m.get("timestamp")
+                    )
+
+                    # Get oldest and newest measurements
+                    oldest = measurements[0]
+                    newest = measurements[-1]
+
+                    oldest_time = oldest.get("timestamp")
+                    oldest_level = oldest.get("water_level")
+                    newest_time = newest.get("timestamp")
+                    newest_level = newest.get("water_level")
+
+                    # Convert timestamp strings to datetime if needed
+                    if isinstance(oldest_time, str):
+                        oldest_time = datetime.datetime.fromisoformat(oldest_time)
+                    if isinstance(newest_time, str):
+                        newest_time = datetime.datetime.fromisoformat(newest_time)
+
+                    # Calculate rate of change
+                    time_diff_hours = (newest_time - oldest_time).total_seconds() / 3600
+                    if time_diff_hours == 0:
+                        continue
+
+                    level_change = newest_level - oldest_level
+                    rate_per_hour = level_change / time_diff_hours
+
+                    # Only alert on rising water (positive rate)
+                    if rate_per_hour <= 0:
+                        continue
+
+                    # Get station info from latest data
+                    station_info = next(
+                        (m for m in latest if m.get("station_code") == station_code), {}
+                    )
+                    station_name = station_info.get("station_name_th", station_code)
+
+                    # Get thresholds for this station
+                    station_rate_threshold = rate_thresholds.get(
+                        station_code, rate_thresholds["default"]
+                    )
+
+                    alert_level = None
+                    threshold_value = None
+                    alert_type = None
+
+                    if rate_per_hour >= station_rate_threshold["emergency"]:
+                        alert_level = AlertLevel.EMERGENCY
+                        threshold_value = station_rate_threshold["emergency"]
+                        alert_type = "Very Rapid Water Level Rise"
+                    elif rate_per_hour >= station_rate_threshold["critical"]:
+                        alert_level = AlertLevel.CRITICAL
+                        threshold_value = station_rate_threshold["critical"]
+                        alert_type = "Rapid Water Level Rise"
+                    elif rate_per_hour >= station_rate_threshold["warning"]:
+                        alert_level = AlertLevel.WARNING
+                        threshold_value = station_rate_threshold["warning"]
+                        alert_type = "Moderate Water Level Rise"
+
+                    if alert_level:
+                        message = (
+                            f"Rising at {rate_per_hour:.2f}m/h over last {time_diff_hours:.1f}h "
+                            f"(change: {level_change:+.2f}m)"
+                        )
+
+                        alert = WaterAlert(
+                            station_code=station_code,
+                            station_name=station_name or f"Station {station_code}",
+                            alert_type=alert_type,
+                            level=alert_level,
+                            water_level=newest_level,
+                            threshold=threshold_value,
+                            timestamp=newest_time,
+                            message=message,
+                        )
+                        alerts.append(alert)
+
+                except Exception as station_error:
+                    logger.debug(
+                        f"Error checking rate of change for station {station_code}: {station_error}"
+                    )
+                    continue
+
+        except Exception as e:
+            logger.error(f"Error checking rate of change: {e}")
+
+        return alerts
+
+    def send_alerts(self, alerts: List[WaterAlert]) -> int:
+        """Send alerts via configured channels"""
+        sent_count = 0
+
+        if not alerts:
+            return sent_count
+
+        if self.matrix_notifier:
+            for alert in alerts:
+                if self.matrix_notifier.send_alert(alert):
+                    sent_count += 1
+
+        # Could add other notification channels here:
+        # - Email
+        # - Discord
+        # - Telegram
+        # - SMS
+
+        return sent_count
+
+    def run_alert_check(self) -> Dict[str, int]:
+        """Run complete alert check cycle"""
+        if not self.connect_database():
+            return {"error": 1}
+
+        # Check water levels
+        water_alerts = self.check_water_levels()
+
+        # Check data freshness
+        data_alerts = self.check_data_freshness()
+
+        # Check rate of change (rapid rises)
+        rate_alerts = self.check_rate_of_change()
+
+        # Combine alerts
+        all_alerts = water_alerts + rate_alerts
+
+        # Send alerts
+        sent_count = self.send_alerts(all_alerts)
+
+        logger.info(
+            f"Alert check complete: {len(all_alerts)} alerts, {sent_count} sent"
+        )
+
+        return {
+            "water_alerts": len(water_alerts),
+            "data_alerts": len(data_alerts),
+            "rate_alerts": len(rate_alerts),
+            "total_alerts": len(all_alerts),
+            "sent": sent_count,
+        }
+
+
+def main():
+    """Standalone alerting check"""
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Water Level Alert System")
+    parser.add_argument("--check", action="store_true", help="Run alert check")
+    parser.add_argument("--test", action="store_true", help="Send test message")
+    args = parser.parse_args()
+
+    alerting = WaterLevelAlertSystem()
+
+    if args.test:
+        if alerting.matrix_notifier:
+            test_message = (
+                "🧪 **Test Alert**\n\nThis is a test message from the Water Level Alert System.\n\n"
+                "If you received this, Matrix notifications are working correctly!"
+            )
+            success = alerting.matrix_notifier.send_message(test_message)
+            print(f"Test message sent: {success}")
+        else:
+            print("Matrix notifier not configured")
+
+    elif args.check:
+        results = alerting.run_alert_check()
+        print(f"Alert check results: {results}")
+
+    else:
+        print("Use --check or --test")
+
+
+if __name__ == "__main__":
+    main()

src/config.py

@@ -1,16 +1,25 @@
 import os
-from typing import Dict, Any, Optional
+from typing import Any, Dict
+
+# Load environment variables from .env file
+try:
+    from dotenv import load_dotenv
+
+    load_dotenv()
+except ImportError:
+    # python-dotenv not installed, continue without it
+    pass
 
 try:
     from .exceptions import ConfigurationError
-    from .models import DatabaseType, DatabaseConfig
+    from .models import DatabaseType
 except ImportError:
     # Handle case when running as standalone script
     class ConfigurationError(Exception):
         pass
-    
+
     from enum import Enum
-    
+
     class DatabaseType(Enum):
         SQLITE = "sqlite"
         MYSQL = "mysql"
@@ -18,174 +27,273 @@ except ImportError:
         INFLUXDB = "influxdb"
         VICTORIAMETRICS = "victoriametrics"
 
+
 class Config:
     """Configuration class for the Water Level Monitor"""
-    
+
     # Database settings
-    DATABASE_PATH = os.getenv('WATER_DB_PATH', 'water_levels.db')
-    
+    DATABASE_PATH = os.getenv("WATER_DB_PATH", "water_levels.db")
+
     # Website settings
     TARGET_URL = "https://hyd-app-db.rid.go.th/hydro1h.html"
     API_URL = "https://hyd-app-db.rid.go.th/webservice/getGroupHourlyWaterLevelReportAllHL.ashx"
-    REQUEST_TIMEOUT = int(os.getenv('REQUEST_TIMEOUT', '30'))
-    USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
-    
+    THAIWATER_API_KEY = os.getenv("THAIWATER_API_KEY")
+    REQUEST_TIMEOUT = int(os.getenv("REQUEST_TIMEOUT", "30"))
+    USER_AGENT = (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+        "(KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
+    )
+
     # Database configuration
-    DB_TYPE = os.getenv('DB_TYPE', 'sqlite').lower()
-    
+    # When DB_TYPE is not set explicitly, a configured Postgres connection wins over the sqlite default
+    DB_TYPE = os.getenv(
+        "DB_TYPE",
+        "postgresql" if os.getenv("POSTGRES_CONNECTION_STRING") else "sqlite",
+    ).lower()
+
     # VictoriaMetrics settings
-    VM_HOST = os.getenv('VM_HOST', 'vm.newedge.house')
-    VM_PORT = int(os.getenv('VM_PORT', '443'))
-    
+    # Default to localhost; set VM_HOST in the environment for real deployments
+    # (avoids committing infrastructure hostnames to the repo).
+    VM_HOST = os.getenv("VM_HOST", "localhost")
+    VM_PORT = int(os.getenv("VM_PORT", "443"))
+
     # Support for HTTPS URLs (e.g., behind reverse proxy)
-    VM_URL = os.getenv('VM_URL')  # Full URL override (e.g., https://vm.example.com)
-    
+    VM_URL = os.getenv("VM_URL")  # Full URL override (e.g., https://vm.example.com)
+
     # InfluxDB settings
-    INFLUX_HOST = os.getenv('INFLUX_HOST', 'localhost')
-    INFLUX_PORT = int(os.getenv('INFLUX_PORT', '8086'))
-    INFLUX_DATABASE = os.getenv('INFLUX_DATABASE', 'water_monitoring')
-    INFLUX_USERNAME = os.getenv('INFLUX_USERNAME')
-    INFLUX_PASSWORD = os.getenv('INFLUX_PASSWORD')
-    
+    INFLUX_HOST = os.getenv("INFLUX_HOST", "localhost")
+    INFLUX_PORT = int(os.getenv("INFLUX_PORT", "8086"))
+    INFLUX_DATABASE = os.getenv("INFLUX_DATABASE", "water_monitoring")
+    INFLUX_USERNAME = os.getenv("INFLUX_USERNAME")
+    INFLUX_PASSWORD = os.getenv("INFLUX_PASSWORD")
+
     # PostgreSQL settings
-    POSTGRES_CONNECTION_STRING = os.getenv('POSTGRES_CONNECTION_STRING')
-    
+    POSTGRES_CONNECTION_STRING = os.getenv("POSTGRES_CONNECTION_STRING")
+    POSTGRES_HOST = os.getenv("POSTGRES_HOST", "localhost")
+    POSTGRES_PORT = int(os.getenv("POSTGRES_PORT", "5432"))
+    POSTGRES_DB = os.getenv("POSTGRES_DB", "water_monitoring")
+    POSTGRES_USER = os.getenv("POSTGRES_USER", "postgres")
+    POSTGRES_PASSWORD = os.getenv("POSTGRES_PASSWORD")
+
     # MySQL settings
-    MYSQL_CONNECTION_STRING = os.getenv('MYSQL_CONNECTION_STRING')
-    
+    MYSQL_CONNECTION_STRING = os.getenv("MYSQL_CONNECTION_STRING")
+
+    # HII/ThaiWater open api-v3 collection (rainfall + backup water level)
+    # See docs/DATA_SOURCES.md. Requires a SQL DB_TYPE (sqlite/postgresql/mysql).
+    ENABLE_HII_COLLECTION = os.getenv("ENABLE_HII_COLLECTION", "true").lower() in (
+        "1",
+        "true",
+        "yes",
+    )
+    HII_BASIN_CODE = int(os.getenv("HII_BASIN_CODE", "6"))  # 6 = Ping Basin
+
+    # RID large-dam daily status (app.rid.go.th/reservoir) — Mae Ngat et al.
+    ENABLE_RESERVOIR_COLLECTION = os.getenv(
+        "ENABLE_RESERVOIR_COLLECTION", "true"
+    ).lower() in ("1", "true", "yes")
+    # TTL for the /api/hii/*/latest response cache; source data changes hourly
+    HII_CACHE_TTL_SECONDS = int(os.getenv("HII_CACHE_TTL_SECONDS", "120"))
+    # TTL for the /measurements/latest response cache (hottest endpoint)
+    LATEST_CACHE_TTL_SECONDS = int(os.getenv("LATEST_CACHE_TTL_SECONDS", "45"))
+
+    # TTL for /health check results (includes an external RID-API probe)
+    HEALTH_CACHE_TTL_SECONDS = int(os.getenv("HEALTH_CACHE_TTL_SECONDS", "30"))
+
+    # Thread-pool size for blocking work in the web process (DB queries,
+    # inference, health probes). Waiting threads are cheap; starving the pool
+    # stalls every endpoint that needs a thread.
+    EXECUTOR_THREADS = int(os.getenv("EXECUTOR_THREADS", "48"))
+
+    # Web server worker processes. Above 1, uvicorn forks workers and a
+    # localhost lock port elects a single background-collection leader.
+    WEB_WORKERS = int(os.getenv("WEB_WORKERS", "2"))
+    COLLECTION_LEADER_PORT = int(os.getenv("COLLECTION_LEADER_PORT", "8901"))
+
+    # Umami analytics (self-hosted). The website id is public (it ships in the
+    # dashboard <script> tag); server-side API tracking posts to /api/send.
+    UMAMI_API_URL = os.getenv("UMAMI_API_URL", "https://stats.buildfor.life/api/send")
+    UMAMI_WEBSITE_ID = os.getenv(
+        "UMAMI_WEBSITE_ID", "00b2be73-8f5f-4400-9029-3be852eb08f7"
+    )
+    UMAMI_TRACK_API = os.getenv("UMAMI_TRACK_API", "true").lower() in (
+        "1",
+        "true",
+        "yes",
+    )
+
     # Scheduler settings
-    SCRAPING_INTERVAL_HOURS = int(os.getenv('SCRAPING_INTERVAL_HOURS', '1'))
-    
+    SCRAPING_INTERVAL_HOURS = int(os.getenv("SCRAPING_INTERVAL_HOURS", "1"))
+
     # Logging settings
-    LOG_LEVEL = os.getenv('LOG_LEVEL', 'INFO')
-    LOG_FILE = os.getenv('LOG_FILE', 'water_monitor.log')
-    LOG_FORMAT = '%(asctime)s - %(levelname)s - %(message)s'
-    
+    LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO")
+    LOG_FILE = os.getenv("LOG_FILE", "water_monitor.log")
+    LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s"
+
     # Data retention
-    DATA_RETENTION_DAYS = int(os.getenv('DATA_RETENTION_DAYS', '365'))
-    
+    DATA_RETENTION_DAYS = int(os.getenv("DATA_RETENTION_DAYS", "365"))
+
     # Retry settings
-    MAX_RETRIES = int(os.getenv('MAX_RETRIES', '3'))
-    RETRY_DELAY_SECONDS = int(os.getenv('RETRY_DELAY_SECONDS', '60'))
-    
+    MAX_RETRIES = int(os.getenv("MAX_RETRIES", "3"))
+    RETRY_DELAY_SECONDS = int(os.getenv("RETRY_DELAY_SECONDS", "60"))
+
+    # Station configuration
+    # Runtime-writable JSON file that persists the station mapping across restarts
+    # (station CRUD via the API writes here). If it does not exist, the bundled
+    # defaults in src/data/stations.json are used to seed it.
+    STATION_CONFIG_PATH = os.getenv("STATION_CONFIG_PATH", "stations.json")
+
+    # Web API / CORS settings
+    # Comma-separated list of allowed origins. Defaults to none (same-origin only);
+    # set CORS_ALLOW_ORIGINS to a specific list of front-end origins in production.
+    # Credentials are only enabled when explicit (non-wildcard) origins are set,
+    # because "*" + credentials is rejected by browsers and unsafe.
+    CORS_ALLOW_ORIGINS = [
+        origin.strip()
+        for origin in os.getenv("CORS_ALLOW_ORIGINS", "").split(",")
+        if origin.strip()
+    ]
+
     @classmethod
     def validate_config(cls) -> bool:
         """Validate configuration settings"""
         errors = []
-        
+
         # Validate database type
         try:
             DatabaseType(cls.DB_TYPE)
         except ValueError:
             errors.append(f"Invalid DB_TYPE: {cls.DB_TYPE}")
-        
+
         # Validate database-specific settings
-        if cls.DB_TYPE == 'victoriametrics':
+        if cls.DB_TYPE == "victoriametrics":
             if not cls.VM_HOST:
                 errors.append("VM_HOST is required for VictoriaMetrics")
             if not isinstance(cls.VM_PORT, int) or cls.VM_PORT <= 0:
                 errors.append("VM_PORT must be a positive integer")
-        
-        elif cls.DB_TYPE == 'influxdb':
+
+        elif cls.DB_TYPE == "influxdb":
             if not cls.INFLUX_HOST:
                 errors.append("INFLUX_HOST is required for InfluxDB")
             if not cls.INFLUX_DATABASE:
                 errors.append("INFLUX_DATABASE is required for InfluxDB")
-        
-        elif cls.DB_TYPE in ['postgresql', 'mysql']:
-            connection_string = (cls.POSTGRES_CONNECTION_STRING if cls.DB_TYPE == 'postgresql' 
-                               else cls.MYSQL_CONNECTION_STRING)
-            if not connection_string:
-                errors.append(f"Connection string is required for {cls.DB_TYPE.upper()}")
-        
+
+        elif cls.DB_TYPE in ["postgresql", "mysql"]:
+            if cls.DB_TYPE == "postgresql":
+                # Check if either connection string or individual components are provided
+                if not cls.POSTGRES_CONNECTION_STRING:
+                    # If no connection string, check individual components
+                    if not cls.POSTGRES_HOST:
+                        errors.append("POSTGRES_HOST is required for PostgreSQL")
+                    if not cls.POSTGRES_USER:
+                        errors.append("POSTGRES_USER is required for PostgreSQL")
+                    if not cls.POSTGRES_PASSWORD:
+                        errors.append("POSTGRES_PASSWORD is required for PostgreSQL")
+                    if not cls.POSTGRES_DB:
+                        errors.append("POSTGRES_DB is required for PostgreSQL")
+            else:  # mysql
+                if not cls.MYSQL_CONNECTION_STRING:
+                    errors.append("MYSQL_CONNECTION_STRING is required for MySQL")
+
         # Validate numeric settings
         if cls.SCRAPING_INTERVAL_HOURS <= 0:
             errors.append("SCRAPING_INTERVAL_HOURS must be positive")
-        
+
         if cls.DATA_RETENTION_DAYS <= 0:
             errors.append("DATA_RETENTION_DAYS must be positive")
-        
+
         if errors:
             raise ConfigurationError(f"Configuration errors: {'; '.join(errors)}")
-        
+
         return True
-    
+
     @classmethod
     def get_database_config(cls) -> Dict[str, Any]:
         """Returns database configuration based on DB_TYPE"""
-        if cls.DB_TYPE == 'victoriametrics':
+        if cls.DB_TYPE == "victoriametrics":
+            return {"type": "victoriametrics", "host": cls.VM_HOST, "port": cls.VM_PORT}
+        elif cls.DB_TYPE == "influxdb":
             return {
-                'type': 'victoriametrics',
-                'host': cls.VM_HOST,
-                'port': cls.VM_PORT
-            }
-        elif cls.DB_TYPE == 'influxdb':
-            return {
-                'type': 'influxdb',
-                'host': cls.INFLUX_HOST,
-                'port': cls.INFLUX_PORT,
-                'database': cls.INFLUX_DATABASE,
-                'username': cls.INFLUX_USERNAME,
-                'password': cls.INFLUX_PASSWORD
-            }
-        elif cls.DB_TYPE == 'postgresql':
-            return {
-                'type': 'postgresql',
-                'connection_string': cls.POSTGRES_CONNECTION_STRING or 
-                    'postgresql://postgres:password@localhost:5432/water_monitoring'
-            }
-        elif cls.DB_TYPE == 'mysql':
-            return {
-                'type': 'mysql',
-                'connection_string': cls.MYSQL_CONNECTION_STRING or 
-                    'mysql://root:password@localhost:3306/water_monitoring'
+                "type": "influxdb",
+                "host": cls.INFLUX_HOST,
+                "port": cls.INFLUX_PORT,
+                "database": cls.INFLUX_DATABASE,
+                "username": cls.INFLUX_USERNAME,
+                "password": cls.INFLUX_PASSWORD,
             }
+        elif cls.DB_TYPE == "postgresql":
+            # Use individual components if POSTGRES_CONNECTION_STRING is not provided
+            if cls.POSTGRES_CONNECTION_STRING:
+                return {
+                    "type": "postgresql",
+                    "connection_string": cls.POSTGRES_CONNECTION_STRING,
+                }
+            else:
+                # Build connection string from components (automatically URL-encodes password)
+                import urllib.parse
+
+                if not cls.POSTGRES_PASSWORD:
+                    raise ConfigurationError(
+                        "POSTGRES_PASSWORD is required for PostgreSQL (no default is provided)"
+                    )
+                password = urllib.parse.quote(cls.POSTGRES_PASSWORD, safe="")
+                connection_string = (
+                    f"postgresql://{cls.POSTGRES_USER}:{password}"
+                    f"@{cls.POSTGRES_HOST}:{cls.POSTGRES_PORT}/{cls.POSTGRES_DB}"
+                )
+                return {"type": "postgresql", "connection_string": connection_string}
+        elif cls.DB_TYPE == "mysql":
+            if not cls.MYSQL_CONNECTION_STRING:
+                raise ConfigurationError(
+                    "MYSQL_CONNECTION_STRING is required for MySQL (no default is provided)"
+                )
+            return {"type": "mysql", "connection_string": cls.MYSQL_CONNECTION_STRING}
         else:  # sqlite
             return {
-                'type': 'sqlite',
-                'connection_string': f'sqlite:///{cls.DATABASE_PATH}'
+                "type": "sqlite",
+                "connection_string": f"sqlite:///{cls.DATABASE_PATH}",
             }
-    
+
     @classmethod
     def get_all_settings(cls) -> Dict[str, Any]:
         """Returns all configuration settings"""
         return {
-            'DB_TYPE': cls.DB_TYPE,
-            'DATABASE_PATH': cls.DATABASE_PATH,
-            'TARGET_URL': cls.TARGET_URL,
-            'API_URL': cls.API_URL,
-            'REQUEST_TIMEOUT': cls.REQUEST_TIMEOUT,
-            'SCRAPING_INTERVAL_HOURS': cls.SCRAPING_INTERVAL_HOURS,
-            'LOG_LEVEL': cls.LOG_LEVEL,
-            'LOG_FILE': cls.LOG_FILE,
-            'DATA_RETENTION_DAYS': cls.DATA_RETENTION_DAYS,
-            'MAX_RETRIES': cls.MAX_RETRIES,
-            'RETRY_DELAY_SECONDS': cls.RETRY_DELAY_SECONDS,
-            'VM_HOST': cls.VM_HOST,
-            'VM_PORT': cls.VM_PORT,
-            'INFLUX_HOST': cls.INFLUX_HOST,
-            'INFLUX_PORT': cls.INFLUX_PORT,
-            'INFLUX_DATABASE': cls.INFLUX_DATABASE
+            "DB_TYPE": cls.DB_TYPE,
+            "DATABASE_PATH": cls.DATABASE_PATH,
+            "TARGET_URL": cls.TARGET_URL,
+            "API_URL": cls.API_URL,
+            "REQUEST_TIMEOUT": cls.REQUEST_TIMEOUT,
+            "SCRAPING_INTERVAL_HOURS": cls.SCRAPING_INTERVAL_HOURS,
+            "LOG_LEVEL": cls.LOG_LEVEL,
+            "LOG_FILE": cls.LOG_FILE,
+            "DATA_RETENTION_DAYS": cls.DATA_RETENTION_DAYS,
+            "MAX_RETRIES": cls.MAX_RETRIES,
+            "RETRY_DELAY_SECONDS": cls.RETRY_DELAY_SECONDS,
+            "VM_HOST": cls.VM_HOST,
+            "VM_PORT": cls.VM_PORT,
+            "INFLUX_HOST": cls.INFLUX_HOST,
+            "INFLUX_PORT": cls.INFLUX_PORT,
+            "INFLUX_DATABASE": cls.INFLUX_DATABASE,
         }
-    
+
     @classmethod
     def print_settings(cls):
         """Prints all current settings"""
         print("=== Water Level Monitor Configuration ===")
         for key, value in cls.get_all_settings().items():
             # Hide sensitive information
-            if 'PASSWORD' in key and value:
-                value = '*' * len(str(value))
+            if "PASSWORD" in key and value:
+                value = "*" * len(str(value))
             print(f"{key}: {value}")
         print("=" * 45)
-        
+
         print("\nDatabase Configuration:")
         db_config = cls.get_database_config()
         for key, value in db_config.items():
-            if 'password' in key and value:
-                value = '*' * len(str(value))
+            if "password" in key and value:
+                value = "*" * len(str(value))
             print(f"  {key}: {value}")
         print("=" * 45)
 
+
 if __name__ == "__main__":
     Config.print_settings()

src/data/stations.json

@@ -0,0 +1,130 @@
+{
+  "1": {
+    "code": "P.20",
+    "thai_name": "บ้านเชียงดาว",
+    "english_name": "Ban Chiang Dao",
+    "latitude": 19.36731448032191,
+    "longitude": 98.9688487015384,
+    "geohash": null
+  },
+  "2": {
+    "code": "P.75",
+    "thai_name": "บ้านช่อแล",
+    "english_name": "Ban Chai Lat",
+    "latitude": 19.145972935976225,
+    "longitude": 99.00735727149247,
+    "geohash": null
+  },
+  "3": {
+    "code": "P.92",
+    "thai_name": "บ้านเมืองกึ๊ด",
+    "english_name": "Ban Muang Aut",
+    "latitude": 19.220518985435646,
+    "longitude": 98.84733127007874,
+    "geohash": null
+  },
+  "4": {
+    "code": "P.4A",
+    "thai_name": "บ้านแม่แตง",
+    "english_name": "Ban Mae Taeng",
+    "latitude": 19.1222679952378,
+    "longitude": 98.94437462084075,
+    "geohash": null
+  },
+  "5": {
+    "code": "P.67",
+    "thai_name": "บ้านแม่แต",
+    "english_name": "Ban Tae",
+    "latitude": 19.009762080002453,
+    "longitude": 98.95978297135508,
+    "geohash": null
+  },
+  "6": {
+    "code": "P.21",
+    "thai_name": "บ้านริมใต้",
+    "english_name": "Ban Rim Tai",
+    "latitude": 18.917459157963293,
+    "longitude": 98.97018092996231,
+    "geohash": null
+  },
+  "7": {
+    "code": "P.103",
+    "thai_name": "สะพานวงแหวนรอบ 3",
+    "english_name": "Ring Bridge 3",
+    "latitude": 18.86664807441675,
+    "longitude": 98.9781107622432,
+    "geohash": null
+  },
+  "8": {
+    "code": "P.1",
+    "thai_name": "สะพานนวรัฐ",
+    "english_name": "Nawarat Bridge",
+    "latitude": 18.7875,
+    "longitude": 99.0045,
+    "geohash": "w5q6uuhvfcfp25"
+  },
+  "9": {
+    "code": "P.82",
+    "thai_name": "บ้านสบวิน",
+    "english_name": "Ban Sob win",
+    "latitude": 18.6519444,
+    "longitude": 98.69,
+    "geohash": null
+  },
+  "10": {
+    "code": "P.84",
+    "thai_name": "บ้านพันตน",
+    "english_name": "Ban Panton",
+    "latitude": 18.591315274591334,
+    "longitude": 98.79657058508496,
+    "geohash": null
+  },
+  "11": {
+    "code": "P.81",
+    "thai_name": "บ้านโป่ง",
+    "english_name": "Ban Pong",
+    "latitude": 18.693611,
+    "longitude": 99.081944,
+    "geohash": null
+  },
+  "12": {
+    "code": "P.5",
+    "thai_name": "สะพานท่านาง",
+    "english_name": "Tha Nang Bridge",
+    "latitude": 18.580269437546555,
+    "longitude": 99.01021397084362,
+    "geohash": null
+  },
+  "13": {
+    "code": "P.77",
+    "thai_name": "บ้านสบแม่สะป๊วด",
+    "english_name": "Baan Sop Mae Sapuord",
+    "latitude": 18.433347475179602,
+    "longitude": 99.08510036666527,
+    "geohash": null
+  },
+  "14": {
+    "code": "P.87",
+    "thai_name": "บ้านป่าซาง",
+    "english_name": "Ban Pa Sang",
+    "latitude": 18.519121825282486,
+    "longitude": 98.94224374138238,
+    "geohash": null
+  },
+  "15": {
+    "code": "P.76",
+    "thai_name": "บ้านแม่อีไฮ",
+    "english_name": "Banb Mae I Hai",
+    "latitude": 18.141465831254404,
+    "longitude": 98.89642508267181,
+    "geohash": null
+  },
+  "16": {
+    "code": "P.85",
+    "thai_name": "บ้านหล่ายแก้ว",
+    "english_name": "Baan Lai Kaew",
+    "latitude": 18.17856361002219,
+    "longitude": 98.63023114782287,
+    "geohash": null
+  }
+}

src/demo_databases.py

@@ -3,124 +3,133 @@
 Demo script showing different database backend options for water monitoring
 """
 
+import datetime
 import os
 import sys
-import datetime
+
 from water_scraper_v3 import EnhancedWaterMonitorScraper
 
+
 def demo_sqlite():
     """Demo with SQLite (local development)"""
     print("=" * 60)
     print("🗄️  SQLite Demo (Local Development)")
     print("=" * 60)
-    
-    config = {
-        'type': 'sqlite',
-        'connection_string': 'sqlite:///demo_water_sqlite.db'
-    }
-    
+
+    config = {"type": "sqlite", "connection_string": "sqlite:///demo_water_sqlite.db"}
+
     try:
         scraper = EnhancedWaterMonitorScraper(config)
-        
+
         # Fetch and save data
         print("Fetching data from API...")
         data = scraper.fetch_water_data()
-        
+
         if data:
             print(f"✓ Fetched {len(data)} data points")
             success = scraper.save_to_database(data)
-            
+
             if success:
                 print("✓ Data saved to SQLite database")
-                
+
                 # Show latest data
                 latest = scraper.get_latest_data(5)
                 print(f"\nLatest 5 measurements:")
                 for measurement in latest:
-                    print(f"  • {measurement['station_code']} ({measurement['station_name_en']}): "
-                          f"{measurement['water_level']:.2f}m, {measurement['discharge']:.1f} cms")
+                    print(
+                        f"  • {measurement['station_code']} ({measurement['station_name_en']}): "
+                        f"{measurement['water_level']:.2f}m, {measurement['discharge']:.1f} cms"
+                    )
             else:
                 print("✗ Failed to save data")
         else:
             print("✗ No data fetched")
-            
+
     except Exception as e:
         print(f"Error: {e}")
 
+
 def demo_influxdb():
     """Demo with InfluxDB (requires InfluxDB running)"""
     print("\n" + "=" * 60)
     print("📊 InfluxDB Demo (Time-Series Database)")
     print("=" * 60)
-    
+
     config = {
-        'type': 'influxdb',
-        'host': 'localhost',
-        'port': 8086,
-        'database': 'water_monitoring_demo',
-        'username': None,  # Set if authentication is enabled
-        'password': None
+        "type": "influxdb",
+        "host": "localhost",
+        "port": 8086,
+        "database": "water_monitoring_demo",
+        "username": None,  # Set if authentication is enabled
+        "password": None,
     }
-    
+
     try:
         scraper = EnhancedWaterMonitorScraper(config)
-        
+
         if scraper.db_adapter and scraper.db_adapter.client:
             print("✓ Connected to InfluxDB")
-            
+
             # Fetch and save data
             print("Fetching data from API...")
             data = scraper.fetch_water_data()
-            
+
             if data:
                 print(f"✓ Fetched {len(data)} data points")
                 success = scraper.save_to_database(data)
-                
+
                 if success:
                     print("✓ Data saved to InfluxDB")
                     print("💡 You can now query this data in Grafana or InfluxDB CLI")
-                    print("   Example query: SELECT * FROM water_data ORDER BY time DESC LIMIT 10")
+                    print(
+                        "   Example query: SELECT * FROM water_data ORDER BY time DESC LIMIT 10"
+                    )
                 else:
                     print("✗ Failed to save data")
             else:
                 print("✗ No data fetched")
         else:
             print("✗ Could not connect to InfluxDB")
-            print("💡 Make sure InfluxDB is running: docker run -p 8086:8086 influxdb:1.8")
-            
+            print(
+                "💡 Make sure InfluxDB is running: docker run -p 8086:8086 influxdb:1.8"
+            )
+
     except Exception as e:
         print(f"Error: {e}")
         print("💡 InfluxDB might not be running or accessible")
 
+
 def demo_postgresql():
     """Demo with PostgreSQL (requires PostgreSQL running)"""
     print("\n" + "=" * 60)
     print("🐘 PostgreSQL Demo (Relational Database)")
     print("=" * 60)
-    
+
     config = {
-        'type': 'postgresql',
-        'connection_string': 'postgresql://postgres:password@localhost:5432/water_monitoring'
+        "type": "postgresql",
+        "connection_string": "postgresql://postgres:password@localhost:5432/water_monitoring",
     }
-    
+
     try:
         scraper = EnhancedWaterMonitorScraper(config)
-        
+
         if scraper.db_adapter and scraper.db_adapter.engine:
             print("✓ Connected to PostgreSQL")
-            
+
             # Fetch and save data
             print("Fetching data from API...")
             data = scraper.fetch_water_data()
-            
+
             if data:
                 print(f"✓ Fetched {len(data)} data points")
                 success = scraper.save_to_database(data)
-                
+
                 if success:
                     print("✓ Data saved to PostgreSQL")
                     print("💡 You can now query this data with SQL")
-                    print("   Example: SELECT * FROM water_measurements ORDER BY timestamp DESC LIMIT 10;")
+                    print(
+                        "   Example: SELECT * FROM water_measurements ORDER BY timestamp DESC LIMIT 10;"
+                    )
                 else:
                     print("✗ Failed to save data")
             else:
@@ -128,40 +137,43 @@ def demo_postgresql():
         else:
             print("✗ Could not connect to PostgreSQL")
             print("💡 Make sure PostgreSQL is running with correct credentials")
-            
+
     except Exception as e:
         print(f"Error: {e}")
         print("💡 PostgreSQL might not be running or credentials might be wrong")
 
+
 def demo_mysql():
     """Demo with MySQL (requires MySQL running)"""
     print("\n" + "=" * 60)
     print("🐬 MySQL Demo (Relational Database)")
     print("=" * 60)
-    
+
     config = {
-        'type': 'mysql',
-        'connection_string': 'mysql://root:password@localhost:3306/water_monitoring'
+        "type": "mysql",
+        "connection_string": "mysql://root:password@localhost:3306/water_monitoring",
     }
-    
+
     try:
         scraper = EnhancedWaterMonitorScraper(config)
-        
+
         if scraper.db_adapter and scraper.db_adapter.engine:
             print("✓ Connected to MySQL")
-            
+
             # Fetch and save data
             print("Fetching data from API...")
             data = scraper.fetch_water_data()
-            
+
             if data:
                 print(f"✓ Fetched {len(data)} data points")
                 success = scraper.save_to_database(data)
-                
+
                 if success:
                     print("✓ Data saved to MySQL")
                     print("💡 You can now query this data with SQL")
-                    print("   Example: SELECT * FROM water_measurements ORDER BY timestamp DESC LIMIT 10;")
+                    print(
+                        "   Example: SELECT * FROM water_measurements ORDER BY timestamp DESC LIMIT 10;"
+                    )
                 else:
                     print("✗ Failed to save data")
             else:
@@ -169,53 +181,51 @@ def demo_mysql():
         else:
             print("✗ Could not connect to MySQL")
             print("💡 Make sure MySQL is running with correct credentials")
-            
+
     except Exception as e:
         print(f"Error: {e}")
         print("💡 MySQL might not be running or credentials might be wrong")
 
+
 def demo_victoriametrics():
     """Demo with VictoriaMetrics (supports both local and HTTPS configurations)"""
     print("\n" + "=" * 60)
     print("⚡ VictoriaMetrics Demo (High-Performance Metrics)")
     print("=" * 60)
-    
+
     # Use configuration from environment or config.py
     from config import Config
+
     db_config = Config.get_database_config()
-    
-    if db_config['type'] != 'victoriametrics':
+
+    if db_config["type"] != "victoriametrics":
         # Fallback to default local configuration
-        config = {
-            'type': 'victoriametrics',
-            'host': 'vm.newedge.house',
-            'port': 443
-        }
+        config = {"type": "victoriametrics", "host": "vm.newedge.house", "port": 443}
     else:
         config = db_config
-    
+
     print(f"Connecting to: {config['host']}:{config['port']}")
-    
+
     try:
         scraper = EnhancedWaterMonitorScraper(config)
-        
+
         if scraper.db_adapter:
             # Test connection using the adapter's connect method
             if scraper.db_adapter.connect():
                 print("✓ Connected to VictoriaMetrics")
-                
+
                 # Fetch and save data
                 print("Fetching data from API...")
                 data = scraper.fetch_water_data()
-                
+
                 if data:
                     print(f"✓ Fetched {len(data)} data points")
                     success = scraper.save_to_database(data)
-                    
+
                     if success:
                         print("✓ Data saved to VictoriaMetrics")
                         print("💡 You can now query this data via Prometheus API")
-                        
+
                         # Show appropriate query URL based on configuration
                         base_url = scraper.db_adapter.base_url
                         print(f"   Example: {base_url}/api/v1/query?query=water_level")
@@ -226,56 +236,63 @@ def demo_victoriametrics():
                     print("✗ No data fetched")
             else:
                 print("✗ Could not connect to VictoriaMetrics")
-                if config['host'] == 'localhost':
+                if config["host"] == "localhost":
                     print("💡 Make sure VictoriaMetrics is running locally:")
                     print("   docker run -p 8428:8428 victoriametrics/victoria-metrics")
                 else:
-                    print(f"💡 Check if VictoriaMetrics is accessible at {config['host']}:{config['port']}")
+                    print(
+                        f"💡 Check if VictoriaMetrics is accessible at {config['host']}:{config['port']}"
+                    )
                     print("💡 Verify HTTPS configuration and network connectivity")
         else:
             print("✗ Failed to initialize VictoriaMetrics adapter")
-            
+
     except Exception as e:
         print(f"Error: {e}")
         print("💡 Check your VictoriaMetrics configuration and network connectivity")
 
+
 def show_recommendations():
     """Show database recommendations"""
     print("\n" + "=" * 60)
     print("🏆 Database Recommendations")
     print("=" * 60)
-    
+
     recommendations = [
         {
-            'name': 'InfluxDB',
-            'best_for': 'Time-series data, Grafana dashboards',
-            'pros': ['Purpose-built for time-series', 'Great compression', 'Built-in retention'],
-            'cons': ['Learning curve', 'Less flexible for complex queries'],
-            'use_case': 'Recommended for most water monitoring deployments'
+            "name": "InfluxDB",
+            "best_for": "Time-series data, Grafana dashboards",
+            "pros": [
+                "Purpose-built for time-series",
+                "Great compression",
+                "Built-in retention",
+            ],
+            "cons": ["Learning curve", "Less flexible for complex queries"],
+            "use_case": "Recommended for most water monitoring deployments",
         },
         {
-            'name': 'PostgreSQL + TimescaleDB',
-            'best_for': 'Complex queries, existing PostgreSQL infrastructure',
-            'pros': ['Mature ecosystem', 'SQL compatibility', 'ACID compliance'],
-            'cons': ['More complex setup', 'Higher resource usage'],
-            'use_case': 'Best for organizations already using PostgreSQL'
+            "name": "PostgreSQL + TimescaleDB",
+            "best_for": "Complex queries, existing PostgreSQL infrastructure",
+            "pros": ["Mature ecosystem", "SQL compatibility", "ACID compliance"],
+            "cons": ["More complex setup", "Higher resource usage"],
+            "use_case": "Best for organizations already using PostgreSQL",
         },
         {
-            'name': 'VictoriaMetrics',
-            'best_for': 'High-performance metrics, Prometheus compatibility',
-            'pros': ['Extremely fast', 'Low resource usage', 'Better compression'],
-            'cons': ['Newer ecosystem', 'Less tooling'],
-            'use_case': 'Best for high-volume, performance-critical deployments'
+            "name": "VictoriaMetrics",
+            "best_for": "High-performance metrics, Prometheus compatibility",
+            "pros": ["Extremely fast", "Low resource usage", "Better compression"],
+            "cons": ["Newer ecosystem", "Less tooling"],
+            "use_case": "Best for high-volume, performance-critical deployments",
         },
         {
-            'name': 'MySQL',
-            'best_for': 'Existing MySQL infrastructure, familiar SQL',
-            'pros': ['Familiar', 'Mature', 'Wide support'],
-            'cons': ['Not optimized for time-series', 'Manual optimization needed'],
-            'use_case': 'Good for organizations with existing MySQL expertise'
-        }
+            "name": "MySQL",
+            "best_for": "Existing MySQL infrastructure, familiar SQL",
+            "pros": ["Familiar", "Mature", "Wide support"],
+            "cons": ["Not optimized for time-series", "Manual optimization needed"],
+            "use_case": "Good for organizations with existing MySQL expertise",
+        },
     ]
-    
+
     for rec in recommendations:
         print(f"\n📊 {rec['name']}")
         print(f"   Best for: {rec['best_for']}")
@@ -283,34 +300,37 @@ def show_recommendations():
         print(f"   Cons: {', '.join(rec['cons'])}")
         print(f"   💡 {rec['use_case']}")
 
+
 def main():
     """Main demo function"""
     print("🌊 Thailand Water Monitor - Database Backend Demo")
     print("This demo shows how to use different database backends")
-    
+
     # Always run SQLite demo (no external dependencies)
     demo_sqlite()
-    
+
     # Check for command line arguments to run specific demos
     if len(sys.argv) > 1:
         db_type = sys.argv[1].lower()
-        
-        if db_type == 'influxdb':
+
+        if db_type == "influxdb":
             demo_influxdb()
-        elif db_type == 'postgresql':
+        elif db_type == "postgresql":
             demo_postgresql()
-        elif db_type == 'mysql':
+        elif db_type == "mysql":
             demo_mysql()
-        elif db_type == 'victoriametrics':
+        elif db_type == "victoriametrics":
             demo_victoriametrics()
-        elif db_type == 'all':
+        elif db_type == "all":
             demo_influxdb()
             demo_postgresql()
             demo_mysql()
             demo_victoriametrics()
         else:
             print(f"\nUnknown database type: {db_type}")
-            print("Available options: influxdb, postgresql, mysql, victoriametrics, all")
+            print(
+                "Available options: influxdb, postgresql, mysql, victoriametrics, all"
+            )
     else:
         print("\n💡 To test other databases, run:")
         print("   python demo_databases.py influxdb")
@@ -318,14 +338,15 @@ def main():
         print("   python demo_databases.py mysql")
         print("   python demo_databases.py victoriametrics")
         print("   python demo_databases.py all")
-    
+
     # Show recommendations
     show_recommendations()
-    
+
     print("\n" + "=" * 60)
     print("✅ Demo completed!")
     print("📖 See DATABASE_DEPLOYMENT_GUIDE.md for production setup instructions")
     print("=" * 60)
 
+
 if __name__ == "__main__":
     main()

src/exceptions.py

@@ -3,30 +3,44 @@
 Custom exceptions for water monitoring system
 """
 
+
 class WaterMonitorException(Exception):
     """Base exception for water monitoring system"""
+
     pass
 
+
 class DatabaseConnectionError(WaterMonitorException):
     """Raised when database connection fails"""
+
     pass
 
+
 class APIConnectionError(WaterMonitorException):
     """Raised when API connection fails"""
+
     pass
 
+
 class DataValidationError(WaterMonitorException):
     """Raised when data validation fails"""
+
     pass
 
+
 class ConfigurationError(WaterMonitorException):
     """Raised when configuration is invalid"""
+
     pass
 
+
 class DataParsingError(WaterMonitorException):
     """Raised when data parsing fails"""
+
     pass
 
+
 class RetryExhaustedError(WaterMonitorException):
     """Raised when all retry attempts are exhausted"""
-    pass
+
+    pass

src/forecast_history.py

@@ -0,0 +1,172 @@
+"""Persistence for issued flood forecasts.
+
+Every background precompute stores what the deployed model predicted at that
+moment — predicted 24/12/6 h peak, warning/danger probabilities, model
+version. Keyed by (as_of, station, horizon), so hourly data yields one row
+per station-horizon per hour regardless of how often the precompute runs.
+This is the operational record that lets "predicted vs actual" be graphed
+later without retraining historical models.
+"""
+
+import datetime
+import logging
+from typing import Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+
+class ForecastHistoryStore:
+    """SQL store (sqlite / postgresql / mysql), same pattern as HiiStore."""
+
+    def __init__(self, connection_string: str, db_type: str):
+        self.db_type = db_type.lower()
+        if self.db_type not in ("sqlite", "postgresql", "mysql"):
+            raise ValueError(
+                f"Forecast history requires a SQL database, got '{db_type}'"
+            )
+        self.connection_string = connection_string
+        self.engine = None
+
+    def connect(self) -> bool:
+        try:
+            from sqlalchemy import create_engine, text
+
+            self.engine = create_engine(self.connection_string, pool_pre_ping=True)
+            ddl = """
+                CREATE TABLE IF NOT EXISTS forecast_history (
+                    as_of TIMESTAMP NOT NULL,
+                    station_code VARCHAR(10) NOT NULL,
+                    horizon_hours INTEGER NOT NULL,
+                    predicted_max_level NUMERIC(8,3),
+                    p_warning NUMERIC(7,5),
+                    p_danger NUMERIC(7,5),
+                    current_level NUMERIC(8,3),
+                    model_version VARCHAR(64),
+                    source VARCHAR(16),
+                    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+                    PRIMARY KEY (as_of, station_code, horizon_hours)
+                )
+            """
+            index = (
+                "CREATE INDEX IF NOT EXISTS idx_forecast_history_station "
+                "ON forecast_history(station_code, as_of)"
+            )
+            with self.engine.begin() as conn:
+                conn.execute(text(ddl))
+                if self.db_type != "mysql":  # MySQL lacks IF NOT EXISTS for indexes
+                    conn.execute(text(index))
+            return True
+        except Exception as error:
+            logger.error(f"ForecastHistoryStore failed to connect: {error}")
+            self.engine = None
+            return False
+
+    def save_rows(self, rows: List[Dict]) -> int:
+        """Upsert forecast rows as returned by ml.predict (idempotent)."""
+        if not rows:
+            return 0
+        if not self.engine and not self.connect():
+            return 0
+        from sqlalchemy import text
+
+        cols = (
+            "(as_of, station_code, horizon_hours, predicted_max_level, "
+            "p_warning, p_danger, current_level, model_version, source)"
+        )
+        values = (
+            "(:as_of, :station_code, :horizon_hours, :predicted_max_level, "
+            ":p_warning, :p_danger, :current_level, :model_version, :source)"
+        )
+        update_cols = (
+            "predicted_max_level",
+            "p_warning",
+            "p_danger",
+            "current_level",
+            "model_version",
+            "source",
+        )
+        if self.db_type == "mysql":
+            updates = ", ".join(f"{c} = VALUES({c})" for c in update_cols)
+            sql = (
+                f"INSERT INTO forecast_history {cols} VALUES {values} "
+                f"ON DUPLICATE KEY UPDATE {updates}"
+            )
+        else:
+            updates = ", ".join(f"{c} = EXCLUDED.{c}" for c in update_cols)
+            sql = (
+                f"INSERT INTO forecast_history {cols} VALUES {values} "
+                f"ON CONFLICT (as_of, station_code, horizon_hours) "
+                f"DO UPDATE SET {updates}"
+            )
+
+        params = []
+        for row in rows:
+            as_of = row.get("as_of")
+            if isinstance(as_of, str):
+                as_of = datetime.datetime.fromisoformat(as_of)
+            if as_of is None or row.get("station_code") is None:
+                continue
+            params.append(
+                {
+                    "as_of": as_of,
+                    "station_code": row["station_code"],
+                    "horizon_hours": row.get("horizon_hours"),
+                    "predicted_max_level": row.get("predicted_max_level"),
+                    "p_warning": row.get("p_warning"),
+                    "p_danger": row.get("p_danger"),
+                    "current_level": row.get("current_level"),
+                    "model_version": row.get("model_version"),
+                    "source": row.get("source"),
+                }
+            )
+        if not params:
+            return 0
+        try:
+            with self.engine.begin() as conn:
+                conn.execute(text(sql), params)
+            return len(params)
+        except Exception as error:
+            logger.error(f"ForecastHistoryStore save failed: {error}")
+            return 0
+
+    def fetch(
+        self,
+        station_code: str,
+        start: Optional[datetime.datetime] = None,
+        end: Optional[datetime.datetime] = None,
+        horizon_hours: Optional[int] = None,
+        limit: int = 5000,
+    ) -> List[Dict]:
+        """Issued forecasts for one station, ascending by as_of."""
+        if not self.engine and not self.connect():
+            return []
+        from sqlalchemy import text
+
+        clauses = ["station_code = :code"]
+        params: Dict = {"code": station_code, "limit": limit}
+        if start is not None:
+            clauses.append("as_of >= :start")
+            params["start"] = start
+        if end is not None:
+            clauses.append("as_of <= :end")
+            params["end"] = end
+        if horizon_hours is not None:
+            clauses.append("horizon_hours = :horizon")
+            params["horizon"] = horizon_hours
+        sql = (
+            "SELECT as_of, station_code, horizon_hours, predicted_max_level, "
+            "p_warning, p_danger, current_level, model_version, source "
+            f"FROM forecast_history WHERE {' AND '.join(clauses)} "
+            "ORDER BY as_of ASC, horizon_hours ASC LIMIT :limit"
+        )
+        try:
+            with self.engine.connect() as conn:
+                rows = [dict(r._mapping) for r in conn.execute(text(sql), params)]
+            for row in rows:
+                for key, value in row.items():
+                    if hasattr(value, "is_finite"):  # Decimal -> float
+                        row[key] = float(value)
+            return rows
+        except Exception as error:
+            logger.error(f"ForecastHistoryStore fetch failed: {error}")
+            return []

src/health_check.py

@@ -3,24 +3,27 @@
 Health check system for water monitoring application
 """
 
-import time
-import threading
-from datetime import datetime, timedelta
-from typing import Dict, Any, Optional, List, Callable
-from dataclasses import dataclass
-from enum import Enum
 import logging
+import threading
+import time
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from enum import Enum
+from typing import Any, Callable, Dict, List, Optional
 
 logger = logging.getLogger(__name__)
 
+
 class HealthStatus(Enum):
     HEALTHY = "healthy"
     DEGRADED = "degraded"
     UNHEALTHY = "unhealthy"
 
+
 @dataclass
 class HealthCheckResult:
     """Result of a health check"""
+
     name: str
     status: HealthStatus
     message: str
@@ -28,238 +31,260 @@ class HealthCheckResult:
     response_time_ms: Optional[float] = None
     details: Optional[Dict[str, Any]] = None
 
+
 class HealthCheck:
     """Base health check class"""
-    
+
     def __init__(self, name: str, timeout_seconds: int = 30):
         self.name = name
         self.timeout_seconds = timeout_seconds
-    
+
     def check(self) -> HealthCheckResult:
         """Perform the health check"""
         start_time = time.time()
-        
+
         try:
             result = self._perform_check()
             response_time = (time.time() - start_time) * 1000
-            
+
             return HealthCheckResult(
                 name=self.name,
-                status=result.get('status', HealthStatus.HEALTHY),
-                message=result.get('message', 'OK'),
+                status=result.get("status", HealthStatus.HEALTHY),
+                message=result.get("message", "OK"),
                 timestamp=datetime.now(),
                 response_time_ms=response_time,
-                details=result.get('details')
+                details=result.get("details"),
             )
-            
+
         except Exception as e:
             response_time = (time.time() - start_time) * 1000
             logger.error(f"Health check {self.name} failed: {e}")
-            
+
             return HealthCheckResult(
                 name=self.name,
                 status=HealthStatus.UNHEALTHY,
                 message=f"Check failed: {str(e)}",
                 timestamp=datetime.now(),
-                response_time_ms=response_time
+                response_time_ms=response_time,
             )
-    
+
     def _perform_check(self) -> Dict[str, Any]:
         """Override this method to implement the actual check"""
         raise NotImplementedError
 
+
 class DatabaseHealthCheck(HealthCheck):
     """Health check for database connectivity"""
-    
+
     def __init__(self, db_adapter, name: str = "database"):
         super().__init__(name)
         self.db_adapter = db_adapter
-    
+
     def _perform_check(self) -> Dict[str, Any]:
         if not self.db_adapter:
             return {
-                'status': HealthStatus.UNHEALTHY,
-                'message': 'Database adapter not initialized'
+                "status": HealthStatus.UNHEALTHY,
+                "message": "Database adapter not initialized",
             }
-        
+
         try:
-            # Try to connect
-            if hasattr(self.db_adapter, 'connect'):
+            # Connect only when there is no live engine yet: connect() re-runs
+            # the CREATE TABLE DDL suite, which is far too heavy per probe.
+            if getattr(self.db_adapter, "engine", None) is None and hasattr(
+                self.db_adapter, "connect"
+            ):
                 connected = self.db_adapter.connect()
                 if not connected:
                     return {
-                        'status': HealthStatus.UNHEALTHY,
-                        'message': 'Database connection failed'
+                        "status": HealthStatus.UNHEALTHY,
+                        "message": "Database connection failed",
                     }
-            
+
             # Try to get latest data
             latest_data = self.db_adapter.get_latest_measurements(limit=1)
-            
+
             if latest_data:
-                latest_timestamp = latest_data[0].get('timestamp')
+                latest_timestamp = latest_data[0].get("timestamp")
                 if isinstance(latest_timestamp, str):
-                    latest_timestamp = datetime.fromisoformat(latest_timestamp.replace('Z', '+00:00'))
-                
+                    latest_timestamp = datetime.fromisoformat(
+                        latest_timestamp.replace("Z", "+00:00")
+                    )
+
                 # Check if data is recent (within last 2 hours)
-                if datetime.now() - latest_timestamp.replace(tzinfo=None) > timedelta(hours=2):
+                if datetime.now() - latest_timestamp.replace(tzinfo=None) > timedelta(
+                    hours=2
+                ):
                     return {
-                        'status': HealthStatus.DEGRADED,
-                        'message': f'Latest data is old: {latest_timestamp}',
-                        'details': {'latest_data_timestamp': str(latest_timestamp)}
+                        "status": HealthStatus.DEGRADED,
+                        "message": f"Latest data is old: {latest_timestamp}",
+                        "details": {"latest_data_timestamp": str(latest_timestamp)},
                     }
-            
+
             return {
-                'status': HealthStatus.HEALTHY,
-                'message': 'Database connection OK',
-                'details': {
-                    'latest_data_count': len(latest_data),
-                    'latest_timestamp': str(latest_data[0].get('timestamp')) if latest_data else None
-                }
+                "status": HealthStatus.HEALTHY,
+                "message": "Database connection OK",
+                "details": {
+                    "latest_data_count": len(latest_data),
+                    "latest_timestamp": str(latest_data[0].get("timestamp"))
+                    if latest_data
+                    else None,
+                },
             }
-            
+
         except Exception as e:
             return {
-                'status': HealthStatus.UNHEALTHY,
-                'message': f'Database check failed: {str(e)}'
+                "status": HealthStatus.UNHEALTHY,
+                "message": f"Database check failed: {str(e)}",
             }
 
+
 class APIHealthCheck(HealthCheck):
     """Health check for external API connectivity"""
-    
+
     def __init__(self, api_url: str, session, name: str = "api"):
         super().__init__(name)
+        # A liveness probe should fail fast: the default 30s timeout meant a
+        # slow upstream pinned executor threads for longer than the /health
+        # cache TTL, so the pool never drained under load.
+        self.timeout_seconds = 5
         self.api_url = api_url
         self.session = session
-    
+
     def _perform_check(self) -> Dict[str, Any]:
         try:
             # Simple GET request to check API availability
             response = self.session.get(self.api_url, timeout=self.timeout_seconds)
-            
+
             if response.status_code == 200:
                 return {
-                    'status': HealthStatus.HEALTHY,
-                    'message': 'API connection OK',
-                    'details': {
-                        'status_code': response.status_code,
-                        'response_size': len(response.content)
-                    }
+                    "status": HealthStatus.HEALTHY,
+                    "message": "API connection OK",
+                    "details": {
+                        "status_code": response.status_code,
+                        "response_size": len(response.content),
+                    },
                 }
             else:
                 return {
-                    'status': HealthStatus.DEGRADED,
-                    'message': f'API returned status {response.status_code}',
-                    'details': {'status_code': response.status_code}
+                    "status": HealthStatus.DEGRADED,
+                    "message": f"API returned status {response.status_code}",
+                    "details": {"status_code": response.status_code},
                 }
-                
+
         except Exception as e:
             return {
-                'status': HealthStatus.UNHEALTHY,
-                'message': f'API check failed: {str(e)}'
+                "status": HealthStatus.UNHEALTHY,
+                "message": f"API check failed: {str(e)}",
             }
 
+
 class MemoryHealthCheck(HealthCheck):
     """Health check for memory usage"""
-    
+
     def __init__(self, max_memory_mb: int = 1000, name: str = "memory"):
         super().__init__(name)
         self.max_memory_mb = max_memory_mb
-    
+
     def _perform_check(self) -> Dict[str, Any]:
         try:
             import psutil
+
             process = psutil.Process()
             memory_info = process.memory_info()
             memory_mb = memory_info.rss / 1024 / 1024
-            
+
             if memory_mb > self.max_memory_mb:
                 return {
-                    'status': HealthStatus.DEGRADED,
-                    'message': f'High memory usage: {memory_mb:.1f}MB',
-                    'details': {'memory_mb': memory_mb, 'max_memory_mb': self.max_memory_mb}
+                    "status": HealthStatus.DEGRADED,
+                    "message": f"High memory usage: {memory_mb:.1f}MB",
+                    "details": {
+                        "memory_mb": memory_mb,
+                        "max_memory_mb": self.max_memory_mb,
+                    },
                 }
-            
+
             return {
-                'status': HealthStatus.HEALTHY,
-                'message': f'Memory usage OK: {memory_mb:.1f}MB',
-                'details': {'memory_mb': memory_mb}
+                "status": HealthStatus.HEALTHY,
+                "message": f"Memory usage OK: {memory_mb:.1f}MB",
+                "details": {"memory_mb": memory_mb},
             }
-            
+
         except ImportError:
             return {
-                'status': HealthStatus.HEALTHY,
-                'message': 'Memory check skipped (psutil not available)'
+                "status": HealthStatus.HEALTHY,
+                "message": "Memory check skipped (psutil not available)",
             }
         except Exception as e:
             return {
-                'status': HealthStatus.UNHEALTHY,
-                'message': f'Memory check failed: {str(e)}'
+                "status": HealthStatus.UNHEALTHY,
+                "message": f"Memory check failed: {str(e)}",
             }
 
+
 class HealthCheckManager:
     """Manages multiple health checks"""
-    
+
     def __init__(self):
         self.checks: List[HealthCheck] = []
         self.last_results: Dict[str, HealthCheckResult] = {}
         self._lock = threading.Lock()
-    
+
     def add_check(self, health_check: HealthCheck):
         """Add a health check"""
         with self._lock:
             self.checks.append(health_check)
-    
+
     def run_all_checks(self) -> Dict[str, HealthCheckResult]:
         """Run all health checks"""
         results = {}
-        
+
         for check in self.checks:
             try:
                 result = check.check()
                 results[check.name] = result
-                
+
                 with self._lock:
                     self.last_results[check.name] = result
-                    
+
             except Exception as e:
                 logger.error(f"Error running health check {check.name}: {e}")
                 results[check.name] = HealthCheckResult(
                     name=check.name,
                     status=HealthStatus.UNHEALTHY,
                     message=f"Check execution failed: {str(e)}",
-                    timestamp=datetime.now()
+                    timestamp=datetime.now(),
                 )
-        
+
         return results
-    
+
     def get_overall_status(self) -> HealthStatus:
         """Get overall system health status"""
         if not self.last_results:
             return HealthStatus.UNHEALTHY
-        
+
         statuses = [result.status for result in self.last_results.values()]
-        
+
         if any(status == HealthStatus.UNHEALTHY for status in statuses):
             return HealthStatus.UNHEALTHY
         elif any(status == HealthStatus.DEGRADED for status in statuses):
             return HealthStatus.DEGRADED
         else:
             return HealthStatus.HEALTHY
-    
+
     def get_health_summary(self) -> Dict[str, Any]:
         """Get a summary of system health"""
         overall_status = self.get_overall_status()
-        
+
         return {
-            'overall_status': overall_status.value,
-            'timestamp': datetime.now().isoformat(),
-            'checks': {
+            "overall_status": overall_status.value,
+            "timestamp": datetime.now().isoformat(),
+            "checks": {
                 name: {
-                    'status': result.status.value,
-                    'message': result.message,
-                    'response_time_ms': result.response_time_ms,
-                    'timestamp': result.timestamp.isoformat()
+                    "status": result.status.value,
+                    "message": result.message,
+                    "response_time_ms": result.response_time_ms,
+                    "timestamp": result.timestamp.isoformat(),
                 }
                 for name, result in self.last_results.items()
-            }
-        }
+            },
+        }

src/hii_backfill.py

@@ -0,0 +1,222 @@
+"""Backfill historical water levels from the HII waterlevel_graph endpoint.
+
+The api-v3 waterlevel_graph archive reaches back to ~2019 with hourly
+wl_msl + discharge. This module walks a date range in chunks per station and
+upserts into hii_waterlevel (idempotent; safe to re-run and to overlap with
+the live snapshot collector). Station metadata comes from a live
+waterlevel_load fetch, so hii_wl_stations is populated/refreshed as a side
+effect.
+
+Usage: python scripts/backfill_hii_waterlevel.py --start 2019-01-01
+"""
+
+import argparse
+import datetime
+import logging
+import time
+from typing import Dict, List, Optional
+
+from .hii_collector import (
+    HiiClient,
+    HiiStore,
+    PING_BASIN_CODE,
+    _parse_datetime,
+    _to_float,
+)
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_START = datetime.date(2019, 1, 1)
+DEFAULT_CHUNK_DAYS = 365  # full-year windows verified working (8,760 rows, ~700KB)
+DEFAULT_SLEEP_SECONDS = 1.0
+
+
+def parse_graph_rows(payload: Dict) -> List[Dict]:
+    """Extract history rows from a waterlevel_graph payload (skips empty rows)."""
+    rows = (payload.get("data") or {}).get("graph_data") or []
+    records = []
+    for row in rows:
+        timestamp = _parse_datetime(row.get("datetime"))
+        wl_msl = _to_float(row.get("value"))
+        discharge = _to_float(row.get("discharge"))
+        if timestamp is None or (wl_msl is None and discharge is None):
+            continue
+        records.append(
+            {"timestamp": timestamp, "wl_msl": wl_msl, "discharge": discharge}
+        )
+    return records
+
+
+def fetch_waterlevel_history(
+    client: HiiClient,
+    station_id: int,
+    start_date: datetime.date,
+    end_date: datetime.date,
+) -> List[Dict]:
+    """Hourly wl_msl + discharge history (archive reaches back to ~2019)."""
+    payload = client.get(
+        "waterlevel_graph",
+        params={
+            "station_type": "tele_waterlevel",
+            "station_id": station_id,
+            "start_date": start_date.isoformat(),
+            "end_date": end_date.isoformat(),
+        },
+    )
+    return parse_graph_rows(payload)
+
+
+def chunk_date_range(
+    start: datetime.date, end: datetime.date, chunk_days: int
+) -> List[tuple]:
+    """Split [start, end] into inclusive (start, end) windows."""
+    chunks = []
+    cursor = start
+    while cursor <= end:
+        chunk_end = min(cursor + datetime.timedelta(days=chunk_days - 1), end)
+        chunks.append((cursor, chunk_end))
+        cursor = chunk_end + datetime.timedelta(days=1)
+    return chunks
+
+
+def select_stations(
+    station_records: List[Dict],
+    codes: Optional[List[str]] = None,
+    all_stations: bool = False,
+) -> List[Dict]:
+    """Pick stations to backfill from parsed waterlevel_load records.
+
+    Default: stations that mirror a RID gauge (rid_code) or are flagged
+    is_key_station — the ones relevant to the flood model. Explicit codes
+    match rid_code or oldcode; --all takes every station in the basin.
+    """
+    if all_stations:
+        return station_records
+    if codes:
+        wanted = {c.strip().upper() for c in codes if c.strip()}
+        return [
+            r
+            for r in station_records
+            if (r.get("rid_code") or "").upper() in wanted
+            or (r.get("oldcode") or "").upper() in wanted
+        ]
+    return [r for r in station_records if r.get("rid_code") or r.get("is_key_station")]
+
+
+def backfill(
+    store: HiiStore,
+    client: Optional[HiiClient] = None,
+    start: datetime.date = DEFAULT_START,
+    end: Optional[datetime.date] = None,
+    codes: Optional[List[str]] = None,
+    all_stations: bool = False,
+    chunk_days: int = DEFAULT_CHUNK_DAYS,
+    sleep_seconds: float = DEFAULT_SLEEP_SECONDS,
+    basin_code: int = PING_BASIN_CODE,
+) -> Dict[str, int]:
+    """Run the backfill; returns {'stations': n, 'rows': n, 'errors': n}."""
+    client = client or HiiClient()
+    end = end or datetime.date.today()
+
+    logger.info("Fetching station catalog from waterlevel_load...")
+    station_records = client.fetch_waterlevel(basin_code)
+    # Refresh station metadata (and today's snapshot) while we have it
+    store.save_waterlevel(station_records)
+
+    stations = select_stations(station_records, codes=codes, all_stations=all_stations)
+    if not stations:
+        logger.error("No stations matched the selection")
+        return {"stations": 0, "rows": 0, "errors": 0}
+
+    chunks = chunk_date_range(start, end, chunk_days)
+    logger.info(
+        f"Backfilling {len(stations)} stations x {len(chunks)} windows "
+        f"({start} .. {end}, {chunk_days}-day chunks)"
+    )
+
+    totals = {"stations": len(stations), "rows": 0, "errors": 0}
+    for station in stations:
+        sid = station["station_id"]
+        label = station.get("rid_code") or station.get("oldcode") or str(sid)
+        station_rows = 0
+        for chunk_start, chunk_end in chunks:
+            try:
+                rows = fetch_waterlevel_history(client, sid, chunk_start, chunk_end)
+                station_rows += store.save_waterlevel_history(sid, rows)
+            except Exception as e:
+                totals["errors"] += 1
+                logger.warning(
+                    f"{label}: {chunk_start}..{chunk_end} failed: {e}"
+                )
+            time.sleep(sleep_seconds)
+        totals["rows"] += station_rows
+        logger.info(f"{label} (id {sid}): {station_rows} rows saved")
+
+    logger.info(
+        f"Backfill complete: {totals['rows']} rows across "
+        f"{totals['stations']} stations, {totals['errors']} failed windows"
+    )
+    return totals
+
+
+def main(argv: Optional[List[str]] = None) -> bool:
+    parser = argparse.ArgumentParser(
+        description="Backfill hii_waterlevel from the HII waterlevel_graph archive"
+    )
+    parser.add_argument(
+        "--start",
+        type=datetime.date.fromisoformat,
+        default=DEFAULT_START,
+        help=f"First date to fetch (default {DEFAULT_START})",
+    )
+    parser.add_argument(
+        "--end",
+        type=datetime.date.fromisoformat,
+        default=None,
+        help="Last date to fetch (default today)",
+    )
+    parser.add_argument(
+        "--stations",
+        help="Comma-separated codes (rid_code or oldcode, e.g. P.1,P.67,CHM004). "
+        "Default: all RID-mirror and key stations",
+    )
+    parser.add_argument(
+        "--all",
+        action="store_true",
+        help="Backfill every Ping-basin station (125+; slow)",
+    )
+    parser.add_argument(
+        "--chunk-days", type=int, default=DEFAULT_CHUNK_DAYS, help="Window size"
+    )
+    parser.add_argument(
+        "--sleep",
+        type=float,
+        default=DEFAULT_SLEEP_SECONDS,
+        help="Pause between requests in seconds",
+    )
+    args = parser.parse_args(argv)
+
+    logging.basicConfig(
+        level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
+    )
+
+    from .config import Config
+
+    db_config = Config.get_database_config()
+    if db_config["type"] not in ("sqlite", "postgresql", "mysql"):
+        logger.error(f"Backfill requires a SQL DB_TYPE, got '{db_config['type']}'")
+        return False
+    store = HiiStore(db_config["connection_string"], db_config["type"])
+    if not store.connect():
+        return False
+
+    totals = backfill(
+        store,
+        start=args.start,
+        end=args.end,
+        codes=args.stations.split(",") if args.stations else None,
+        all_stations=args.all,
+        chunk_days=args.chunk_days,
+        sleep_seconds=args.sleep,
+    )
+    return totals["rows"] > 0 and totals["errors"] == 0

src/hii_collector.py

@@ -0,0 +1,484 @@
+"""Collector for HII/ThaiWater open api-v3 feeds (rainfall + water level).
+
+Polls the unauthenticated api-v3.thaiwater.net public endpoints, filters to the
+Ping basin, and persists to dedicated tables alongside the RID data:
+
+- hii_rain_stations / hii_rainfall        (rain_1h / rain_24h gauge telemetry)
+- hii_wl_stations   / hii_waterlevel      (independent water-level source, m MSL)
+
+Water levels are kept in a separate table (not a column on water_measurements)
+because HII reports in m MSL from a different station universe; the per-station
+``offset`` column (gauge zero in m MSL) converts to gauge datum when needed.
+
+See docs/DATA_SOURCES.md for the endpoint catalog and quirks.
+"""
+
+import datetime
+import logging
+import re
+from typing import Any, Dict, List, Optional
+
+import requests
+
+logger = logging.getLogger(__name__)
+
+HII_API_BASE = "https://api-v3.thaiwater.net/api/v1/thaiwater30/public"
+PING_BASIN_CODE = 6
+
+# Matches 'P.1', 'ridhydro_P.67', 'ridtele_TUP.14' -> canonical RID code suffix
+_RID_CODE_RE = re.compile(r"(?:^|_)(P\.\d+[A-Z]?)$")
+
+
+def _to_float(value: Any) -> Optional[float]:
+    """API numerics arrive as strings ('335.00'), numbers, or None."""
+    if value is None or value == "":
+        return None
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _parse_datetime(value: Any) -> Optional[datetime.datetime]:
+    """Timestamps are Thai local time, e.g. '2026-08-11 13:00'."""
+    if not value:
+        return None
+    for fmt in ("%Y-%m-%d %H:%M", "%Y-%m-%d %H:%M:%S"):
+        try:
+            return datetime.datetime.strptime(value, fmt)
+        except ValueError:
+            continue
+    return None
+
+
+def _name(station: Dict, lang: str) -> Optional[str]:
+    name = station.get("tele_station_name")
+    if isinstance(name, dict):
+        return name.get(lang)
+    return name if lang == "th" else None
+
+
+def rid_code_from_oldcode(oldcode: Optional[str]) -> Optional[str]:
+    """Normalize a ThaiWater oldcode to the RID P-code it mirrors, if any."""
+    if not oldcode:
+        return None
+    match = _RID_CODE_RE.search(oldcode)
+    return match.group(1) if match else None
+
+
+def parse_rain_records(
+    payload: Dict, basin_code: int = PING_BASIN_CODE
+) -> List[Dict]:
+    """Extract per-station rainfall rows from a rain_24h payload."""
+    records = []
+    for row in payload.get("data") or []:
+        basin = row.get("basin") or {}
+        if basin.get("basin_code") != basin_code:
+            continue
+        station = row.get("station") or {}
+        station_id = station.get("id")
+        timestamp = _parse_datetime(row.get("rainfall_datetime"))
+        if station_id is None or timestamp is None:
+            continue
+        records.append(
+            {
+                "station_id": station_id,
+                "oldcode": station.get("tele_station_oldcode"),
+                "name_th": _name(station, "th"),
+                "name_en": _name(station, "en"),
+                "latitude": _to_float(station.get("tele_station_lat")),
+                "longitude": _to_float(station.get("tele_station_long")),
+                "sub_basin_id": str(station.get("sub_basin_id") or "") or None,
+                "agency": ((row.get("agency") or {}).get("agency_shortname") or {}).get(
+                    "en"
+                ),
+                "timestamp": timestamp,
+                "rain_1h": _to_float(row.get("rain_1h")),
+                "rain_24h": _to_float(row.get("rain_24h")),
+            }
+        )
+    return records
+
+
+def parse_waterlevel_records(
+    payload: Dict, basin_code: int = PING_BASIN_CODE
+) -> List[Dict]:
+    """Extract per-station water-level rows from a waterlevel_load payload."""
+    data = (payload.get("waterlevel_data") or {}).get("data") or []
+    records = []
+    for row in data:
+        basin = row.get("basin") or {}
+        if basin.get("basin_code") != basin_code:
+            continue
+        station = row.get("station") or {}
+        station_id = station.get("id")
+        timestamp = _parse_datetime(row.get("waterlevel_datetime"))
+        if station_id is None or timestamp is None:
+            continue
+        oldcode = station.get("tele_station_oldcode")
+        records.append(
+            {
+                "station_id": station_id,
+                "oldcode": oldcode,
+                "rid_code": rid_code_from_oldcode(oldcode),
+                "name_th": _name(station, "th"),
+                "name_en": _name(station, "en"),
+                "latitude": _to_float(station.get("tele_station_lat")),
+                "longitude": _to_float(station.get("tele_station_long")),
+                "agency": ((row.get("agency") or {}).get("agency_shortname") or {}).get(
+                    "en"
+                ),
+                "river_name": row.get("river_name"),
+                "offset_msl": _to_float(station.get("offset")),
+                "ground_level_msl": _to_float(station.get("ground_level")),
+                "min_bank_msl": _to_float(station.get("min_bank")),
+                "critical_level_msl": _to_float(station.get("critical_level_msl")),
+                "critical_level_m": _to_float(station.get("critical_level_m")),
+                "qmax": _to_float(station.get("qmax")),
+                "is_key_station": bool(station.get("is_key_station")),
+                "timestamp": timestamp,
+                "wl_msl": _to_float(row.get("waterlevel_msl")),
+                "wl_m": _to_float(row.get("waterlevel_m")),
+                "discharge": _to_float(row.get("discharge")),
+                "flow_rate": _to_float(row.get("flow_rate")),
+                "storage_percent": _to_float(row.get("storage_percent")),
+                "situation_level": row.get("situation_level"),
+                "diff_wl_bank": _to_float(row.get("diff_wl_bank")),
+            }
+        )
+    return records
+
+
+class HiiClient:
+    """HTTP client for the open api-v3 public endpoints."""
+
+    def __init__(
+        self,
+        base_url: str = HII_API_BASE,
+        session: Optional[requests.Session] = None,
+        timeout: int = 90,
+    ):
+        self.base_url = base_url.rstrip("/")
+        self.session = session or requests.Session()
+        self.timeout = timeout
+
+    def get(self, endpoint: str, params: Optional[Dict] = None) -> Dict:
+        response = self.session.get(
+            f"{self.base_url}/{endpoint}", params=params, timeout=self.timeout
+        )
+        response.raise_for_status()
+        return response.json()
+
+    def fetch_rain(self, basin_code: int = PING_BASIN_CODE) -> List[Dict]:
+        return parse_rain_records(self.get("rain_24h"), basin_code)
+
+    def fetch_waterlevel(self, basin_code: int = PING_BASIN_CODE) -> List[Dict]:
+        return parse_waterlevel_records(self.get("waterlevel_load"), basin_code)
+
+
+class HiiStore:
+    """SQL persistence for HII feeds (sqlite / postgresql / mysql).
+
+    Reuses the app's main relational database (same connection string as
+    the RID tables) but writes to its own hii_* tables.
+    """
+
+    def __init__(self, connection_string: str, db_type: str):
+        self.db_type = db_type.lower()
+        if self.db_type not in ("sqlite", "postgresql", "mysql"):
+            raise ValueError(
+                f"HII collection requires a SQL database, got '{db_type}'"
+            )
+        self.connection_string = connection_string
+        self.engine = None
+
+    def connect(self) -> bool:
+        try:
+            from sqlalchemy import create_engine
+
+            self.engine = create_engine(self.connection_string, pool_pre_ping=True)
+            self._create_tables()
+            return True
+        except Exception as e:
+            logger.error(f"HiiStore failed to connect: {e}")
+            self.engine = None
+            return False
+
+    def _create_tables(self):
+        from sqlalchemy import text
+
+        bool_type = "BOOLEAN" if self.db_type != "mysql" else "TINYINT(1)"
+        ddl = [
+            """
+            CREATE TABLE IF NOT EXISTS hii_rain_stations (
+                id INTEGER PRIMARY KEY,
+                oldcode VARCHAR(60),
+                name_th VARCHAR(255),
+                name_en VARCHAR(255),
+                latitude NUMERIC(10,6),
+                longitude NUMERIC(10,6),
+                sub_basin_id VARCHAR(10),
+                agency VARCHAR(40),
+                updated_at TIMESTAMP
+            )
+            """,
+            # Composite natural PK (no surrogate id): TimescaleDB hypertable
+            # conversion requires every unique index to include the time column.
+            """
+            CREATE TABLE IF NOT EXISTS hii_rainfall (
+                station_id INTEGER NOT NULL,
+                timestamp TIMESTAMP NOT NULL,
+                rain_1h NUMERIC(7,2),
+                rain_24h NUMERIC(8,2),
+                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+                PRIMARY KEY (station_id, timestamp)
+            )
+            """,
+            f"""
+            CREATE TABLE IF NOT EXISTS hii_wl_stations (
+                id INTEGER PRIMARY KEY,
+                oldcode VARCHAR(60),
+                rid_code VARCHAR(10),
+                name_th VARCHAR(255),
+                name_en VARCHAR(255),
+                latitude NUMERIC(10,6),
+                longitude NUMERIC(10,6),
+                agency VARCHAR(40),
+                river_name VARCHAR(255),
+                offset_msl NUMERIC(8,3),
+                ground_level_msl NUMERIC(8,3),
+                min_bank_msl NUMERIC(8,3),
+                critical_level_msl NUMERIC(8,3),
+                critical_level_m NUMERIC(8,3),
+                qmax NUMERIC(10,2),
+                is_key_station {bool_type},
+                updated_at TIMESTAMP
+            )
+            """,
+            """
+            CREATE TABLE IF NOT EXISTS hii_waterlevel (
+                station_id INTEGER NOT NULL,
+                timestamp TIMESTAMP NOT NULL,
+                wl_msl NUMERIC(8,3),
+                wl_m NUMERIC(8,3),
+                discharge NUMERIC(10,2),
+                flow_rate NUMERIC(10,2),
+                storage_percent NUMERIC(6,2),
+                situation_level INTEGER,
+                diff_wl_bank NUMERIC(8,3),
+                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+                PRIMARY KEY (station_id, timestamp)
+            )
+            """,
+            "CREATE INDEX IF NOT EXISTS idx_hii_rainfall_ts ON hii_rainfall(timestamp)",
+            "CREATE INDEX IF NOT EXISTS idx_hii_waterlevel_ts ON hii_waterlevel(timestamp)",
+        ]
+        # MySQL (<8.0.13 semantics) lacks CREATE INDEX IF NOT EXISTS; the unique
+        # constraints already cover the hot (station_id, timestamp) lookups there.
+        if self.db_type == "mysql":
+            ddl = ddl[:4]
+        with self.engine.begin() as conn:
+            for statement in ddl:
+                conn.execute(text(statement))
+
+    def _upsert(self, table: str, key_cols: List[str], value_cols: List[str]) -> str:
+        cols = key_cols + value_cols
+        col_list = ", ".join(cols)
+        params = ", ".join(f":{c}" for c in cols)
+        if self.db_type == "sqlite":
+            return f"INSERT OR REPLACE INTO {table} ({col_list}) VALUES ({params})"
+        if self.db_type == "postgresql":
+            updates = ", ".join(f"{c} = EXCLUDED.{c}" for c in value_cols)
+            conflict = ", ".join(key_cols)
+            return (
+                f"INSERT INTO {table} ({col_list}) VALUES ({params}) "
+                f"ON CONFLICT ({conflict}) DO UPDATE SET {updates}"
+            )
+        updates = ", ".join(f"{c} = VALUES({c})" for c in value_cols)
+        return (
+            f"INSERT INTO {table} ({col_list}) VALUES ({params}) "
+            f"ON DUPLICATE KEY UPDATE {updates}"
+        )
+
+    def save_rain(self, records: List[Dict]) -> int:
+        return self._save(
+            records,
+            station_table="hii_rain_stations",
+            station_cols=[
+                "oldcode",
+                "name_th",
+                "name_en",
+                "latitude",
+                "longitude",
+                "sub_basin_id",
+                "agency",
+            ],
+            measurement_table="hii_rainfall",
+            measurement_cols=["rain_1h", "rain_24h"],
+        )
+
+    def save_waterlevel(self, records: List[Dict]) -> int:
+        return self._save(
+            records,
+            station_table="hii_wl_stations",
+            station_cols=[
+                "oldcode",
+                "rid_code",
+                "name_th",
+                "name_en",
+                "latitude",
+                "longitude",
+                "agency",
+                "river_name",
+                "offset_msl",
+                "ground_level_msl",
+                "min_bank_msl",
+                "critical_level_msl",
+                "critical_level_m",
+                "qmax",
+                "is_key_station",
+            ],
+            measurement_table="hii_waterlevel",
+            measurement_cols=[
+                "wl_msl",
+                "wl_m",
+                "discharge",
+                "flow_rate",
+                "storage_percent",
+                "situation_level",
+                "diff_wl_bank",
+            ],
+        )
+
+    def save_waterlevel_history(self, station_id: int, rows: List[Dict]) -> int:
+        """Upsert backfilled history rows, touching only wl_msl and discharge.
+
+        Live-snapshot rows for the same (station, hour) keep their extra
+        columns (storage_percent, situation_level, ...) untouched.
+        """
+        if not rows:
+            return 0
+        if not self.engine and not self.connect():
+            return 0
+        from sqlalchemy import text
+
+        cols = "(station_id, timestamp, wl_msl, discharge)"
+        values = "(:station_id, :timestamp, :wl_msl, :discharge)"
+        if self.db_type == "mysql":
+            sql = (
+                f"INSERT INTO hii_waterlevel {cols} VALUES {values} "
+                "ON DUPLICATE KEY UPDATE wl_msl = VALUES(wl_msl), "
+                "discharge = VALUES(discharge)"
+            )
+        else:  # sqlite (>=3.24) and postgresql share upsert syntax
+            sql = (
+                f"INSERT INTO hii_waterlevel {cols} VALUES {values} "
+                "ON CONFLICT (station_id, timestamp) DO UPDATE SET "
+                "wl_msl = EXCLUDED.wl_msl, discharge = EXCLUDED.discharge"
+            )
+        params = [{**row, "station_id": station_id} for row in rows]
+        try:
+            with self.engine.begin() as conn:
+                conn.execute(text(sql), params)
+            return len(params)
+        except Exception as e:
+            logger.error(f"HiiStore history save failed: {e}")
+            return 0
+
+    def _save(
+        self,
+        records: List[Dict],
+        station_table: str,
+        station_cols: List[str],
+        measurement_table: str,
+        measurement_cols: List[str],
+    ) -> int:
+        if not records:
+            return 0
+        if not self.engine and not self.connect():
+            return 0
+        from sqlalchemy import text
+
+        now = datetime.datetime.now()
+        station_sql = self._upsert(
+            station_table, ["id"], station_cols + ["updated_at"]
+        )
+        measurement_sql = self._upsert(
+            measurement_table, ["station_id", "timestamp"], measurement_cols
+        )
+        # Dedupe stations (one row per station per snapshot anyway) and build
+        # parameter dicts limited to each statement's columns.
+        stations = {}
+        measurements = []
+        for record in records:
+            sid = record["station_id"]
+            station_row = {c: record.get(c) for c in station_cols}
+            station_row.update({"id": sid, "updated_at": now})
+            stations[sid] = station_row
+            measurement_row = {c: record.get(c) for c in measurement_cols}
+            measurement_row.update(
+                {"station_id": sid, "timestamp": record["timestamp"]}
+            )
+            measurements.append(measurement_row)
+
+        try:
+            with self.engine.begin() as conn:
+                conn.execute(text(station_sql), list(stations.values()))
+                conn.execute(text(measurement_sql), measurements)
+            return len(measurements)
+        except Exception as e:
+            logger.error(f"HiiStore save to {measurement_table} failed: {e}")
+            return 0
+
+
+class HiiCollector:
+    """Fetch + persist one snapshot of both HII feeds."""
+
+    def __init__(
+        self,
+        db_config: Dict,
+        basin_code: int = PING_BASIN_CODE,
+        client: Optional[HiiClient] = None,
+    ):
+        self.client = client or HiiClient()
+        self.basin_code = basin_code
+        self.store = HiiStore(
+            connection_string=db_config["connection_string"],
+            db_type=db_config["type"],
+        )
+
+    def run_cycle(self) -> Dict[str, int]:
+        """Collect both feeds; each is independent and failure-isolated."""
+        counts = {"rainfall": 0, "waterlevel": 0}
+        try:
+            counts["rainfall"] = self.store.save_rain(
+                self.client.fetch_rain(self.basin_code)
+            )
+        except Exception as e:
+            logger.error(f"HII rainfall collection failed: {e}")
+        try:
+            counts["waterlevel"] = self.store.save_waterlevel(
+                self.client.fetch_waterlevel(self.basin_code)
+            )
+        except Exception as e:
+            logger.error(f"HII waterlevel collection failed: {e}")
+        logger.info(
+            f"HII collection: {counts['rainfall']} rainfall, "
+            f"{counts['waterlevel']} waterlevel rows saved"
+        )
+        return counts
+
+
+def create_collector_from_config() -> Optional[HiiCollector]:
+    """Build a collector from app Config; None when disabled or non-SQL DB."""
+    from .config import Config
+
+    if not Config.ENABLE_HII_COLLECTION:
+        return None
+    db_config = Config.get_database_config()
+    if db_config["type"] not in ("sqlite", "postgresql", "mysql"):
+        logger.warning(
+            f"HII collection skipped: DB_TYPE '{db_config['type']}' is not SQL"
+        )
+        return None
+    return HiiCollector(db_config, basin_code=Config.HII_BASIN_CODE)

src/logging_config.py

@@ -9,35 +9,37 @@ import os
 from datetime import datetime
 from typing import Optional
 
+
 class ColoredFormatter(logging.Formatter):
     """Colored console formatter"""
-    
+
     COLORS = {
-        'DEBUG': '\033[36m',    # Cyan
-        'INFO': '\033[32m',     # Green
-        'WARNING': '\033[33m',  # Yellow
-        'ERROR': '\033[31m',    # Red
-        'CRITICAL': '\033[35m', # Magenta
-        'RESET': '\033[0m'      # Reset
+        "DEBUG": "\033[36m",  # Cyan
+        "INFO": "\033[32m",  # Green
+        "WARNING": "\033[33m",  # Yellow
+        "ERROR": "\033[31m",  # Red
+        "CRITICAL": "\033[35m",  # Magenta
+        "RESET": "\033[0m",  # Reset
     }
-    
+
     def format(self, record):
-        if hasattr(record, 'levelname'):
-            color = self.COLORS.get(record.levelname, self.COLORS['RESET'])
+        if hasattr(record, "levelname"):
+            color = self.COLORS.get(record.levelname, self.COLORS["RESET"])
             record.levelname = f"{color}{record.levelname}{self.COLORS['RESET']}"
         return super().format(record)
 
+
 def setup_logging(
     log_level: str = "INFO",
     log_file: Optional[str] = None,
     max_file_size: int = 10 * 1024 * 1024,  # 10MB
     backup_count: int = 5,
     enable_console: bool = True,
-    enable_colors: bool = True
+    enable_colors: bool = True,
 ) -> logging.Logger:
     """
     Setup comprehensive logging configuration
-    
+
     Args:
         log_level: Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
         log_file: Path to log file (optional)
@@ -45,91 +47,89 @@ def setup_logging(
         backup_count: Number of backup files to keep
         enable_console: Whether to enable console logging
         enable_colors: Whether to enable colored console output
-    
+
     Returns:
         Configured logger instance
     """
-    
+
     # Create logs directory if it doesn't exist
     if log_file:
         log_dir = os.path.dirname(log_file)
         if log_dir and not os.path.exists(log_dir):
             os.makedirs(log_dir)
-    
+
     # Configure root logger
     logger = logging.getLogger()
     logger.setLevel(getattr(logging, log_level.upper()))
-    
+
     # Clear existing handlers
     logger.handlers.clear()
-    
+
     # Create formatters
     detailed_formatter = logging.Formatter(
-        '%(asctime)s - %(name)s - %(levelname)s - %(filename)s:%(lineno)d - %(message)s',
-        datefmt='%Y-%m-%d %H:%M:%S'
+        "%(asctime)s - %(name)s - %(levelname)s - %(filename)s:%(lineno)d - %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
     )
-    
+
     simple_formatter = logging.Formatter(
-        '%(asctime)s - %(levelname)s - %(message)s',
-        datefmt='%H:%M:%S'
+        "%(asctime)s - %(levelname)s - %(message)s", datefmt="%H:%M:%S"
     )
-    
+
     # Console handler
     if enable_console:
         console_handler = logging.StreamHandler()
-        if enable_colors and os.name != 'nt':  # Don't use colors on Windows
+        if enable_colors and os.name != "nt":  # Don't use colors on Windows
             console_formatter = ColoredFormatter(
-                '%(asctime)s - %(levelname)s - %(message)s',
-                datefmt='%H:%M:%S'
+                "%(asctime)s - %(levelname)s - %(message)s", datefmt="%H:%M:%S"
             )
         else:
             console_formatter = simple_formatter
-        
+
         console_handler.setFormatter(console_formatter)
         console_handler.setLevel(getattr(logging, log_level.upper()))
         logger.addHandler(console_handler)
-    
+
     # File handler with rotation
     if log_file:
         file_handler = logging.handlers.RotatingFileHandler(
-            log_file,
-            maxBytes=max_file_size,
-            backupCount=backup_count,
-            encoding='utf-8'
+            log_file, maxBytes=max_file_size, backupCount=backup_count, encoding="utf-8"
         )
         file_handler.setFormatter(detailed_formatter)
         file_handler.setLevel(logging.DEBUG)  # Always log everything to file
         logger.addHandler(file_handler)
-    
+
     # Add performance logger for metrics
-    perf_logger = logging.getLogger('performance')
+    perf_logger = logging.getLogger("performance")
     if log_file:
-        perf_file = log_file.replace('.log', '_performance.log')
+        perf_file = log_file.replace(".log", "_performance.log")
         perf_handler = logging.handlers.RotatingFileHandler(
             perf_file,
             maxBytes=max_file_size,
             backupCount=backup_count,
-            encoding='utf-8'
+            encoding="utf-8",
         )
         perf_formatter = logging.Formatter(
-            '%(asctime)s - %(message)s',
-            datefmt='%Y-%m-%d %H:%M:%S'
+            "%(asctime)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S"
         )
         perf_handler.setFormatter(perf_formatter)
         perf_logger.addHandler(perf_handler)
         perf_logger.setLevel(logging.INFO)
         perf_logger.propagate = False
-    
+
     return logger
 
-def log_performance_metric(operation: str, duration: float, additional_info: Optional[str] = None):
+
+def log_performance_metric(
+    operation: str, duration: float, additional_info: Optional[str] = None
+):
     """Log performance metrics"""
-    perf_logger = logging.getLogger('performance')
+    perf_logger = logging.getLogger("performance")
     message = f"PERF: {operation} took {duration:.3f}s"
     if additional_info:
         message += f" - {additional_info}"
     perf_logger.info(message)
 
+
 def get_logger(name: str) -> logging.Logger:
     """Get a logger with the specified name"""
-    return logging.getLogger(name)
+    return logging.getLogger(name)

src/main.py

@@ -5,174 +5,364 @@ Main entry point for the Thailand Water Monitor system
 
 import argparse
 import asyncio
-import sys
 import signal
+import sys
 import time
 from datetime import datetime
 from typing import Optional
 
 from .config import Config
-from .water_scraper_v3 import EnhancedWaterMonitorScraper
-from .logging_config import setup_logging, get_logger
 from .exceptions import ConfigurationError, DatabaseConnectionError
+from .logging_config import get_logger, setup_logging
 from .metrics import get_metrics_collector
+from .water_scraper_v3 import EnhancedWaterMonitorScraper
 
 logger = get_logger(__name__)
 
+
 def setup_signal_handlers(scraper: Optional[EnhancedWaterMonitorScraper] = None):
     """Setup signal handlers for graceful shutdown"""
+
     def signal_handler(signum, frame):
         logger.info(f"Received signal {signum}, shutting down gracefully...")
         if scraper:
             logger.info("Stopping scraper...")
         sys.exit(0)
-    
+
     signal.signal(signal.SIGINT, signal_handler)
     signal.signal(signal.SIGTERM, signal_handler)
 
+
 def run_test_cycle():
     """Run a single test cycle"""
     logger.info("Running test cycle...")
-    
+
     try:
         # Validate configuration
         Config.validate_config()
-        
+
         # Initialize scraper
         db_config = Config.get_database_config()
         scraper = EnhancedWaterMonitorScraper(db_config)
-        
+
         # Run single scraping cycle
         result = scraper.run_scraping_cycle()
-        
+
         if result:
             logger.info("✅ Test cycle completed successfully")
-            
+
             # Show some statistics
             latest_data = scraper.get_latest_data(5)
             if latest_data:
                 logger.info(f"Latest data points: {len(latest_data)}")
                 for data in latest_data[:3]:  # Show first 3
-                    logger.info(f"  • {data['station_code']}: {data['water_level']:.2f}m, {data['discharge']:.1f} cms")
+                    logger.info(
+                        f"  • {data['station_code']}: {data['water_level']:.2f}m, {data['discharge']:.1f} cms"
+                    )
         else:
             logger.warning("⚠️ Test cycle completed but no new data was found")
-        
+
         return True
-        
+
     except Exception as e:
         logger.error(f"❌ Test cycle failed: {e}")
         return False
 
+
 def run_continuous_monitoring():
-    """Run continuous monitoring with scheduling"""
+    """Run continuous monitoring with adaptive scheduling and alerting"""
     logger.info("Starting continuous monitoring...")
-    
+
     try:
         # Validate configuration
         Config.validate_config()
-        
+
         # Initialize scraper
         db_config = Config.get_database_config()
         scraper = EnhancedWaterMonitorScraper(db_config)
-        
+
+        # Initialize alerting system
+        from .alerting import WaterLevelAlertSystem
+
+        alerting = WaterLevelAlertSystem()
+
+        # Initialize HII/ThaiWater collector (rainfall + backup water level)
+        hii_collector = None
+        try:
+            from .hii_collector import create_collector_from_config
+
+            hii_collector = create_collector_from_config()
+            if hii_collector:
+                logger.info(
+                    "HII collection enabled (Ping-basin rainfall + water level)"
+                )
+        except Exception as e:
+            logger.error(f"HII collector initialization failed: {e}")
+
         # Setup signal handlers
         setup_signal_handlers(scraper)
-        
-        logger.info(f"Monitoring started with {Config.SCRAPING_INTERVAL_HOURS}h interval")
+
+        logger.info(
+            f"Monitoring started with {Config.SCRAPING_INTERVAL_HOURS}h interval"
+        )
+        logger.info(
+            "Adaptive retry: switches to 1-minute intervals when no data available"
+        )
+        logger.info("Alerts: automatic check after each successful data fetch")
         logger.info("Press Ctrl+C to stop")
-        
+
         # Run initial cycle
         logger.info("Running initial data collection...")
-        scraper.run_scraping_cycle()
-        
-        # Start scheduled monitoring
-        import schedule
-        
-        schedule.every(Config.SCRAPING_INTERVAL_HOURS).hours.do(scraper.run_scraping_cycle)
-        
+        initial_success = scraper.run_scraping_cycle()
+
+        # Adaptive scheduling state
+        from datetime import datetime, timedelta
+
+        retry_mode = not initial_success
+        last_successful_fetch = None if not initial_success else datetime.now()
+
+        last_hii_run = None
+        if hii_collector:
+            try:
+                hii_collector.run_cycle()
+                last_hii_run = datetime.now()
+            except Exception as e:
+                logger.error(f"HII collection failed: {e}")
+
+        if retry_mode:
+            logger.warning("No data fetched in initial run - entering retry mode")
+            next_run = datetime.now() + timedelta(minutes=1)
+        else:
+            logger.info("Initial data fetch successful - using hourly schedule")
+            next_run = (datetime.now() + timedelta(hours=1)).replace(
+                minute=0, second=0, microsecond=0
+            )
+
+        logger.info(f"Next run at {next_run.strftime('%H:%M')}")
+
         while True:
-            schedule.run_pending()
-            time.sleep(60)  # Check every minute
-            
+            current_time = datetime.now()
+
+            if current_time >= next_run:
+                logger.info("Running scheduled data collection...")
+                success = scraper.run_scraping_cycle()
+
+                # HII feeds update hourly; keep collecting on that cadence even
+                # when the RID scraper is in 1-minute retry mode.
+                if hii_collector and (
+                    last_hii_run is None
+                    or current_time - last_hii_run >= timedelta(minutes=55)
+                ):
+                    try:
+                        hii_collector.run_cycle()
+                        last_hii_run = current_time
+                    except Exception as e:
+                        logger.error(f"HII collection failed: {e}")
+
+                if success:
+                    last_successful_fetch = current_time
+
+                    # Run alert check after every successful new data fetch
+                    logger.info("Running alert check...")
+                    try:
+                        alert_results = alerting.run_alert_check()
+                        if alert_results.get("total_alerts", 0) > 0:
+                            logger.info(
+                                f"Alerts: {alert_results['total_alerts']} generated, {alert_results['sent']} sent"
+                            )
+                    except Exception as e:
+                        logger.error(f"Alert check failed: {e}")
+
+                    if retry_mode:
+                        logger.info(
+                            "✅ Data fetch successful - switching back to hourly schedule"
+                        )
+                        retry_mode = False
+                        # Schedule next run at the next full hour
+                        next_run = (current_time + timedelta(hours=1)).replace(
+                            minute=0, second=0, microsecond=0
+                        )
+                    else:
+                        # Continue hourly schedule
+                        next_run = (
+                            current_time
+                            + timedelta(hours=Config.SCRAPING_INTERVAL_HOURS)
+                        ).replace(minute=0, second=0, microsecond=0)
+
+                    logger.info(f"Next scheduled run at {next_run.strftime('%H:%M')}")
+                else:
+                    if not retry_mode:
+                        logger.warning(
+                            "⚠️ No data fetched - switching to retry mode (1-minute intervals)"
+                        )
+                        retry_mode = True
+
+                    # Schedule retry in 1 minute
+                    next_run = current_time + timedelta(minutes=1)
+                    logger.info(f"Retrying in 1 minute at {next_run.strftime('%H:%M')}")
+
+            # Sleep for 10 seconds and check again
+            time.sleep(10)
+
     except KeyboardInterrupt:
         logger.info("Monitoring stopped by user")
     except Exception as e:
         logger.error(f"Monitoring failed: {e}")
         return False
-    
+
     return True
 
-def run_gap_filling(days_back: int):
-    """Run gap filling for missing data"""
-    logger.info(f"Checking for data gaps in the last {days_back} days...")
-    
+
+def run_hii_collection():
+    """Run a single HII/ThaiWater collection cycle (rainfall + water level)"""
+    try:
+        Config.validate_config()
+        from .hii_collector import create_collector_from_config
+
+        collector = create_collector_from_config()
+        if not collector:
+            logger.error(
+                "HII collection unavailable (disabled via ENABLE_HII_COLLECTION "
+                "or DB_TYPE is not a SQL database)"
+            )
+            return False
+
+        counts = collector.run_cycle()
+        return counts["rainfall"] > 0 or counts["waterlevel"] > 0
+    except Exception as e:
+        logger.error(f"HII collection failed: {e}")
+        return False
+
+
+def run_gap_filling(days_back: Optional[int]):
+    """Run gap filling for missing data (days_back=None scans the whole range)"""
+    if days_back is None:
+        logger.info("Checking for data gaps across the whole data range...")
+    else:
+        logger.info(f"Checking for data gaps in the last {days_back} days...")
+
     try:
         # Validate configuration
         Config.validate_config()
-        
+
         # Initialize scraper
         db_config = Config.get_database_config()
         scraper = EnhancedWaterMonitorScraper(db_config)
-        
+
         # Fill gaps
         filled_count = scraper.fill_data_gaps(days_back)
-        
+
         if filled_count > 0:
             logger.info(f"✅ Filled {filled_count} missing data points")
         else:
             logger.info("✅ No data gaps found")
-        
+
         return True
-        
+
     except Exception as e:
         logger.error(f"❌ Gap filling failed: {e}")
         return False
 
+
 def run_data_update(days_back: int):
     """Update existing data with latest values"""
     logger.info(f"Updating existing data for the last {days_back} days...")
-    
+
     try:
         # Validate configuration
         Config.validate_config()
-        
+
         # Initialize scraper
         db_config = Config.get_database_config()
         scraper = EnhancedWaterMonitorScraper(db_config)
-        
+
         # Update data
         updated_count = scraper.update_existing_data(days_back)
-        
+
         if updated_count > 0:
             logger.info(f"✅ Updated {updated_count} data points")
         else:
             logger.info("✅ No data updates needed")
-        
+
         return True
-        
+
     except Exception as e:
         logger.error(f"❌ Data update failed: {e}")
         return False
 
+
+def run_historical_import(
+    start_date_str: str, end_date_str: str, skip_existing: bool = True
+):
+    """Import historical data for a date range"""
+    try:
+        # Parse dates
+        start_date = datetime.strptime(start_date_str, "%Y-%m-%d")
+        end_date = datetime.strptime(end_date_str, "%Y-%m-%d")
+
+        if start_date > end_date:
+            logger.error("Start date must be before or equal to end date")
+            return False
+
+        logger.info(
+            f"Importing historical data from {start_date.date()} to {end_date.date()}"
+        )
+        if skip_existing:
+            logger.info("Skipping dates that already have data")
+
+        # Validate configuration
+        Config.validate_config()
+
+        # Initialize scraper
+        db_config = Config.get_database_config()
+        scraper = EnhancedWaterMonitorScraper(db_config)
+
+        # Import historical data
+        imported_count = scraper.import_historical_data(
+            start_date, end_date, skip_existing
+        )
+
+        if imported_count > 0:
+            logger.info(f"✅ Imported {imported_count} historical data points")
+        else:
+            logger.info("✅ No new data imported")
+
+        return True
+
+    except ValueError as e:
+        logger.error(f"❌ Invalid date format. Use YYYY-MM-DD: {e}")
+        return False
+    except Exception as e:
+        logger.error(f"❌ Historical import failed: {e}")
+        return False
+
+
 def run_web_api():
     """Run the FastAPI web interface"""
     logger.info("Starting web API server...")
-    
+
     try:
         import uvicorn
-        from .web_api import app
-        
+
         # Validate configuration
         Config.validate_config()
-        
-        # Run the server
-        uvicorn.run(
-            app,
-            host="0.0.0.0",
-            port=8000,
-            log_config=None  # Use our custom logging
-        )
-        
+
+        workers = max(1, Config.WEB_WORKERS)
+        if workers > 1:
+            # Multi-worker needs the app as an import string; a localhost lock
+            # port keeps background collection in exactly one worker.
+            uvicorn.run(
+                "src.web_api:app",
+                host="0.0.0.0",
+                port=8000,
+                workers=workers,
+                log_config=None,
+            )
+        else:
+            from .web_api import app
+
+            uvicorn.run(app, host="0.0.0.0", port=8000, log_config=None)
+
     except ImportError:
         logger.error("FastAPI not installed. Run: pip install fastapi uvicorn")
         return False
@@ -180,52 +370,136 @@ def run_web_api():
         logger.error(f"Web API failed: {e}")
         return False
 
+
+def run_alert_check():
+    """Run water level alert check"""
+    logger.info("Running water level alert check...")
+
+    try:
+        from .alerting import WaterLevelAlertSystem
+
+        # Initialize alerting system
+        alerting = WaterLevelAlertSystem()
+
+        # Run alert check
+        results = alerting.run_alert_check()
+
+        if "error" in results:
+            logger.error("❌ Alert check failed due to database connection")
+            return False
+
+        logger.info(f"✅ Alert check completed:")
+        logger.info(f"  • Water level alerts: {results['water_alerts']}")
+        logger.info(f"  • Data freshness alerts: {results['data_alerts']}")
+        logger.info(f"  • Total alerts generated: {results['total_alerts']}")
+        logger.info(f"  • Alerts sent: {results['sent']}")
+
+        return True
+
+    except Exception as e:
+        logger.error(f"❌ Alert check failed: {e}")
+        return False
+
+
+def run_alert_test():
+    """Send test alert message"""
+    logger.info("Sending test alert message...")
+
+    try:
+        from .alerting import WaterLevelAlertSystem
+
+        # Initialize alerting system
+        alerting = WaterLevelAlertSystem()
+
+        if not alerting.matrix_notifier:
+            logger.error("❌ Matrix notifier not configured")
+            logger.info(
+                "Please set MATRIX_ACCESS_TOKEN and MATRIX_ROOM_ID in your .env file"
+            )
+            return False
+
+        # Send test message
+        test_message = "🧪 **Test Alert**\n\nThis is a test message from the Northern Thailand Ping River Monitor.\n\nIf you received this, Matrix notifications are working correctly!"
+        success = alerting.matrix_notifier.send_message(test_message)
+
+        if success:
+            logger.info("✅ Test alert message sent successfully")
+        else:
+            logger.error("❌ Test alert message failed to send")
+
+        return success
+
+    except Exception as e:
+        logger.error(f"❌ Test alert failed: {e}")
+        return False
+
+
 def show_status():
     """Show current system status"""
     logger.info("=== Northern Thailand Ping River Monitor Status ===")
-    
+
     try:
         # Show configuration
         Config.print_settings()
-        
+
         # Test database connection
         logger.info("\n=== Database Connection Test ===")
         db_config = Config.get_database_config()
         scraper = EnhancedWaterMonitorScraper(db_config)
-        
+
         if scraper.db_adapter:
             logger.info("✅ Database connection successful")
-            
+
             # Show latest data
             latest_data = scraper.get_latest_data(3)
             if latest_data:
                 logger.info(f"\n=== Latest Data ({len(latest_data)} points) ===")
                 for data in latest_data:
-                    timestamp = data['timestamp']
+                    timestamp = data["timestamp"]
                     if isinstance(timestamp, str):
-                        timestamp = datetime.fromisoformat(timestamp.replace('Z', '+00:00'))
-                    logger.info(f"  • {data['station_code']} ({timestamp}): {data['water_level']:.2f}m")
+                        timestamp = datetime.fromisoformat(
+                            timestamp.replace("Z", "+00:00")
+                        )
+                    logger.info(
+                        f"  • {data['station_code']} ({timestamp}): {data['water_level']:.2f}m"
+                    )
             else:
                 logger.info("No data found in database")
         else:
             logger.error("❌ Database connection failed")
-        
+
+        # Test alerting system
+        logger.info("\n=== Alerting System Status ===")
+        try:
+            from .alerting import WaterLevelAlertSystem
+
+            alerting = WaterLevelAlertSystem()
+
+            if alerting.matrix_notifier:
+                logger.info("✅ Matrix notifications configured")
+            else:
+                logger.warning("⚠️ Matrix notifications not configured")
+                logger.info("Set MATRIX_ACCESS_TOKEN and MATRIX_ROOM_ID in .env file")
+        except Exception as e:
+            logger.error(f"❌ Alerting system error: {e}")
+
         # Show metrics if available
         metrics_collector = get_metrics_collector()
         metrics = metrics_collector.get_all_metrics()
-        
+
         if any(metrics.values()):
             logger.info("\n=== Metrics Summary ===")
             for metric_type, values in metrics.items():
                 if values:
                     logger.info(f"{metric_type.title()}: {len(values)} metrics")
-        
+
         return True
-        
+
     except Exception as e:
         logger.error(f"Status check failed: {e}")
         return False
 
+
 def main():
     """Main entry point"""
     parser = argparse.ArgumentParser(
@@ -237,93 +511,134 @@ Examples:
   %(prog)s                          # Run continuous monitoring
   %(prog)s --web-api                # Start web API server
   %(prog)s --fill-gaps 7            # Fill missing data for last 7 days
+  %(prog)s --fill-gaps all          # Fill missing data across the whole data range
   %(prog)s --update-data 2          # Update existing data for last 2 days
+  %(prog)s --import-historical 2024-01-01 2024-01-31  # Import historical data
   %(prog)s --status                 # Show system status
-        """
+  %(prog)s --alert-check            # Check water levels and send alerts
+  %(prog)s --alert-test             # Send test Matrix message
+        """,
     )
-    
+
+    parser.add_argument("--test", action="store_true", help="Run a single test cycle")
+
     parser.add_argument(
-        "--test", 
-        action="store_true",
-        help="Run a single test cycle"
+        "--web-api", action="store_true", help="Start the web API server"
     )
-    
-    parser.add_argument(
-        "--web-api",
-        action="store_true", 
-        help="Start the web API server"
-    )
-    
+
     parser.add_argument(
         "--fill-gaps",
-        type=int,
-        metavar="DAYS",
-        help="Fill missing data gaps for the specified number of days back"
+        metavar="DAYS|all",
+        help=(
+            "Fill missing data gaps for the specified number of days back, "
+            "or 'all' to scan the entire data range in the database"
+        ),
     )
-    
+
     parser.add_argument(
         "--update-data",
         type=int,
-        metavar="DAYS", 
-        help="Update existing data for the specified number of days back"
+        metavar="DAYS",
+        help="Update existing data for the specified number of days back",
     )
-    
+
     parser.add_argument(
-        "--status",
-        action="store_true",
-        help="Show current system status"
+        "--import-historical",
+        nargs=2,
+        metavar=("START_DATE", "END_DATE"),
+        help="Import historical data for date range (YYYY-MM-DD format)",
     )
-    
+
+    parser.add_argument(
+        "--force-overwrite",
+        action="store_true",
+        help="Overwrite existing data when importing historical data",
+    )
+
+    parser.add_argument(
+        "--status", action="store_true", help="Show current system status"
+    )
+
+    parser.add_argument(
+        "--alert-check", action="store_true", help="Run water level alert check"
+    )
+
+    parser.add_argument(
+        "--alert-test", action="store_true", help="Send test alert message to Matrix"
+    )
+
+    parser.add_argument(
+        "--collect-hii",
+        action="store_true",
+        help="Run one HII/ThaiWater collection cycle (rainfall + water level)",
+    )
+
     parser.add_argument(
         "--log-level",
         choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
         default=Config.LOG_LEVEL,
-        help="Set logging level"
+        help="Set logging level",
     )
-    
-    parser.add_argument(
-        "--log-file",
-        default=Config.LOG_FILE,
-        help="Log file path"
-    )
-    
+
+    parser.add_argument("--log-file", default=Config.LOG_FILE, help="Log file path")
+
     args = parser.parse_args()
-    
+
     # Setup logging
     setup_logging(
         log_level=args.log_level,
         log_file=args.log_file,
         enable_console=True,
-        enable_colors=True
+        enable_colors=True,
     )
-    
+
     logger.info("🏔️ Northern Thailand Ping River Monitor starting...")
     logger.info(f"Version: 3.1.3")
     logger.info(f"Log level: {args.log_level}")
-    
+
     try:
         success = False
-        
+
         if args.test:
             success = run_test_cycle()
         elif args.web_api:
             success = run_web_api()
         elif args.fill_gaps is not None:
-            success = run_gap_filling(args.fill_gaps)
+            if args.fill_gaps.lower() == "all":
+                success = run_gap_filling(None)
+            else:
+                try:
+                    success = run_gap_filling(int(args.fill_gaps))
+                except ValueError:
+                    logger.error(
+                        f"Invalid --fill-gaps value '{args.fill_gaps}': "
+                        "expected a number of days or 'all'"
+                    )
+                    sys.exit(1)
         elif args.update_data is not None:
             success = run_data_update(args.update_data)
+        elif args.import_historical is not None:
+            start_date, end_date = args.import_historical
+            skip_existing = not args.force_overwrite
+            success = run_historical_import(start_date, end_date, skip_existing)
         elif args.status:
             success = show_status()
+        elif args.alert_check:
+            success = run_alert_check()
+        elif args.alert_test:
+            success = run_alert_test()
+        elif args.collect_hii:
+            success = run_hii_collection()
         else:
             success = run_continuous_monitoring()
-        
+
         if success:
             logger.info("✅ Operation completed successfully")
             sys.exit(0)
         else:
             logger.error("❌ Operation failed")
             sys.exit(1)
-            
+
     except ConfigurationError as e:
         logger.error(f"Configuration error: {e}")
         sys.exit(1)
@@ -334,5 +649,6 @@ Examples:
         logger.error(f"Unexpected error: {e}")
         sys.exit(1)
 
+
 if __name__ == "__main__":
-    main()
+    main()

src/metrics.py

@@ -3,26 +3,29 @@
 Metrics collection and monitoring for water monitoring system
 """
 
-import time
-import threading
-from datetime import datetime, timedelta
-from typing import Dict, Any, Optional, List
-from dataclasses import dataclass, field
-from collections import defaultdict, deque
 import logging
+import threading
+import time
+from collections import defaultdict, deque
+from dataclasses import dataclass, field
+from datetime import datetime, timedelta
+from typing import Any, Dict, List, Optional
 
 logger = logging.getLogger(__name__)
 
+
 @dataclass
 class MetricPoint:
     """Single metric data point"""
+
     timestamp: datetime
     value: float
     labels: Dict[str, str] = field(default_factory=dict)
 
+
 class MetricsCollector:
     """Collects and manages application metrics"""
-    
+
     def __init__(self, retention_hours: int = 24):
         self.retention_hours = retention_hours
         self.metrics: Dict[str, deque] = defaultdict(lambda: deque(maxlen=1000))
@@ -30,26 +33,36 @@ class MetricsCollector:
         self.gauges: Dict[str, float] = defaultdict(float)
         self.histograms: Dict[str, List[float]] = defaultdict(list)
         self._lock = threading.Lock()
-        
+
         # Start cleanup thread
-        self._cleanup_thread = threading.Thread(target=self._cleanup_old_metrics, daemon=True)
+        self._cleanup_thread = threading.Thread(
+            target=self._cleanup_old_metrics, daemon=True
+        )
         self._cleanup_thread.start()
-    
-    def increment_counter(self, name: str, value: float = 1.0, labels: Optional[Dict[str, str]] = None):
+
+    def increment_counter(
+        self, name: str, value: float = 1.0, labels: Optional[Dict[str, str]] = None
+    ):
         """Increment a counter metric"""
         with self._lock:
             key = self._make_key(name, labels)
             self.counters[key] += value
-            self.metrics[key].append(MetricPoint(datetime.now(), self.counters[key], labels or {}))
-    
-    def set_gauge(self, name: str, value: float, labels: Optional[Dict[str, str]] = None):
+            self.metrics[key].append(
+                MetricPoint(datetime.now(), self.counters[key], labels or {})
+            )
+
+    def set_gauge(
+        self, name: str, value: float, labels: Optional[Dict[str, str]] = None
+    ):
         """Set a gauge metric"""
         with self._lock:
             key = self._make_key(name, labels)
             self.gauges[key] = value
             self.metrics[key].append(MetricPoint(datetime.now(), value, labels or {}))
-    
-    def record_histogram(self, name: str, value: float, labels: Optional[Dict[str, str]] = None):
+
+    def record_histogram(
+        self, name: str, value: float, labels: Optional[Dict[str, str]] = None
+    ):
         """Record a histogram value"""
         with self._lock:
             key = self._make_key(name, labels)
@@ -57,73 +70,77 @@ class MetricsCollector:
             # Keep only recent values
             if len(self.histograms[key]) > 1000:
                 self.histograms[key] = self.histograms[key][-1000:]
-            
+
             self.metrics[key].append(MetricPoint(datetime.now(), value, labels or {}))
-    
+
     def get_counter(self, name: str, labels: Optional[Dict[str, str]] = None) -> float:
         """Get current counter value"""
         key = self._make_key(name, labels)
         return self.counters.get(key, 0.0)
-    
+
     def get_gauge(self, name: str, labels: Optional[Dict[str, str]] = None) -> float:
         """Get current gauge value"""
         key = self._make_key(name, labels)
         return self.gauges.get(key, 0.0)
-    
-    def get_histogram_stats(self, name: str, labels: Optional[Dict[str, str]] = None) -> Dict[str, float]:
+
+    def get_histogram_stats(
+        self, name: str, labels: Optional[Dict[str, str]] = None
+    ) -> Dict[str, float]:
         """Get histogram statistics"""
         key = self._make_key(name, labels)
         values = self.histograms.get(key, [])
-        
+
         if not values:
-            return {'count': 0, 'sum': 0, 'avg': 0, 'min': 0, 'max': 0}
-        
+            return {"count": 0, "sum": 0, "avg": 0, "min": 0, "max": 0}
+
         return {
-            'count': len(values),
-            'sum': sum(values),
-            'avg': sum(values) / len(values),
-            'min': min(values),
-            'max': max(values)
+            "count": len(values),
+            "sum": sum(values),
+            "avg": sum(values) / len(values),
+            "min": min(values),
+            "max": max(values),
         }
-    
+
     def get_all_metrics(self) -> Dict[str, Any]:
         """Get all current metrics"""
         with self._lock:
             return {
-                'counters': dict(self.counters),
-                'gauges': dict(self.gauges),
-                'histograms': {k: self.get_histogram_stats(k) for k in self.histograms}
+                "counters": dict(self.counters),
+                "gauges": dict(self.gauges),
+                "histograms": {k: self.get_histogram_stats(k) for k in self.histograms},
             }
-    
+
     def _make_key(self, name: str, labels: Optional[Dict[str, str]]) -> str:
         """Create a unique key for metric with labels"""
         if not labels:
             return name
-        
-        label_str = ','.join(f"{k}={v}" for k, v in sorted(labels.items()))
+
+        label_str = ",".join(f"{k}={v}" for k, v in sorted(labels.items()))
         return f"{name}{{{label_str}}}"
-    
+
     def _cleanup_old_metrics(self):
         """Clean up old metric data points"""
         while True:
             try:
                 cutoff_time = datetime.now() - timedelta(hours=self.retention_hours)
-                
+
                 with self._lock:
                     for metric_name, points in self.metrics.items():
                         # Remove old points
                         while points and points[0].timestamp < cutoff_time:
                             points.popleft()
-                
+
                 time.sleep(3600)  # Run cleanup every hour
-                
+
             except Exception as e:
                 logger.error(f"Error in metrics cleanup: {e}")
                 time.sleep(60)  # Wait a minute before retrying
 
+
 # Global metrics collector instance
 _metrics_collector = None
 
+
 def get_metrics_collector() -> MetricsCollector:
     """Get the global metrics collector instance"""
     global _metrics_collector
@@ -131,41 +148,51 @@ def get_metrics_collector() -> MetricsCollector:
         _metrics_collector = MetricsCollector()
     return _metrics_collector
 
+
 # Convenience functions
-def increment_counter(name: str, value: float = 1.0, labels: Optional[Dict[str, str]] = None):
+def increment_counter(
+    name: str, value: float = 1.0, labels: Optional[Dict[str, str]] = None
+):
     """Increment a counter metric"""
     get_metrics_collector().increment_counter(name, value, labels)
 
+
 def set_gauge(name: str, value: float, labels: Optional[Dict[str, str]] = None):
     """Set a gauge metric"""
     get_metrics_collector().set_gauge(name, value, labels)
 
+
 def record_histogram(name: str, value: float, labels: Optional[Dict[str, str]] = None):
     """Record a histogram value"""
     get_metrics_collector().record_histogram(name, value, labels)
 
+
 class Timer:
     """Context manager for timing operations"""
-    
+
     def __init__(self, metric_name: str, labels: Optional[Dict[str, str]] = None):
         self.metric_name = metric_name
         self.labels = labels
         self.start_time = None
-    
+
     def __enter__(self):
         self.start_time = time.time()
         return self
-    
+
     def __exit__(self, exc_type, exc_val, exc_tb):
         if self.start_time:
             duration = time.time() - self.start_time
             record_histogram(self.metric_name, duration, self.labels)
 
+
 def timer(metric_name: str, labels: Optional[Dict[str, str]] = None):
     """Decorator for timing function execution"""
+
     def decorator(func):
         def wrapper(*args, **kwargs):
             with Timer(metric_name, labels):
                 return func(*args, **kwargs)
+
         return wrapper
-    return decorator
+
+    return decorator

src/ml/__init__.py

@@ -0,0 +1 @@
+"""Flood forecasting ML package: data loading, feature/label engineering, training, and prediction."""

src/ml/data.py

@@ -0,0 +1,362 @@
+"""Loaders for flood-model training/inference data.
+
+Primary path reads raw measurements straight from PostgreSQL (keeping NULL
+discharge as NULL). HTTP fallback goes through the public API's history
+endpoint, which backfills missing discharge with a synthetic rating-curve
+estimate -- callers are told about that via the `discharge_maybe_synthetic`
+cache metadata flag.
+"""
+
+import datetime
+import gzip
+import json
+import logging
+import os
+from pathlib import Path
+from typing import Dict, List, Optional
+
+import pandas as pd
+from sqlalchemy import create_engine, text
+
+from ..config import Config
+from .features import UPSTREAM_LEADS
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_API_URL = "http://100.81.167.42:8000"
+# Anchored to the repo root so training/prediction work from any CWD; a relative
+# path here silently produced 0 rows when the CLI ran outside the repo root.
+CACHE_DIR = Path(__file__).resolve().parents[2] / "models" / "cache"
+_MEASUREMENT_COLUMNS = ["timestamp", "station_code", "water_level", "discharge"]
+
+
+def resolve_db_url(db_url: Optional[str] = None) -> Optional[str]:
+    """Resolve a Postgres connection string: explicit param > FLOOD_ML_DB_URL env >
+    Config's postgresql connection string > None (caller should fall back to HTTP)."""
+    if db_url:
+        return db_url
+
+    env_url = os.getenv("FLOOD_ML_DB_URL")
+    if env_url:
+        return env_url
+
+    try:
+        db_config = Config.get_database_config()
+    except Exception as error:
+        logger.warning(f"Could not resolve database config: {error}")
+        return None
+
+    if db_config.get("type") == "postgresql":
+        return db_config.get("connection_string")
+    return None
+
+
+def _default_stations() -> List[str]:
+    return list(UPSTREAM_LEADS.keys())
+
+
+def _normalize_long(df: pd.DataFrame) -> pd.DataFrame:
+    if df.empty:
+        return pd.DataFrame(columns=_MEASUREMENT_COLUMNS)
+    df = df.copy()
+    df["timestamp"] = pd.to_datetime(df["timestamp"]).dt.floor("h")
+    df["water_level"] = pd.to_numeric(df["water_level"], errors="coerce")
+    df["discharge"] = pd.to_numeric(df["discharge"], errors="coerce")
+    df = df.drop_duplicates(subset=["station_code", "timestamp"], keep="last")
+    df = df.sort_values("timestamp").reset_index(drop=True)
+    return df[_MEASUREMENT_COLUMNS]
+
+
+def _fetch_from_db(
+    db_url: str,
+    stations: Optional[List[str]],
+    start: Optional[datetime.datetime],
+    end: Optional[datetime.datetime],
+) -> pd.DataFrame:
+    engine = create_engine(db_url, pool_pre_ping=True)
+    query = (
+        "SELECT m.timestamp, s.station_code, m.water_level, m.discharge "
+        "FROM water_measurements m JOIN stations s ON m.station_id = s.id WHERE 1=1"
+    )
+    params: Dict = {}
+    if start is not None:
+        query += " AND m.timestamp >= :start_time"
+        params["start_time"] = start
+    if end is not None:
+        query += " AND m.timestamp <= :end_time"
+        params["end_time"] = end
+    if stations:
+        placeholders = ", ".join(f":station_{i}" for i in range(len(stations)))
+        query += f" AND s.station_code IN ({placeholders})"
+        for i, code in enumerate(stations):
+            params[f"station_{i}"] = code
+    query += " ORDER BY m.timestamp"
+
+    with engine.connect() as connection:
+        df = pd.read_sql(text(query), connection, params=params)
+    return _normalize_long(df)
+
+
+# Stations whose HII mirror is the SAME telemetry (corr ≈ 1.000, median diff
+# == station offset exactly — validated 2026-08-11) plus P.81, where the HII
+# twin reads the same river with a bias (corr 0.906, MAE 19 cm) that the
+# dynamic overlap offset corrects. P.76/P.77/P.85/P.87 HII twins are DIFFERENT
+# physical sensors (corr 0.25-0.62) and must never be merged into RID series.
+HII_FILL_STATIONS = (
+    "P.1",
+    "P.103",
+    "P.20",
+    "P.4A",
+    "P.67",
+    "P.75",
+    "P.82",
+    "P.84",
+    "P.92",
+    "P.81",
+)
+_HII_EXACT_MIRRORS = frozenset(HII_FILL_STATIONS) - {"P.81"}
+_HII_MIN_OVERLAP_HOURS = 168
+
+
+def _fetch_hii_levels(
+    db_url: str,
+    stations: List[str],
+    start: Optional[datetime.datetime],
+    end: Optional[datetime.datetime],
+) -> pd.DataFrame:
+    engine = create_engine(db_url, pool_pre_ping=True)
+    query = (
+        "SELECT m.timestamp, s.rid_code AS station_code, m.wl_msl, m.discharge "
+        "FROM hii_waterlevel m JOIN hii_wl_stations s ON s.id = m.station_id "
+        "WHERE s.rid_code IS NOT NULL"
+    )
+    params: Dict = {}
+    if start is not None:
+        query += " AND m.timestamp >= :start_time"
+        params["start_time"] = start
+    if end is not None:
+        query += " AND m.timestamp <= :end_time"
+        params["end_time"] = end
+    placeholders = ", ".join(f":station_{i}" for i in range(len(stations)))
+    query += f" AND s.rid_code IN ({placeholders})"
+    for i, code in enumerate(stations):
+        params[f"station_{i}"] = code
+
+    with engine.connect() as connection:
+        df = pd.read_sql(text(query), connection, params=params)
+    df = df.dropna(subset=["wl_msl"])
+    if df.empty:
+        return df
+    df["timestamp"] = pd.to_datetime(df["timestamp"]).dt.floor("h")
+    df["wl_msl"] = pd.to_numeric(df["wl_msl"], errors="coerce")
+    df["discharge"] = pd.to_numeric(df["discharge"], errors="coerce")
+    df = df.sort_values("timestamp").drop_duplicates(
+        subset=["station_code", "timestamp"], keep="last"
+    )
+    return df
+
+
+def fill_from_hii(
+    df: pd.DataFrame,
+    db_url: str,
+    start: Optional[datetime.datetime] = None,
+    end: Optional[datetime.datetime] = None,
+    stations: Optional[List[str]] = None,
+    min_overlap_hours: int = _HII_MIN_OVERLAP_HOURS,
+) -> pd.DataFrame:
+    """Fill missing (station, hour) rows from the HII mirror telemetry.
+
+    In-memory only — water_measurements is never written. Each station's
+    MSL→gauge offset is derived from the overlap between the two series
+    (median of wl_msl − water_level over ≥ `min_overlap_hours` shared hours),
+    which reproduces the published offset for exact mirrors and bias-corrects
+    P.81. Discharge is copied only for exact mirrors; P.81 fills get NaN
+    discharge (its discharge bias was never validated). Failures degrade to
+    returning `df` unchanged, so DBs without hii_* tables keep working.
+    """
+    codes = [c for c in (stations or HII_FILL_STATIONS) if c in set(df["station_code"])]
+    if not codes:
+        return df
+    try:
+        hii = _fetch_hii_levels(db_url, codes, start, end)
+    except Exception as error:
+        logger.warning(f"HII gap-fill skipped (fetch failed): {error}")
+        return df
+    if hii.empty:
+        return df
+
+    fills = []
+    for code, mirror in hii.groupby("station_code"):
+        base = df[df["station_code"] == code]
+        overlap = base.merge(
+            mirror[["timestamp", "wl_msl"]], on="timestamp", how="inner"
+        ).dropna(subset=["water_level", "wl_msl"])
+        if len(overlap) < min_overlap_hours:
+            continue
+        offset = (overlap["wl_msl"] - overlap["water_level"]).median()
+        # Hours the RID series lacks entirely OR carries only a NaN level;
+        # _normalize_long keeps the later (fill) row on collision.
+        present = base.loc[base["water_level"].notna(), "timestamp"]
+        missing = mirror[~mirror["timestamp"].isin(present)]
+        if missing.empty:
+            continue
+        fill = pd.DataFrame(
+            {
+                "timestamp": missing["timestamp"],
+                "station_code": code,
+                "water_level": missing["wl_msl"] - offset,
+                "discharge": missing["discharge"]
+                if code in _HII_EXACT_MIRRORS
+                else float("nan"),
+            }
+        )
+        fills.append(fill)
+        logger.info(
+            f"HII gap-fill {code}: +{len(fill)} hours (offset {offset:.3f} m)"
+        )
+    if not fills:
+        return df
+    return _normalize_long(pd.concat([df] + fills, ignore_index=True))
+
+
+def _fetch_station_from_api(
+    api_url: str, station_code: str, hours: int, limit: int = 100000
+) -> pd.DataFrame:
+    import requests
+
+    response = requests.get(
+        f"{api_url}/measurements/history/{station_code}",
+        params={"hours": hours, "limit": limit},
+        timeout=30,
+    )
+    response.raise_for_status()
+    rows = response.json()
+    for row in rows:
+        row["station_code"] = station_code
+    return pd.DataFrame(rows, columns=_MEASUREMENT_COLUMNS + ["discharge_percent"])
+
+
+def _fetch_from_api(
+    api_url: str,
+    stations: List[str],
+    start: Optional[datetime.datetime],
+    end: Optional[datetime.datetime],
+) -> pd.DataFrame:
+    now = datetime.datetime.now()
+    reference_end = end or now
+    reference_start = start or (reference_end - datetime.timedelta(days=365 * 8))
+    hours = max(1, int((reference_end - reference_start).total_seconds() // 3600) + 1)
+
+    frames = []
+    for code in stations:
+        try:
+            frames.append(_fetch_station_from_api(api_url, code, hours))
+        except Exception as error:
+            logger.warning(f"HTTP fallback failed for station {code}: {error}")
+    if not frames:
+        return pd.DataFrame(columns=_MEASUREMENT_COLUMNS)
+    df = pd.concat(frames, ignore_index=True)
+    return _normalize_long(df)
+
+
+def _write_cache(
+    df: pd.DataFrame, cache_dir: Path, source: str, discharge_maybe_synthetic: bool
+) -> None:
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    for code, group in df.groupby("station_code"):
+        path = cache_dir / f"{code}.csv.gz"
+        with gzip.open(path, "wt", encoding="utf-8", newline="") as handle:
+            group.to_csv(handle, index=False)
+    meta = {
+        "fetched_at": datetime.datetime.now().isoformat(),
+        "source": source,
+        "discharge_maybe_synthetic": discharge_maybe_synthetic,
+    }
+    with open(cache_dir / "meta.json", "w", encoding="utf-8") as handle:
+        json.dump(meta, handle)
+
+
+def _read_cache(cache_dir: Path, stations: Optional[List[str]]) -> pd.DataFrame:
+    if not cache_dir.exists():
+        return pd.DataFrame(columns=_MEASUREMENT_COLUMNS)
+    frames = []
+    for path in sorted(cache_dir.glob("*.csv.gz")):
+        code = path.name[: -len(".csv.gz")]
+        if stations and code not in stations:
+            continue
+        with gzip.open(path, "rt", encoding="utf-8") as handle:
+            frames.append(pd.read_csv(handle, parse_dates=["timestamp"]))
+    if not frames:
+        return pd.DataFrame(columns=_MEASUREMENT_COLUMNS)
+    return _normalize_long(pd.concat(frames, ignore_index=True))
+
+
+def load_measurements(
+    db_url: Optional[str] = None,
+    stations: Optional[List[str]] = None,
+    start: Optional[datetime.datetime] = None,
+    end: Optional[datetime.datetime] = None,
+    use_cache: bool = True,
+    cache_dir: Path = CACHE_DIR,
+    api_url: str = DEFAULT_API_URL,
+    hii_fill: bool = True,
+) -> pd.DataFrame:
+    """Load the long-format [timestamp, station_code, water_level, discharge] history.
+
+    Tries PostgreSQL first, then the HTTP API, then the on-disk cache as a last
+    resort. A successful DB/API fetch refreshes the cache; the cache itself is
+    never treated as a source of fresh data. With `hii_fill` (DB path only),
+    gaps are patched in memory from the HII mirror telemetry — training and
+    serving both flow through here, so the two sides see identical series.
+    """
+    resolved_db_url = resolve_db_url(db_url)
+
+    if resolved_db_url:
+        try:
+            df = _fetch_from_db(resolved_db_url, stations, start, end)
+            if hii_fill:
+                df = fill_from_hii(df, resolved_db_url, start=start, end=end)
+            if use_cache:
+                _write_cache(
+                    df, cache_dir, source="postgres", discharge_maybe_synthetic=False
+                )
+            return df
+        except Exception as error:
+            logger.warning(
+                f"PostgreSQL fetch failed, falling back to HTTP API: {error}"
+            )
+
+    try:
+        api_stations = stations or _default_stations()
+        df = _fetch_from_api(api_url, api_stations, start, end)
+        if not df.empty:
+            if use_cache:
+                _write_cache(
+                    df, cache_dir, source="api", discharge_maybe_synthetic=True
+                )
+            return df
+    except Exception as error:
+        logger.warning(f"HTTP API fetch failed: {error}")
+
+    if use_cache:
+        logger.warning("Falling back to on-disk cache for measurement history")
+        return _read_cache(cache_dir, stations)
+
+    return pd.DataFrame(columns=_MEASUREMENT_COLUMNS)
+
+
+def load_latest(
+    db_url: Optional[str] = None,
+    hours: int = 336,
+    stations: Optional[List[str]] = None,
+) -> pd.DataFrame:
+    """Load the last `hours` of history for all (or given) stations. Never cached to disk."""
+    end = datetime.datetime.now()
+    start = end - datetime.timedelta(hours=hours)
+    return load_measurements(
+        db_url=db_url,
+        stations=stations,
+        start=start,
+        end=end,
+        use_cache=False,
+    )

src/ml/evaluate.py

@@ -0,0 +1,390 @@
+"""Rolling-origin, event-aware evaluation of forecast-model variants.
+
+Replaces the single fixed holdout (which contained only ~4 warning events)
+with one fold per monsoon season: train on everything through 30 April of the
+season's year (labels' rescue statistics bounded to the same cutoff, and label
+windows cannot reach the June+ test span, so the folds are leak-free), test on
+June-November. Metrics are event-level — first-alert lead versus each warning
+crossing, peak error at 24 h — plus pointwise MAE and false-alarm episodes,
+because pointwise PR-AUC alone hid the things that matter operationally.
+
+Variants under test target the two failures documented in
+docs/FLOOD_FORECASTING.md's re-examination note: absolute-level regression
+cannot extrapolate past its training maximum, and the flat sigma miscalibrates
+probabilities.
+"""
+
+import json
+import logging
+from typing import Callable, Dict, List, Optional, Tuple
+
+import numpy as np
+import pandas as pd
+from scipy.special import erf
+
+from . import data, features
+from .train import HGB_PARAMS, _make_regressor
+
+logger = logging.getLogger(__name__)
+
+HORIZON = 24
+SEASONS = (2021, 2022, 2023, 2024, 2025)
+TEST_MONTHS = ("06-01", "11-30")
+TRAIN_END_MD = "04-30"
+ALERT_P = 0.5
+FIXED_SIGMA = 0.15
+EVENT_GAP_H = 24  # merge >=thr runs closer than this into one event
+FALSE_ALARM_GRACE_H = 48
+
+
+def _phi(z: np.ndarray) -> np.ndarray:
+    return 0.5 * (1.0 + erf(z / np.sqrt(2.0)))
+
+
+def _quantile_regressor(q: float):
+    from sklearn.ensemble import HistGradientBoostingRegressor
+
+    return HistGradientBoostingRegressor(loss="quantile", quantile=q, **HGB_PARAMS)
+
+
+def _flood_weights(y_abs: pd.Series) -> np.ndarray:
+    """Upweight the flood regime: 1x below 2.5 m ramping to 5x at >= 3.7 m."""
+    return 1.0 + 4.0 * np.clip((y_abs.to_numpy() - 2.5) / 1.2, 0.0, 1.0)
+
+
+class Variant:
+    """A trainable candidate producing (pred_abs, sigma_per_row) on test rows."""
+
+    def __init__(self, name: str, target: str, weighted: bool = False,
+                 quantile: bool = False, use_rain: bool = False):
+        self.name = name
+        self.target = target  # 'abs' or 'rise'
+        self.weighted = weighted
+        self.quantile = quantile
+        self.use_rain = use_rain
+
+    def fit_predict(
+        self, X_tr, y_abs_tr, X_te
+    ) -> Tuple[np.ndarray, np.ndarray]:
+        if not self.use_rain:
+            drop = [c for c in features.RAIN_FEATURES if c in X_tr.columns]
+            X_tr = X_tr.drop(columns=drop)
+            X_te = X_te.drop(columns=drop)
+        elif "rain_24h" not in X_tr.columns:
+            raise ValueError(
+                f"{self.name} requires the rain series (run without --no-rain)"
+            )
+        level_tr = X_tr["level"]
+        level_te = X_te["level"].to_numpy()
+        y_tr = (y_abs_tr - level_tr) if self.target == "rise" else y_abs_tr
+        weights = _flood_weights(y_abs_tr) if self.weighted else None
+
+        if self.quantile:
+            q50 = _quantile_regressor(0.5).fit(X_tr, y_tr, sample_weight=weights)
+            q90 = _quantile_regressor(0.9).fit(X_tr, y_tr, sample_weight=weights)
+            p50 = q50.predict(X_te)
+            spread = np.maximum(q90.predict(X_te) - p50, 0.0)
+            sigma = np.maximum(spread / 1.2816, 0.05)
+            pred = p50
+        else:
+            reg = _make_regressor().fit(X_tr, y_tr, sample_weight=weights)
+            pred = reg.predict(X_te)
+            sigma = np.full(len(X_te), FIXED_SIGMA)
+
+        pred_abs = pred + level_te if self.target == "rise" else pred
+        pred_abs = np.maximum(pred_abs, level_te)  # peak >= current, as served
+        return pred_abs, sigma
+
+
+VARIANTS: Dict[str, Variant] = {
+    "baseline_abs": Variant("baseline_abs", target="abs"),
+    "rise": Variant("rise", target="rise"),
+    "rise_weighted": Variant("rise_weighted", target="rise", weighted=True),
+    "rise_quantile": Variant("rise_quantile", target="rise", weighted=True,
+                             quantile=True),
+    "rise_rain": Variant("rise_rain", target="rise", use_rain=True),
+}
+
+
+def _find_events(observed: pd.Series, thr: float) -> List[dict]:
+    """Contiguous >=thr episodes (gaps under EVENT_GAP_H merged)."""
+    above = observed[observed >= thr]
+    if above.empty:
+        return []
+    events = []
+    start = prev = above.index[0]
+    for ts in above.index[1:]:
+        if (ts - prev) > pd.Timedelta(hours=EVENT_GAP_H):
+            events.append((start, prev))
+            start = ts
+        prev = ts
+    events.append((start, prev))
+    out = []
+    for begin, end in events:
+        window = observed.loc[begin:end]
+        out.append(
+            {
+                "crossing": begin,
+                "end": end,
+                "peak_ts": window.idxmax(),
+                "peak_level": float(window.max()),
+            }
+        )
+    return out
+
+
+def _first_alert_lead(
+    p: pd.Series,
+    crossing: pd.Timestamp,
+    window_start_floor: Optional[pd.Timestamp] = None,
+) -> Optional[float]:
+    """Hours between the first SUSTAINED alert near the crossing and the
+    crossing. Positive = warned in advance; negative = late.
+
+    Sustained = two consecutive hourly samples with p >= ALERT_P (a single
+    noisy spike gets no credit). The lookback never reaches past
+    ``window_start_floor`` (the previous event's end), so one event's tail
+    cannot be credited as early warning for the next crossing.
+    """
+    start = crossing - pd.Timedelta(hours=72)
+    if window_start_floor is not None and window_start_floor > start:
+        start = window_start_floor
+    window = p.loc[start: crossing + pd.Timedelta(hours=24)]
+    if len(window) < 2:
+        return None
+    alert = (window >= ALERT_P) & (window.shift(-1) >= ALERT_P) & (
+        (window.index.to_series().shift(-1) - window.index.to_series())
+        <= pd.Timedelta(hours=2)
+    )
+    hits = window.index[alert.fillna(False)]
+    if len(hits) == 0:
+        return None
+    return float((crossing - hits[0]).total_seconds() / 3600.0)
+
+
+def _false_alarm_episodes(
+    p: pd.Series, observed: pd.Series, thr: float
+) -> int:
+    """Alert episodes with no observed >=thr within +/- FALSE_ALARM_GRACE_H."""
+    alert_hours = p[p >= ALERT_P].index
+    if len(alert_hours) == 0:
+        return 0
+    grace = pd.Timedelta(hours=FALSE_ALARM_GRACE_H)
+    exceed_times = observed[observed >= thr].index
+    episodes = 0
+    episode_start = None
+    prev = None
+    for ts in alert_hours:
+        # 12h gap tolerance: a data hole mid-alarm must not double-count it
+        if prev is None or (ts - prev) > pd.Timedelta(hours=12):
+            if episode_start is not None:
+                episodes += _is_false(episode_start, prev, exceed_times, grace)
+            episode_start = ts
+        prev = ts
+    episodes += _is_false(episode_start, prev, exceed_times, grace)
+    return episodes
+
+
+def _is_false(start, end, exceed_times, grace) -> int:
+    if len(exceed_times) == 0:
+        return 1
+    near = (exceed_times >= start - grace) & (exceed_times <= end + grace)
+    return 0 if near.any() else 1
+
+
+def evaluate_station(
+    df_long: pd.DataFrame,
+    station: str,
+    variants: Optional[List[str]] = None,
+    seasons: Tuple[int, ...] = SEASONS,
+    rain: Optional[pd.Series] = None,
+) -> Dict:
+    """Run every fold x variant for one station; returns the results tree."""
+    warn_thr, _ = features.get_thresholds(station)
+    grid = features.make_hourly_grid(df_long)
+    X_all = features.build_features(grid, station, rain=rain)
+    observed = grid.observed[(station, "water_level")]
+
+    keep = X_all["obs_age_h"].notna()
+    train_start = features.TRAIN_START.get(station)
+    if train_start:
+        keep &= X_all.index >= pd.Timestamp(train_start)
+    X_all = X_all.loc[keep]
+
+    chosen = {k: VARIANTS[k] for k in (variants or VARIANTS)}
+    results: Dict = {"station": station, "warn_thr": warn_thr, "folds": []}
+
+    for year in seasons:
+        train_end = pd.Timestamp(f"{year}-{TRAIN_END_MD}")
+        test_lo = pd.Timestamp(f"{year}-{TEST_MONTHS[0]}")
+        test_hi = pd.Timestamp(f"{year}-{TEST_MONTHS[1]} 23:00")
+
+        # Labels rebuilt per fold so rescue statistics stop at the cutoff
+        Y = features.build_labels(
+            grid, station, (HORIZON,), stats_end=train_end.isoformat()
+        ).loc[X_all.index]
+        y_abs = Y[f"max_level_{HORIZON}"]
+
+        tr = (X_all.index <= train_end) & y_abs.notna()
+        te = (X_all.index >= test_lo) & (X_all.index <= test_hi)
+        if tr.sum() < 5000 or te.sum() < 500:
+            logger.info(f"{station} {year}: skipped (train {tr.sum()}, test {te.sum()})")
+            continue
+
+        X_tr, X_te = X_all.loc[tr], X_all.loc[te]
+        y_tr = y_abs.loc[tr]
+        y_te = y_abs.loc[te]
+        obs_test = observed.loc[test_lo:test_hi].dropna()
+        events = _find_events(obs_test, warn_thr)
+
+        fold: Dict = {
+            "year": year,
+            "n_train": int(tr.sum()),
+            "n_test": int(te.sum()),
+            "events": [
+                {
+                    "crossing": e["crossing"].isoformat(),
+                    "peak_ts": e["peak_ts"].isoformat(),
+                    "peak_level": e["peak_level"],
+                }
+                for e in events
+            ],
+            "variants": {},
+        }
+
+        for name, variant in chosen.items():
+            pred_abs, sigma = variant.fit_predict(X_tr, y_tr, X_te)
+            pred_series = pd.Series(pred_abs, index=X_te.index)
+            p_warn = pd.Series(
+                1.0 - _phi((warn_thr - pred_abs) / sigma), index=X_te.index
+            )
+
+            labeled = y_te.notna()
+            errors = (pred_series[labeled] - y_te[labeled]).abs()
+            high = y_te[labeled] >= warn_thr - 1.2  # flood-regime rows
+            # Brier score on within-24h warning exceedance: unlike the p>=0.5
+            # alert metrics (where sigma cancels algebraically), this actually
+            # exercises each variant's uncertainty model.
+            exceed = Y[f"exceed_warn_{HORIZON}"].loc[te]
+            scored = exceed.notna()
+            brier = (
+                float(((p_warn[scored] - exceed[scored]) ** 2).mean())
+                if scored.any()
+                else None
+            )
+            event_rows = []
+            for i, event in enumerate(events):
+                floor = events[i - 1]["end"] if i > 0 else None
+                lead = _first_alert_lead(p_warn, event["crossing"], floor)
+                issue_ts = event["peak_ts"] - pd.Timedelta(hours=HORIZON)
+                peak_pred = None
+                if len(pred_series):
+                    nearest = pred_series.index.get_indexer(
+                        [issue_ts], method="nearest"
+                    )[0]
+                    matched_ts = pred_series.index[nearest]
+                    # Tolerance: a "24h-ahead" prediction matched to a row
+                    # hours away (data outage) is not that prediction at all.
+                    if abs(matched_ts - issue_ts) <= pd.Timedelta(hours=3):
+                        peak_pred = float(pred_series.iloc[nearest])
+                event_rows.append(
+                    {
+                        "crossing": event["crossing"].isoformat(),
+                        "lead_h": lead,
+                        "peak_level": event["peak_level"],
+                        "peak_pred_24h_before": peak_pred,
+                    }
+                )
+            fold["variants"][name] = {
+                "mae": float(errors.mean()) if len(errors) else None,
+                "mae_above_2p5": (
+                    float(errors[high].mean()) if high.any() else None
+                ),
+                "brier_warn": brier,
+                "events": event_rows,
+                "false_alarm_episodes": _false_alarm_episodes(
+                    p_warn, obs_test, warn_thr
+                ),
+            }
+        results["folds"].append(fold)
+    return results
+
+
+def summarize(results: Dict) -> str:
+    """Compact comparison table across folds for one station."""
+    lines = [f"\n=== {results['station']} (warn {results['warn_thr']:.2f} m) ==="]
+    header = (
+        f"{'variant':16} {'year':>5} {'MAE':>6} {'MAE_hi':>7} {'Brier':>7} "
+        f"{'FA':>3}  events (lead h | peak err m)"
+    )
+    lines.append(header)
+    for fold in results["folds"]:
+        for name, m in fold["variants"].items():
+            events = "  ".join(
+                f"[{e['crossing'][:10]}: "
+                f"{'—' if e['lead_h'] is None else format(e['lead_h'], '+.0f')}h"
+                + (
+                    f" | {e['peak_pred_24h_before'] - e['peak_level']:+.2f}"
+                    if e["peak_pred_24h_before"] is not None
+                    else ""
+                )
+                + "]"
+                for e in m["events"]
+            ) or "no events"
+            lines.append(
+                f"{name:16} {fold['year']:>5} "
+                f"{m['mae'] if m['mae'] is not None else float('nan'):6.3f} "
+                f"{m['mae_above_2p5'] if m['mae_above_2p5'] is not None else float('nan'):7.3f} "
+                f"{m['brier_warn'] if m.get('brier_warn') is not None else float('nan'):7.4f} "
+                f"{m['false_alarm_episodes']:>3}  {events}"
+            )
+    return "\n".join(lines)
+
+
+def main(argv=None) -> int:
+    import argparse
+
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--stations", default="P.1")
+    parser.add_argument("--db-url", default=None)
+    parser.add_argument("--variants", default=None,
+                        help="comma list; default all")
+    parser.add_argument("--out", default="models/eval_variants.json")
+    parser.add_argument("--no-rain", action="store_true",
+                        help="skip loading the Open-Meteo rain series")
+    args = parser.parse_args(argv)
+
+    logging.basicConfig(
+        level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s"
+    )
+    df = data.load_measurements(db_url=args.db_url)
+    if df.empty:
+        logger.error("no measurement data")
+        return 1
+
+    rain_series = None
+    if not args.no_rain:
+        from . import rain as rain_mod
+
+        rain_series = rain_mod.catchment_mean(rain_mod.load_history())
+        if rain_series is None:
+            logger.warning("rain history unavailable; rain features will be NaN")
+        else:
+            logger.info(
+                f"rain series loaded: {rain_series.index.min()} .. "
+                f"{rain_series.index.max()}"
+            )
+
+    variant_names = args.variants.split(",") if args.variants else None
+    all_results = []
+    for station in args.stations.split(","):
+        station = station.strip()
+        logger.info(f"Evaluating {station}...")
+        results = evaluate_station(df, station, variant_names, rain=rain_series)
+        all_results.append(results)
+        print(summarize(results))
+
+    with open(args.out, "w", encoding="utf-8") as fh:
+        json.dump(all_results, fh, indent=1)
+    logger.info(f"results written to {args.out}")
+    return 0

src/ml/features.py

@@ -0,0 +1,390 @@
+"""Static config and feature/label engineering for the Ping River flood forecast models.
+
+All feature computation is strictly causal (no row uses information timestamped after
+itself) so it is safe to run identically at training time and at prediction time.
+"""
+
+import datetime
+import logging
+from dataclasses import dataclass
+from typing import Dict, List, Optional, Tuple
+
+import numpy as np
+import pandas as pd
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Static configuration
+# ---------------------------------------------------------------------------
+
+# Per-station (warning, danger) levels in metres on each gauge's own datum.
+# "*" is the default applied to any station without an explicit override.
+# Calibrated 2026-08-10 from the DB's discharge_percent (RID % of channel
+# capacity): warning = median level at 75-85% capacity, danger = median level
+# at 95-105%. P.1 instead uses the official Chiang Mai inundation map keyed to
+# the P.1 gauge: city flooding begins at 3.70 m (stage 1) and reaches most
+# districts by 4.20 m (stage 5) — see P1_FLOOD_STAGES.
+THRESHOLDS: Dict[str, Tuple[float, float]] = {
+    "*": (3.0, 4.5),
+    "P.1": (3.70, 4.20),
+    "P.103": (5.95, 6.75),
+    "P.20": (2.35, 2.80),
+    "P.21": (3.20, 3.60),
+    # P.4A: LOW CONFIDENCE — its sensor was dead 2019-2024 and only 11 readings
+    # ever reached 3.40 m, so the capacity calibration rests on very few points.
+    # It only affects the heuristic sigmoid (P.4A is NOT_TRAINABLE).
+    "P.4A": (3.40, 3.90),
+    "P.5": (4.55, 4.95),
+    "P.67": (2.45, 2.90),
+    "P.75": (2.75, 3.50),
+    "P.76": (5.35, 5.45),
+    "P.77": (2.85, 3.35),
+    "P.81": (5.15, 6.30),
+    # P.82 never reached 100% capacity in the record (max level 3.78, max 96.4%);
+    # danger sits just below the observed maximum so the head can actually train.
+    "P.82": (3.40, 3.75),
+    "P.84": (3.45, 3.90),
+    "P.85": (2.90, 3.35),
+    "P.87": (3.75, 4.05),
+    "P.92": (2.95, 3.60),
+}
+
+# Official Chiang Mai flood-onset stages at the P.1 gauge (Nawarat Bridge),
+# from the municipal inundation map (พื้นที่ท่วมตัวเมืองเชียงใหม่, events of
+# 2548/2554/2565 BE): gauge level in m, RID discharge in m³/s. Each stage
+# floods progressively more city zones.
+P1_FLOOD_STAGES: List[Dict[str, float]] = [
+    {"stage": 1, "level": 3.70, "discharge_cms": 405},
+    {"stage": 2, "level": 3.90, "discharge_cms": 438},
+    {"stage": 3, "level": 4.00, "discharge_cms": 458},
+    {"stage": 4, "level": 4.10, "discharge_cms": 478},
+    {"stage": 5, "level": 4.20, "discharge_cms": 493},
+    {"stage": 6, "level": 4.30, "discharge_cms": 508},
+    {"stage": 7, "level": 4.60, "discharge_cms": 558},
+]
+
+FLOOD_STAGES: Dict[str, List[Dict[str, float]]] = {"P.1": P1_FLOOD_STAGES}
+
+MONSOON_MONTHS = {6, 7, 8, 9, 10}
+FFILL_LIMIT_H = 3
+MIN_WINDOW_COVERAGE = 0.5
+
+BASIN_ANCHOR = "P.1"
+
+# Empirical hours a station's water-level anomaly leads the basin anchor (P.1),
+# derived from data-scout cross-correlation analysis. UPSTREAM_LEADS[station]
+# lists, for each station, the (upstream_code, lead_hours) pairs to use as
+# routed-upstream input features when forecasting `station`.
+UPSTREAM_LEADS: Dict[str, List[Tuple[str, int]]] = {
+    "P.1": [
+        ("P.103", 1),
+        ("P.67", 7),
+        ("P.21", 9),
+        ("P.75", 12),
+        ("P.4A", 12),
+        ("P.92", 15),
+        ("P.20", 17),
+    ],
+    "P.103": [
+        ("P.67", 6),
+        ("P.21", 8),
+        ("P.75", 11),
+        ("P.4A", 11),
+        ("P.92", 14),
+        ("P.20", 16),
+    ],
+    "P.21": [("P.67", 1), ("P.75", 3), ("P.4A", 3), ("P.92", 6), ("P.20", 8)],
+    "P.67": [("P.75", 5), ("P.4A", 5), ("P.92", 8), ("P.20", 10)],
+    "P.75": [("P.92", 3), ("P.20", 5)],
+    "P.4A": [("P.92", 3), ("P.20", 5)],
+    "P.92": [("P.20", 2)],
+    "P.20": [],
+    "P.5": [("P.1", 12), ("P.103", 13)],
+    "P.81": [("P.1", 4), ("P.103", 5)],
+    "P.82": [],
+    "P.84": [],
+    "P.87": [],
+    "P.77": [],
+    "P.85": [],
+    "P.76": [],
+}
+
+# Per-station usable-from dates: data before this cutoff is excluded from training
+# because of known data-quality holes (see data-scout inventory).
+TRAIN_START: Dict[str, str] = {"P.5": "2022-01-01"}
+
+# Stations with data too sparse/broken to ever be a regression/classification
+# target. They are still usable as upstream *input* features (HGB tolerates NaN).
+NOT_TRAINABLE: Dict[str, str] = {"P.4A": "17% fill, dead 2019-2024"}
+
+
+def get_thresholds(station_code: str) -> Tuple[float, float]:
+    """Return (warning, danger) level thresholds for a station, falling back to the default."""
+    return THRESHOLDS.get(station_code, THRESHOLDS["*"])
+
+
+# ---------------------------------------------------------------------------
+# Hourly grid
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class HourlyGrid:
+    """A complete hourly time grid pivoted wide across stations.
+
+    observed: raw values, NaN where nothing was recorded that hour (pristine; used for labels).
+    filled:   observed forward-filled per column with limit=FFILL_LIMIT_H (causal; used for features).
+    mask:     boolean, True where `observed` has a real reading.
+    """
+
+    observed: pd.DataFrame
+    filled: pd.DataFrame
+    mask: pd.DataFrame
+
+
+def make_hourly_grid(df_long: pd.DataFrame) -> HourlyGrid:
+    """Pivot a long station/timestamp measurement frame onto a complete hourly grid.
+
+    df_long columns: timestamp, station_code, water_level, discharge.
+    """
+    if df_long.empty:
+        empty = pd.DataFrame(
+            index=pd.DatetimeIndex([], name="timestamp"),
+            columns=pd.MultiIndex.from_tuples([], names=["station_code", "field"]),
+        )
+        return HourlyGrid(observed=empty, filled=empty.copy(), mask=empty.copy())
+
+    df = df_long.copy()
+    df["timestamp"] = pd.to_datetime(df["timestamp"]).dt.floor("h")
+    df = df.drop_duplicates(subset=["station_code", "timestamp"], keep="last")
+
+    full_index = pd.date_range(
+        df["timestamp"].min(), df["timestamp"].max(), freq="h", name="timestamp"
+    )
+
+    wide = df.pivot(
+        index="timestamp", columns="station_code", values=["water_level", "discharge"]
+    )
+    wide = wide.reorder_levels([1, 0], axis=1).sort_index(axis=1)
+    wide = wide.reindex(full_index)
+
+    observed = wide
+    mask = observed.notna()
+    # Forward-fill only — never interpolate — so no row ever depends on a future value.
+    filled = observed.ffill(limit=FFILL_LIMIT_H)
+
+    return HourlyGrid(observed=observed, filled=filled, mask=mask)
+
+
+def _series(
+    grid_frame: pd.DataFrame, station: str, field: str, index: pd.Index
+) -> pd.Series:
+    """Fetch a (station, field) column, or an all-NaN series if the station is absent."""
+    if (station, field) in grid_frame.columns:
+        return grid_frame[(station, field)]
+    return pd.Series(np.nan, index=index)
+
+
+def _hours_since_observed(mask_col: pd.Series) -> pd.Series:
+    """Hours since the last True in `mask_col` (0 at an observed hour; NaN if never observed yet)."""
+    idx = mask_col.index
+    obs_time = pd.Series(idx, index=idx).where(mask_col.to_numpy())
+    last_obs_time = obs_time.ffill()
+    age_hours = (idx.to_series() - last_obs_time).dt.total_seconds() / 3600.0
+    return age_hours
+
+
+# ---------------------------------------------------------------------------
+# Features
+# ---------------------------------------------------------------------------
+
+
+RAIN_FEATURES = ("rain_6h", "rain_24h", "rain_72h", "rain_fc24")
+
+
+def build_features(
+    grid: HourlyGrid, station: str, rain: Optional[pd.Series] = None
+) -> pd.DataFrame:
+    """Build the deterministic-order feature matrix for one target station.
+
+    ``rain`` is the hourly catchment-average precipitation series (Open-Meteo,
+    src/ml/rain.py). Rain columns are added only when a series is passed:
+    HistGradientBoosting REJECTS all-NaN columns at fit time, so training
+    without rain must omit the columns entirely (bundles record their
+    feature_names, and serving subsets to them). At serving, pass an empty
+    series rather than None so the columns exist (as NaN) for rain-trained
+    bundles even when the live fetch fails. rain_fc24 is the forward 24 h
+    sum: the archived forecast series at training time, a real weather
+    forecast at serving time; it never contains river data.
+    """
+    idx = grid.observed.index
+    cols: Dict[str, pd.Series] = {}
+
+    level = _series(grid.filled, station, "water_level", idx)
+    discharge = _series(grid.observed, station, "discharge", idx)
+    obs_mask = _series(grid.mask, station, "water_level", idx).fillna(False)
+
+    cols["level"] = level
+    for k in (1, 2, 3, 6, 12, 24, 48, 72):
+        cols[f"level_lag_{k}"] = level.shift(k)
+    for k in (1, 3, 6, 12, 24):
+        cols[f"rise_{k}"] = level - level.shift(k)
+    cols["roll_mean_6"] = level.rolling(6, min_periods=1).mean()
+    cols["roll_mean_24"] = level.rolling(24, min_periods=1).mean()
+    cols["roll_max_6"] = level.rolling(6, min_periods=1).max()
+    cols["roll_max_24"] = level.rolling(24, min_periods=1).max()
+    cols["roll_max_72"] = level.rolling(72, min_periods=1).max()
+    cols["roll_min_24"] = level.rolling(24, min_periods=1).min()
+
+    cols["discharge"] = discharge
+    cols["discharge_lag_6"] = discharge.shift(6)
+    cols["discharge_lag_24"] = discharge.shift(24)
+    cols["discharge_rise_6"] = discharge - discharge.shift(6)
+
+    obs_age_h = _hours_since_observed(obs_mask)
+    cols["obs_age_h"] = obs_age_h.where(obs_age_h <= FFILL_LIMIT_H)
+    cols["cov_24h"] = obs_mask.rolling(24, min_periods=1).mean()
+
+    for upstream_code, lead_h in UPSTREAM_LEADS.get(station, []):
+        u_level = _series(grid.filled, upstream_code, "water_level", idx)
+        u_rise_6 = u_level - u_level.shift(6)
+        u_rollmax_24 = u_level.rolling(24, min_periods=1).max()
+        near_lag = max(0, lead_h - 3)
+        cols[f"{upstream_code}_level_lag_{near_lag}"] = u_level.shift(near_lag)
+        cols[f"{upstream_code}_level_lag_{lead_h}"] = u_level.shift(lead_h)
+        cols[f"{upstream_code}_level_lag_{lead_h + 3}"] = u_level.shift(lead_h + 3)
+        cols[f"{upstream_code}_rise_6_lag_{lead_h}"] = u_rise_6.shift(lead_h)
+        cols[f"{upstream_code}_rollmax_24_lag_{near_lag}"] = u_rollmax_24.shift(
+            near_lag
+        )
+
+    if station != BASIN_ANCHOR:
+        p1_level = _series(grid.filled, BASIN_ANCHOR, "water_level", idx)
+        cols["P1_level"] = p1_level
+        cols["P1_rollmax_24"] = p1_level.rolling(24, min_periods=1).max()
+        cols["P1_rise_24"] = p1_level - p1_level.shift(24)
+
+    doy = idx.to_series().dt.dayofyear.astype(float)
+    cols["doy_sin"] = np.sin(2 * np.pi * doy / 365.25)
+    cols["doy_cos"] = np.cos(2 * np.pi * doy / 365.25)
+    cols["is_monsoon"] = idx.to_series().dt.month.isin(MONSOON_MONTHS).astype(float)
+
+    if rain is not None:
+        r = rain.reindex(idx)
+        cols["rain_6h"] = r.rolling(6, min_periods=1).sum()
+        cols["rain_24h"] = r.rolling(24, min_periods=1).sum()
+        cols["rain_72h"] = r.rolling(72, min_periods=1).sum()
+        # forward sum over (t, t+24]: shift(-1) starts the window at t+1
+        cols["rain_fc24"] = (
+            r.shift(-1).iloc[::-1].rolling(24, min_periods=1).sum().iloc[::-1]
+        )
+
+    return pd.DataFrame(cols, index=idx)
+
+
+# ---------------------------------------------------------------------------
+# Labels
+# ---------------------------------------------------------------------------
+
+
+def _future_window_stats(col: pd.Series, horizon_h: int) -> Tuple[pd.Series, pd.Series]:
+    """For every t, (max, count) of observed values in the OPEN window (t, t+horizon_h]."""
+    reversed_col = col.iloc[::-1]
+    shifted = reversed_col.shift(1)  # excludes t itself
+    fut_max = shifted.rolling(horizon_h, min_periods=1).max().iloc[::-1]
+    fut_count = shifted.rolling(horizon_h, min_periods=1).count().iloc[::-1]
+    return fut_max, fut_count
+
+
+def build_labels(
+    grid: HourlyGrid,
+    station: str,
+    horizons: Tuple[int, ...] = (6, 12, 24),
+    stats_end: Optional[str] = None,
+) -> pd.DataFrame:
+    """Build max-level and threshold-exceedance labels for one target station.
+
+    ``stats_end`` bounds the data used for label-construction statistics (the
+    rescue quantile below): pass the training cutoff during evaluation so
+    test-period extremes cannot influence which training rows receive labels.
+    """
+    idx = grid.observed.index
+    observed_level = _series(grid.observed, station, "water_level", idx)
+    warn_thr, danger_thr = get_thresholds(station)
+    # Low-coverage windows are still usable regression labels when they contain a
+    # rare high reading. Anchor "rare" to the station's own distribution (p97.5),
+    # NOT to warn_thr: coupling it to the configurable threshold made raising a
+    # station's threshold silently shrink its regression training set (P.5 lost
+    # 34% of rows and +46% MAE when its warning went 3.0 -> 4.55).
+    stats_level = (
+        observed_level.loc[: pd.Timestamp(stats_end)] if stats_end else observed_level
+    )
+    rescue_thr = (
+        float(stats_level.quantile(0.975)) if stats_level.notna().any() else np.inf
+    )
+
+    out: Dict[str, pd.Series] = {}
+    for horizon_h in horizons:
+        fut_max, fut_count = _future_window_stats(observed_level, horizon_h)
+        cov = fut_count / horizon_h
+        enough_cov = cov >= MIN_WINDOW_COVERAGE
+
+        exceed_warn = pd.Series(np.nan, index=idx)
+        exceed_warn[fut_max >= warn_thr] = 1.0
+        exceed_warn[enough_cov & exceed_warn.isna()] = 0.0
+
+        exceed_danger = pd.Series(np.nan, index=idx)
+        exceed_danger[fut_max >= danger_thr] = 1.0
+        exceed_danger[enough_cov & exceed_danger.isna()] = 0.0
+
+        max_level_valid = fut_max.where(enough_cov | (fut_max >= rescue_thr))
+
+        out[f"max_level_{horizon_h}"] = max_level_valid
+        out[f"exceed_warn_{horizon_h}"] = exceed_warn
+        out[f"exceed_danger_{horizon_h}"] = exceed_danger
+
+    return pd.DataFrame(out, index=idx)
+
+
+# ---------------------------------------------------------------------------
+# Glue
+# ---------------------------------------------------------------------------
+
+
+def build_matrix(
+    df_long: pd.DataFrame,
+    station: str,
+    horizons: Tuple[int, ...] = (6, 12, 24),
+    stats_end: Optional[str] = None,
+    rain: Optional[pd.Series] = None,
+) -> Tuple[pd.DataFrame, pd.DataFrame, dict]:
+    """Build (X, Y, meta) training/inference matrices for one station."""
+    grid = make_hourly_grid(df_long)
+    X = build_features(grid, station, rain=rain)
+    Y = build_labels(grid, station, horizons, stats_end=stats_end)
+
+    keep = X["obs_age_h"].notna()
+    train_start = TRAIN_START.get(station)
+    if train_start:
+        keep &= X.index >= pd.Timestamp(train_start)
+
+    X = X.loc[keep]
+    Y = Y.loc[keep]
+
+    positive_counts = {
+        col: int(Y[col].sum())
+        for col in Y.columns
+        if col.startswith("exceed_") and Y[col].notna().any()
+    }
+    meta = {
+        "station_code": station,
+        "n_rows": int(len(X)),
+        "span": (
+            (X.index.min().isoformat(), X.index.max().isoformat())
+            if len(X)
+            else (None, None)
+        ),
+        "positive_counts": positive_counts,
+    }
+    return X, Y, meta

src/ml/predict.py

@@ -0,0 +1,379 @@
+"""Flood forecast inference.
+
+Integration contract (see get_forecasts / get_latest_forecasts): callers pass
+raw station readings, get back one forecast dict per station x horizon. A
+station with a stale, missing, or version-mismatched model transparently
+falls back to a simple persistence heuristic instead of raising -- this
+module must never crash the caller (e.g. the web API).
+"""
+
+import datetime
+import logging
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple, Union
+
+import joblib
+import numpy as np
+import pandas as pd
+
+from . import features
+
+logger = logging.getLogger(__name__)
+
+# Anchored to the repo root so the API finds trained bundles regardless of CWD.
+DEFAULT_MODELS_DIR = Path(__file__).resolve().parents[2] / "models"
+
+DEFAULT_HORIZONS: Tuple[int, ...] = (6, 12, 24)
+STALE_AFTER_H = 6.0
+HEURISTIC_SIGMA = 0.3
+HEURISTIC_VERSION = "heuristic-v1"
+
+# Keyed by (path, mtime) so a retrained model (new mtime) invalidates the old entry.
+_MODEL_CACHE: Dict[Tuple[str, float], dict] = {}
+
+
+def _load_bundle(path: Path) -> dict:
+    # joblib.load runs arbitrary pickle code; safe here because `path` is always
+    # models/flood_{station}.joblib, an artifact this pipeline's own train.py wrote --
+    # never a user- or network-supplied file.
+    key = (str(path), path.stat().st_mtime)
+    cached = _MODEL_CACHE.get(key)
+    if cached is not None:
+        return cached
+    bundle = joblib.load(path)
+    for stale_key in [k for k in _MODEL_CACHE if k[0] == str(path)]:
+        del _MODEL_CACHE[stale_key]
+    _MODEL_CACHE[key] = bundle
+    return bundle
+
+
+def _readings_to_long_df(readings_by_station: Dict[str, List[dict]]) -> pd.DataFrame:
+    rows = []
+    for station_code, readings in readings_by_station.items():
+        for reading in readings:
+            timestamp = reading.get("timestamp")
+            if isinstance(timestamp, str):
+                timestamp = pd.to_datetime(timestamp)
+            rows.append(
+                {
+                    "timestamp": timestamp,
+                    "station_code": station_code,
+                    "water_level": reading.get("water_level"),
+                    "discharge": reading.get("discharge"),
+                }
+            )
+    if not rows:
+        return pd.DataFrame(
+            columns=["timestamp", "station_code", "water_level", "discharge"]
+        )
+    df = pd.DataFrame(rows)
+    return df.dropna(subset=["timestamp"])
+
+
+def _clip_probability(value: float) -> float:
+    return float(min(max(value, 0.0), 1.0))
+
+
+def _sigmoid_probability(predicted_max: float, threshold: float, sigma: float) -> float:
+    return 1.0 / (1.0 + np.exp(-(predicted_max - threshold) / sigma))
+
+
+def _heuristic_forecast(
+    station_code: str,
+    as_of: pd.Timestamp,
+    current_level: float,
+    level_t_minus_3: Optional[float],
+    warn_thr: float,
+    danger_thr: float,
+    horizons: Tuple[int, ...],
+) -> List[dict]:
+    if level_t_minus_3 is None:
+        rate = 0.0
+    else:
+        rate = max(0.0, (current_level - level_t_minus_3) / 3.0)
+
+    results = []
+    for horizon_h in horizons:
+        predicted_max = max(current_level + rate * horizon_h * 0.7, current_level)
+        p_warning = _clip_probability(
+            _sigmoid_probability(predicted_max, warn_thr, HEURISTIC_SIGMA)
+        )
+        p_danger = _clip_probability(
+            _sigmoid_probability(predicted_max, danger_thr, HEURISTIC_SIGMA)
+        )
+        p_danger = min(p_danger, p_warning)
+        results.append(
+            {
+                "station_code": station_code,
+                "horizon_hours": horizon_h,
+                "p_warning": p_warning,
+                "p_danger": p_danger,
+                "predicted_max_level": predicted_max,
+                "current_level": current_level,
+                "as_of": as_of.isoformat(),
+                "model_version": HEURISTIC_VERSION,
+                "trained_at": None,
+                "source": "heuristic",
+                "threshold_warning": warn_thr,
+                "threshold_danger": danger_thr,
+            }
+        )
+    return results
+
+
+def _model_forecast(
+    station_code: str,
+    grid: features.HourlyGrid,
+    bundle: dict,
+    as_of: pd.Timestamp,
+    current_level: float,
+    rain: Optional[pd.Series] = None,
+) -> List[dict]:
+    warn_thr = bundle["thresholds"]["warning"]
+    danger_thr = bundle["thresholds"]["danger"]
+    # If thresholds changed since this bundle was trained, its classifier heads
+    # answer the OLD question (labels for the old levels) while stages/config use
+    # the new ones — a silent contradiction on the dashboard. Until a retrain,
+    # answer the current question consistently: use the regression + sigma against
+    # the configured thresholds and skip the stale heads.
+    cfg_warn, cfg_danger = features.get_thresholds(station_code)
+    thresholds_stale = (cfg_warn, cfg_danger) != (warn_thr, danger_thr)
+    if thresholds_stale:
+        logger.warning(
+            f"{station_code}: bundle thresholds ({warn_thr}, {danger_thr}) differ from "
+            f"configured ({cfg_warn}, {cfg_danger}); using regression-derived probabilities "
+            "until the model is retrained"
+        )
+        warn_thr, danger_thr = cfg_warn, cfg_danger
+
+    feature_row = features.build_features(grid, station_code, rain=rain).loc[[as_of]]
+    expected_columns = bundle["feature_names"]
+    missing = [c for c in expected_columns if c not in feature_row.columns]
+    if missing:
+        logger.error(
+            f"Feature mismatch for {station_code} (missing {missing}); falling back to heuristic"
+        )
+        return None
+    feature_row = feature_row[expected_columns]
+
+    results = []
+    for horizon_h in bundle["horizons"]:
+        reg = bundle["heads"].get(f"max_{horizon_h}")
+        if reg is None:
+            results.append(None)
+            continue
+        raw_prediction = float(reg.predict(feature_row)[0])
+        if bundle.get("regression_target") == "rise":
+            # v2 bundles predict the rise over the current level
+            raw_prediction += current_level
+        predicted_max = max(raw_prediction, current_level)
+        sigma_h = bundle["sigma"].get(horizon_h, HEURISTIC_SIGMA)
+
+        # Belt-and-braces: the classifier head OR the regression-sigmoid path,
+        # whichever is more alarmed. The 2026-08-11 backtest showed a trained
+        # classifier staying silent through the 2024 record flood while the
+        # regression head tracked it — alerting must never be worse than the
+        # regression fallback.
+        warn_head = (
+            None if thresholds_stale else bundle["heads"].get(f"warn_{horizon_h}")
+        )
+        p_warning = _sigmoid_probability(predicted_max, warn_thr, sigma_h)
+        if warn_head is not None:
+            p_warning = max(p_warning, float(warn_head.predict_proba(feature_row)[0][1]))
+
+        danger_head = (
+            None if thresholds_stale else bundle["heads"].get(f"danger_{horizon_h}")
+        )
+        p_danger = _sigmoid_probability(predicted_max, danger_thr, sigma_h)
+        if danger_head is not None:
+            p_danger = max(p_danger, float(danger_head.predict_proba(feature_row)[0][1]))
+
+        p_warning = _clip_probability(p_warning)
+        p_danger = min(_clip_probability(p_danger), p_warning)
+
+        row = {
+            "station_code": station_code,
+            "horizon_hours": horizon_h,
+            "p_warning": p_warning,
+            "p_danger": p_danger,
+            "predicted_max_level": predicted_max,
+            "current_level": current_level,
+            "as_of": as_of.isoformat(),
+            "model_version": bundle["model_version"],
+            "trained_at": bundle["trained_at"],
+            "source": "model",
+            "threshold_warning": warn_thr,
+            "threshold_danger": danger_thr,
+        }
+        stages = features.FLOOD_STAGES.get(station_code)
+        if stages:
+            # Exceedance probability per official inundation stage, from the
+            # regression head and its validation-residual sigma. These are
+            # threshold-agnostic, so no retraining is needed to serve them.
+            row["stages"] = [
+                {
+                    "stage": s["stage"],
+                    "level": s["level"],
+                    "p_exceed": _clip_probability(
+                        _sigmoid_probability(predicted_max, s["level"], sigma_h)
+                    ),
+                }
+                for s in stages
+            ]
+        results.append(row)
+    return results
+
+
+def _forecast_station(
+    station_code: str,
+    grid: features.HourlyGrid,
+    models_dir: Path,
+    now: pd.Timestamp,
+    horizons: Tuple[int, ...],
+    rain: Optional[pd.Series] = None,
+) -> List[dict]:
+    level_col = (station_code, "water_level")
+    if level_col not in grid.observed.columns:
+        logger.warning(f"No data for station {station_code}; omitting")
+        return []
+    observed_level = grid.observed[level_col].dropna()
+    if observed_level.empty:
+        logger.warning(f"No observed readings for station {station_code}; omitting")
+        return []
+
+    as_of = observed_level.index.max()
+    current_level = float(observed_level.loc[as_of])
+    staleness_h = (pd.Timestamp(now) - as_of).total_seconds() / 3600.0
+
+    warn_thr, danger_thr = features.get_thresholds(station_code)
+    t_minus_3 = as_of - pd.Timedelta(hours=3)
+    level_t_minus_3 = (
+        float(observed_level.loc[t_minus_3])
+        if t_minus_3 in observed_level.index
+        else None
+    )
+
+    bundle_path = models_dir / f"flood_{station_code}.joblib"
+    if not bundle_path.exists() or staleness_h > STALE_AFTER_H:
+        return _heuristic_forecast(
+            station_code,
+            as_of,
+            current_level,
+            level_t_minus_3,
+            warn_thr,
+            danger_thr,
+            horizons,
+        )
+
+    bundle = _load_bundle(bundle_path)
+    model_results = _model_forecast(
+        station_code, grid, bundle, as_of, current_level, rain=rain
+    )
+    if model_results is None:
+        return _heuristic_forecast(
+            station_code,
+            as_of,
+            current_level,
+            level_t_minus_3,
+            warn_thr,
+            danger_thr,
+            horizons,
+        )
+
+    # Per-horizon heads that were skipped at train time (e.g. too few positives) still
+    # need a forecast row -- fall back to the single-horizon heuristic for just that row.
+    filled = []
+    for horizon_h, row in zip(bundle["horizons"], model_results):
+        if row is not None:
+            filled.append(row)
+        else:
+            filled.extend(
+                _heuristic_forecast(
+                    station_code,
+                    as_of,
+                    current_level,
+                    level_t_minus_3,
+                    warn_thr,
+                    danger_thr,
+                    (horizon_h,),
+                )
+            )
+    return filled
+
+
+def get_forecasts(
+    readings_by_station: Dict[str, List[dict]],
+    models_dir: Union[str, Path] = DEFAULT_MODELS_DIR,
+    now: Optional[Union[datetime.datetime, str]] = None,
+    rain: Optional[pd.Series] = None,
+) -> List[dict]:
+    """Produce flood forecasts for every station present in `readings_by_station`.
+
+    Each reading dict needs at least {timestamp, water_level, discharge}; extra
+    keys are ignored so raw API/DB rows can be passed straight through. At
+    least 96 hours of span is required to populate every feature; 336 hours
+    (14 days) is recommended.
+    """
+    models_dir = Path(models_dir)
+    if now is None:
+        now = datetime.datetime.now()
+    now = pd.Timestamp(now)
+
+    df_long = _readings_to_long_df(readings_by_station)
+    if df_long.empty:
+        return []
+
+    grid = features.make_hourly_grid(df_long)
+    results: List[dict] = []
+    for station_code in readings_by_station.keys():
+        try:
+            results.extend(
+                _forecast_station(
+                    station_code, grid, models_dir, now, DEFAULT_HORIZONS, rain=rain
+                )
+            )
+        except Exception as error:
+            logger.error(f"Forecast failed for station {station_code}: {error}")
+    return results
+
+
+def get_latest_forecasts(
+    db_url: Optional[str] = None,
+    models_dir: Union[str, Path] = DEFAULT_MODELS_DIR,
+    hours: int = 336,
+) -> List[dict]:
+    """Convenience wrapper for web_api: load the latest window from the DB/API and forecast.
+
+    Raises FileNotFoundError when no trained model bundle exists at all, so the
+    API can 503 instead of serving purely heuristic output as if it were a forecast.
+    """
+    from .data import load_latest
+
+    if not sorted(Path(models_dir).glob("flood_*.joblib")):
+        raise FileNotFoundError(f"no trained model bundles in {models_dir}")
+
+    df_long = load_latest(db_url=db_url, hours=hours)
+    readings_by_station: Dict[str, List[dict]] = {}
+    if not df_long.empty:
+        for station_code, group in df_long.groupby("station_code"):
+            readings_by_station[station_code] = group[
+                ["timestamp", "water_level", "discharge"]
+            ].to_dict("records")
+
+    expected_stations = set(features.UPSTREAM_LEADS.keys())
+    for missing_station in expected_stations - set(readings_by_station.keys()):
+        logger.warning(
+            f"No recent data for station {missing_station}; omitting from forecasts"
+        )
+
+    # Live rain: trailing days + next-48h forecast. On fetch failure pass an
+    # EMPTY series (not None) so rain-trained bundles still find their columns
+    # (as NaN) and serve model output instead of tripping the feature guard.
+    from .rain import serving_series
+
+    rain = serving_series()
+    if rain is None:
+        logger.warning("live rain unavailable; rain features will be NaN")
+        rain = pd.Series(dtype=float)
+
+    return get_forecasts(readings_by_station, models_dir=models_dir, rain=rain)

src/ml/rain.py

@@ -0,0 +1,237 @@
+"""Open-Meteo rainfall series for the upper Ping catchment.
+
+One consistent source for training AND serving: the Open-Meteo forecast-model
+archive (historical-forecast-api, 2021-03 onward) supplies hourly
+precipitation at five catchment points above P.1; the live forecast endpoint
+supplies the same series for recent days plus the next 48 h. Timestamps are
+Asia/Bangkok local, matching the measurement grid. Rows before 2021-03 simply
+have no rain data — HistGradientBoosting handles the NaNs natively.
+
+The forward-looking sum built from this series is a legitimate *forecast*
+feature, not label leakage: the series never contains river observations, and
+at serving time the future values come from an actual weather forecast.
+"""
+
+import datetime
+import logging
+from pathlib import Path
+from typing import Iterable, List, Optional, Tuple
+
+import pandas as pd
+import requests
+
+logger = logging.getLogger(__name__)
+
+# (name, lat, lon) — upper Ping catchment above P.1, headwaters to city
+CATCHMENT_POINTS: Tuple[Tuple[str, float, float], ...] = (
+    ("chiang_dao", 19.37, 98.97),
+    ("mae_taeng", 19.12, 98.94),
+    ("mae_ngat", 19.17, 99.05),
+    ("mae_rim", 18.92, 98.92),
+    ("chiang_mai", 18.79, 99.00),
+)
+
+HISTORY_URL = "https://historical-forecast-api.open-meteo.com/v1/forecast"
+FORECAST_URL = "https://api.open-meteo.com/v1/forecast"
+HISTORY_START = "2021-03-23"  # archive begins here
+CACHE_FILE = "rain_openmeteo.csv.gz"
+
+
+def _points_params() -> dict:
+    return {
+        "latitude": ",".join(str(lat) for _, lat, _ in CATCHMENT_POINTS),
+        "longitude": ",".join(str(lon) for _, _, lon in CATCHMENT_POINTS),
+        "hourly": "precipitation",
+        "timezone": "Asia/Bangkok",
+    }
+
+
+def _parse_multi(payload, columns: Iterable[str]) -> pd.DataFrame:
+    """Open-Meteo returns a list when multiple coordinates are requested."""
+    results = payload if isinstance(payload, list) else [payload]
+    frames = {}
+    for name, result in zip(columns, results):
+        hourly = result.get("hourly", {})
+        idx = pd.to_datetime(hourly.get("time", []))
+        frames[name] = pd.Series(hourly.get("precipitation", []), index=idx)
+    df = pd.DataFrame(frames)
+    df.index.name = "timestamp"
+    return df
+
+
+def fetch_history(
+    start: str, end: str, session: Optional[requests.Session] = None
+) -> pd.DataFrame:
+    """Hourly precipitation for all catchment points over [start, end]."""
+    session = session or requests.Session()
+    response = session.get(
+        HISTORY_URL,
+        params={**_points_params(), "start_date": start, "end_date": end},
+        timeout=120,
+    )
+    response.raise_for_status()
+    return _parse_multi(response.json(), [p[0] for p in CATCHMENT_POINTS])
+
+
+def fetch_forecast(
+    past_days: int = 5,
+    forecast_days: int = 2,
+    session: Optional[requests.Session] = None,
+) -> pd.DataFrame:
+    """Recent + next-48h precipitation from the live forecast endpoint."""
+    session = session or requests.Session()
+    response = session.get(
+        FORECAST_URL,
+        params={
+            **_points_params(),
+            "past_days": past_days,
+            "forecast_days": forecast_days,
+        },
+        timeout=60,
+    )
+    response.raise_for_status()
+    return _parse_multi(response.json(), [p[0] for p in CATCHMENT_POINTS])
+
+
+def load_history(
+    cache_dir: Path = Path("models/cache"),
+    end: Optional[datetime.date] = None,
+    refresh: bool = True,
+) -> Optional[pd.DataFrame]:
+    """Cached catchment rain history from 2021-03 to ~today.
+
+    Fetches year-sized chunks on first use (~6 requests), then only extends
+    the tail. Returns None when the API is unreachable and no cache exists.
+    """
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    cache_path = cache_dir / CACHE_FILE
+    end = end or datetime.date.today()
+
+    cached: Optional[pd.DataFrame] = None
+    if cache_path.exists():
+        cached = pd.read_csv(cache_path, index_col=0, parse_dates=True)
+
+    fetch_from = pd.Timestamp(HISTORY_START)
+    if cached is not None and len(cached):
+        fetch_from = cached.index.max() - pd.Timedelta(days=2)  # re-fetch tail
+    if not refresh and cached is not None:
+        return cached
+
+    chunks: List[pd.DataFrame] = []
+    cursor = fetch_from.date()
+    try:
+        while cursor <= end:
+            chunk_end = min(
+                datetime.date(cursor.year, 12, 31), end
+            )
+            chunks.append(
+                fetch_history(cursor.isoformat(), chunk_end.isoformat())
+            )
+            cursor = datetime.date(cursor.year + 1, 1, 1)
+    except Exception as error:
+        logger.warning(f"Open-Meteo history fetch failed: {error}")
+        if not chunks and cached is None:
+            return None
+
+    if chunks:
+        fresh = pd.concat(chunks)
+        combined = (
+            pd.concat([cached[cached.index < fresh.index.min()], fresh])
+            if cached is not None
+            else fresh
+        )
+        combined = combined[~combined.index.duplicated(keep="last")].sort_index()
+        combined.to_csv(cache_path, compression="gzip")
+        return combined
+    return cached
+
+
+def catchment_mean(df: Optional[pd.DataFrame]) -> Optional[pd.Series]:
+    """Single catchment-average hourly rain series (mm)."""
+    if df is None or df.empty:
+        return None
+    return df.mean(axis=1)
+
+
+def serving_series() -> Optional[pd.Series]:
+    """Catchment rain for inference: trailing days + the next 48 h forecast."""
+    try:
+        return catchment_mean(fetch_forecast())
+    except Exception as error:
+        logger.warning(f"Open-Meteo forecast fetch failed: {error}")
+        return None
+
+
+def backfill_db(engine, db_type: str, chunk_rows: int = 5000) -> int:
+    """Push the full Open-Meteo history (2021+) into openmeteo_rain.
+
+    Loads (or fetches) the archive cache and upserts in chunks; idempotent,
+    safe to re-run, and safe alongside the hourly live writer.
+    """
+    history = load_history()
+    if history is None or history.empty:
+        logger.error("no rain history available to backfill")
+        return 0
+    total = 0
+    for start in range(0, len(history), chunk_rows):
+        part = history.iloc[start: start + chunk_rows]
+        total += save_to_db(part, engine, db_type)
+        logger.info(f"openmeteo_rain backfill: {total}/{len(history)} rows")
+    return total
+
+
+def save_to_db(df: pd.DataFrame, engine, db_type: str) -> int:
+    """Upsert per-point + catchment-mean hourly rain into openmeteo_rain.
+
+    Called by the leader worker's hourly precompute with the live forecast
+    frame, so the DB accumulates both what fell (past rows are the model
+    analysis) and what was forecast (future rows, overwritten as they become
+    past). The ML training path reads Open-Meteo's own archive, not this
+    table — this is for dashboards, SQL analysis, and source independence.
+    """
+    if df is None or df.empty:
+        return 0
+    from sqlalchemy import text
+
+    point_cols = [p[0] for p in CATCHMENT_POINTS]
+    ddl_cols = ", ".join(f"{c} NUMERIC(6,2)" for c in point_cols)
+    ddl = (
+        "CREATE TABLE IF NOT EXISTS openmeteo_rain ("
+        "timestamp TIMESTAMP PRIMARY KEY, "
+        f"{ddl_cols}, catchment_mean NUMERIC(6,2), "
+        "created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP)"
+    )
+    cols = ["timestamp"] + point_cols + ["catchment_mean"]
+    placeholders = ", ".join(f":{c}" for c in cols)
+    updates = ", ".join(
+        f"{c} = "
+        + (f"VALUES({c})" if db_type == "mysql" else f"EXCLUDED.{c}")
+        for c in cols[1:]
+    )
+    if db_type == "mysql":
+        sql = (
+            f"INSERT INTO openmeteo_rain ({', '.join(cols)}) VALUES ({placeholders}) "
+            f"ON DUPLICATE KEY UPDATE {updates}"
+        )
+    else:
+        sql = (
+            f"INSERT INTO openmeteo_rain ({', '.join(cols)}) VALUES ({placeholders}) "
+            f"ON CONFLICT (timestamp) DO UPDATE SET {updates}"
+        )
+    mean = df.mean(axis=1)
+    params = [
+        {
+            "timestamp": ts.to_pydatetime(),
+            **{c: (None if pd.isna(row[c]) else float(row[c])) for c in point_cols},
+            "catchment_mean": None if pd.isna(mean.loc[ts]) else float(mean.loc[ts]),
+        }
+        for ts, row in df.iterrows()
+    ]
+    try:
+        with engine.begin() as conn:
+            conn.execute(text(ddl))
+            conn.execute(text(sql), params)
+        return len(params)
+    except Exception as error:
+        logger.error(f"openmeteo_rain save failed: {error}")
+        return 0

src/ml/train.py

@@ -0,0 +1,583 @@
+"""Training CLI for the Ping River flood forecast models.
+
+Per station: build the feature/label matrix once, evaluate with a strict
+temporal holdout (Split B), then refit each head on the full record for the
+deployed artifact. Hyperparameters are fixed (chosen via an earlier Split A
+sweep, not repeated here) -- no random search, no shuffling, no sklearn
+early_stopping (its internal validation split is random and would leak
+across time).
+"""
+
+import argparse
+import datetime
+import json
+import logging
+import subprocess
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+import joblib
+import numpy as np
+import pandas as pd
+import sklearn
+from sklearn.ensemble import (
+    HistGradientBoostingClassifier,
+    HistGradientBoostingRegressor,
+)
+from sklearn.metrics import (
+    average_precision_score,
+    brier_score_loss,
+    mean_absolute_error,
+    mean_squared_error,
+)
+
+from . import features
+from .data import DEFAULT_API_URL, load_measurements, resolve_db_url
+
+logger = logging.getLogger(__name__)
+
+HORIZONS: Tuple[int, ...] = (6, 12, 24)
+SPLIT_B_TRAIN_END = "2024-12-31"
+SPLIT_B_TEST_START = "2025-01-01"
+SPLIT_B_TEST_END = "2026-08-10"
+MIN_POSITIVES_FOR_CLASSIFIER = 30
+MIN_SIGMA = 0.15
+MIN_ROWS_TO_TRAIN = 200
+MIN_ROWS_FOR_HEAD = 50
+
+HGB_PARAMS = {
+    "max_iter": 300,
+    "learning_rate": 0.06,
+    "max_leaf_nodes": 31,
+    "min_samples_leaf": 50,
+    "l2_regularization": 1.0,
+    "early_stopping": False,
+    "random_state": 42,
+}
+
+
+def _git_short_sha() -> str:
+    try:
+        result = subprocess.run(
+            ["git", "rev-parse", "--short", "HEAD"],
+            capture_output=True,
+            text=True,
+            timeout=5,
+            check=True,
+        )
+        sha = result.stdout.strip()
+        return sha or "nogit"
+    except Exception:
+        return "nogit"
+
+
+def _make_regressor(overrides: Optional[dict] = None) -> HistGradientBoostingRegressor:
+    params = {**HGB_PARAMS, **(overrides or {})}
+    return HistGradientBoostingRegressor(loss="squared_error", **params)
+
+
+def _make_classifier(
+    overrides: Optional[dict] = None,
+) -> HistGradientBoostingClassifier:
+    params = {**HGB_PARAMS, **(overrides or {})}
+    return HistGradientBoostingClassifier(**params)
+
+
+def _safe_fit(
+    estimator,
+    X: pd.DataFrame,
+    y: pd.Series,
+    head_key: str,
+    skipped_heads: Dict[str, str],
+):
+    """Fit an estimator, converting any failure (e.g. HistGradientBoosting's binning
+    step rejecting an all-NaN/constant feature column) into a recorded skip rather
+    than a station-killing exception."""
+    try:
+        estimator.fit(X, y)
+        return estimator
+    except Exception as error:
+        skipped_heads[head_key] = f"fit failed: {error}"
+        logger.warning(f"{head_key}: fit failed, skipping ({error})")
+        return None
+
+
+def _recall_at_far(
+    y_true: np.ndarray, y_score: np.ndarray, target_far: float
+) -> Optional[float]:
+    """Recall at the score threshold whose false-positive rate over true negatives is <= target_far."""
+    y_true = np.asarray(y_true)
+    y_score = np.asarray(y_score)
+    neg_scores = np.sort(y_score[y_true == 0])[::-1]
+    n_pos = int((y_true == 1).sum())
+    n_neg = len(neg_scores)
+    if n_pos == 0 or n_neg == 0:
+        return None
+    k = int(np.floor(target_far * n_neg))
+    threshold = neg_scores[k - 1] if k > 0 else neg_scores[0] + 1e-9
+    predicted_positive = y_score >= threshold
+    tp = int(np.sum(predicted_positive & (y_true == 1)))
+    return tp / n_pos
+
+
+def _p_warning_series(
+    head, reg, X: pd.DataFrame, threshold: float, sigma: float
+) -> pd.Series:
+    """Model score if a classifier head exists, else the sigmoid-derived fallback probability."""
+    if head is not None:
+        return pd.Series(head.predict_proba(X)[:, 1], index=X.index)
+    # reg predicts the RISE over current level; add the level back
+    predicted_max = pd.Series(reg.predict(X), index=X.index) + X["level"]
+    return 1.0 / (1.0 + np.exp(-(predicted_max - threshold) / sigma))
+
+
+def _find_events(observed_level: pd.Series, warn_thr: float) -> List[dict]:
+    """Group contiguous observed hours >= warn_thr into flood events."""
+    above = observed_level >= warn_thr
+    events: List[dict] = []
+    start = None
+    prev_t = None
+    for t, is_above in above.items():
+        if is_above and start is None:
+            start = t
+        elif not is_above and start is not None:
+            window = observed_level.loc[start:prev_t]
+            events.append(
+                {
+                    "crossed_warn_at": start,
+                    "peak_time": window.idxmax(),
+                    "peak_level": float(window.max()),
+                }
+            )
+            start = None
+        prev_t = t
+    if start is not None:
+        window = observed_level.loc[start:]
+        events.append(
+            {
+                "crossed_warn_at": start,
+                "peak_time": window.idxmax(),
+                "peak_level": float(window.max()),
+            }
+        )
+    return events
+
+
+def _first_alert_at(p_series: pd.Series, crossed_at, lookback_h: int = 48):
+    """Earliest time p_warning was sustained (>=0.5 for 2 consecutive hours) within the prior lookback_h."""
+    window = p_series.loc[crossed_at - pd.Timedelta(hours=lookback_h) : crossed_at]
+    sustained = (window >= 0.5) & (window.shift(1) >= 0.5)
+    hits = sustained[sustained].index
+    if len(hits) == 0:
+        return None
+    return hits.min() - pd.Timedelta(hours=1)
+
+
+def _events_with_lead_time(
+    observed_level_test: pd.Series, warn_thr: float, p_warning_test: pd.Series
+) -> List[dict]:
+    events = _find_events(observed_level_test, warn_thr)
+    for event in events:
+        first_alert_at = _first_alert_at(p_warning_test, event["crossed_warn_at"])
+        event["first_alert_at"] = (
+            first_alert_at.isoformat() if first_alert_at is not None else None
+        )
+        if first_alert_at is not None:
+            lead_hours = (
+                event["crossed_warn_at"] - first_alert_at
+            ).total_seconds() / 3600.0
+        else:
+            lead_hours = None
+        event["lead_hours"] = lead_hours
+        event["crossed_warn_at"] = event["crossed_warn_at"].isoformat()
+        event["peak_time"] = event["peak_time"].isoformat()
+    return events
+
+
+def train_station(
+    df_long: pd.DataFrame,
+    station: str,
+    horizons: Tuple[int, ...] = HORIZONS,
+    skip_eval: bool = False,
+    hgb_overrides: Optional[dict] = None,
+    split_train_end: str = SPLIT_B_TRAIN_END,
+    split_test_start: str = SPLIT_B_TEST_START,
+    split_test_end: str = SPLIT_B_TEST_END,
+    rain: Optional[pd.Series] = None,
+) -> Tuple[Optional[dict], dict]:
+    """Train every head for one station. Returns (bundle_or_None, station_metrics)."""
+    X, Y, meta = features.build_matrix(df_long, station, horizons, rain=rain)
+    if meta["n_rows"] < MIN_ROWS_TO_TRAIN:
+        return None, {
+            "status": "failed",
+            "reason": f"only {meta['n_rows']} usable rows (< {MIN_ROWS_TO_TRAIN})",
+        }
+
+    warn_thr, danger_thr = features.get_thresholds(station)
+    feature_names = list(X.columns)
+
+    if skip_eval:
+        train_mask = pd.Series(True, index=X.index)
+        test_mask = pd.Series(False, index=X.index)
+    else:
+        train_mask = X.index <= pd.Timestamp(split_train_end)
+        test_mask = (X.index >= pd.Timestamp(split_test_start)) & (
+            X.index <= pd.Timestamp(split_test_end)
+        )
+    X_train, Y_train = X.loc[train_mask], Y.loc[train_mask]
+    X_test, Y_test = X.loc[test_mask], Y.loc[test_mask]
+    eval_X, eval_Y = (X, Y) if skip_eval else (X_train, Y_train)
+
+    heads: Dict[str, object] = {}
+    sigma: Dict[int, float] = {}
+    skipped_heads: Dict[str, str] = {}
+    per_horizon: Dict[int, dict] = {}
+    observed_grid = features.make_hourly_grid(df_long).observed
+
+    for h in horizons:
+        max_col, warn_col, danger_col = (
+            f"max_level_{h}",
+            f"exceed_warn_{h}",
+            f"exceed_danger_{h}",
+        )
+        horizon_metrics: dict = {}
+
+        # --- regression head (rise to future max) ---
+        # Target = future max MINUS current level ("rise"). Rises are far more
+        # stationary than absolute stages, which softens the cannot-exceed-
+        # training-max ceiling: on the rolling-origin harness (2026-08-12) the
+        # rise target moved P.1 first-alert leads from +0h to +6/+46h and cut
+        # the 2024 record-peak underprediction. Prediction = rise + level.
+        reg_labeled = eval_Y[max_col].notna()
+        reg = None
+        if reg_labeled.sum() >= MIN_ROWS_FOR_HEAD:
+            rise_target = (
+                eval_Y.loc[reg_labeled, max_col] - eval_X.loc[reg_labeled, "level"]
+            )
+            reg = _safe_fit(
+                _make_regressor(hgb_overrides),
+                eval_X.loc[reg_labeled],
+                rise_target,
+                f"max_{h}",
+                skipped_heads,
+            )
+        else:
+            skipped_heads[f"max_{h}"] = f"only {int(reg_labeled.sum())} labeled rows"
+
+        sigma_h = MIN_SIGMA
+        if reg is not None and not skip_eval:
+            test_labeled = Y_test[max_col].notna()
+            if test_labeled.sum() > 0:
+                y_true = Y_test.loc[test_labeled, max_col]
+                y_pred = (
+                    reg.predict(X_test.loc[test_labeled])
+                    + X_test.loc[test_labeled, "level"].to_numpy()
+                )
+                residuals = y_true.to_numpy() - y_pred
+                sigma_h = max(float(np.std(residuals)), MIN_SIGMA)
+                horizon_metrics["n_test"] = int(test_labeled.sum())
+                horizon_metrics["mae"] = float(mean_absolute_error(y_true, y_pred))
+                horizon_metrics["rmse"] = float(
+                    np.sqrt(mean_squared_error(y_true, y_pred))
+                )
+                above_2m = y_true >= 2.0
+                horizon_metrics["mae_above_2m"] = (
+                    float(mean_absolute_error(y_true[above_2m], y_pred[above_2m]))
+                    if above_2m.any()
+                    else None
+                )
+        sigma[h] = sigma_h
+        horizon_metrics["sigma"] = sigma_h
+
+        # --- classification heads (warn / danger) ---
+        p_warning_test = None
+        for label_name, col, thr in (
+            ("warn", warn_col, warn_thr),
+            ("danger", danger_col, danger_thr),
+        ):
+            train_labeled = eval_Y[col].notna()
+            n_pos = (
+                int(eval_Y.loc[train_labeled, col].sum()) if train_labeled.any() else 0
+            )
+            head_key = f"{label_name}_{h}"
+            clf = None
+            if n_pos >= MIN_POSITIVES_FOR_CLASSIFIER:
+                clf = _safe_fit(
+                    _make_classifier(hgb_overrides),
+                    eval_X.loc[train_labeled],
+                    eval_Y.loc[train_labeled, col],
+                    head_key,
+                    skipped_heads,
+                )
+            else:
+                skipped_heads[
+                    head_key
+                ] = f"only {n_pos} positives in train span (< {MIN_POSITIVES_FOR_CLASSIFIER})"
+            heads[head_key] = clf
+
+            if not skip_eval:
+                test_labeled = Y_test[col].notna()
+                horizon_metrics[f"base_rate_{label_name}"] = (
+                    float(Y_test.loc[test_labeled, col].mean())
+                    if test_labeled.any()
+                    else None
+                )
+                if (
+                    clf is not None
+                    and test_labeled.sum() > 0
+                    and Y_test.loc[test_labeled, col].nunique() > 1
+                ):
+                    y_true = Y_test.loc[test_labeled, col]
+                    y_score = clf.predict_proba(X_test.loc[test_labeled])[:, 1]
+                    horizon_metrics[f"pr_auc_{label_name}"] = float(
+                        average_precision_score(y_true, y_score)
+                    )
+                    horizon_metrics[f"brier_{label_name}"] = float(
+                        brier_score_loss(y_true, y_score)
+                    )
+                    horizon_metrics[f"recall_{label_name}_at_far1pct"] = _recall_at_far(
+                        y_true, y_score, 0.01
+                    )
+                    horizon_metrics[f"recall_{label_name}_at_far5pct"] = _recall_at_far(
+                        y_true, y_score, 0.05
+                    )
+                else:
+                    horizon_metrics[f"pr_auc_{label_name}"] = None
+                    horizon_metrics[f"brier_{label_name}"] = None
+                    horizon_metrics[f"recall_{label_name}_at_far1pct"] = None
+                    horizon_metrics[f"recall_{label_name}_at_far5pct"] = None
+
+            if label_name == "warn" and not skip_eval and reg is not None:
+                p_warning_test = _p_warning_series(clf, reg, X_test, thr, sigma_h)
+
+        per_horizon[h] = horizon_metrics
+        heads[f"max_{h}"] = reg
+
+        if not skip_eval and reg is not None and p_warning_test is not None:
+            observed_test_level = observed_grid.get((station, "water_level"))
+            if observed_test_level is not None:
+                observed_test_level = observed_test_level.loc[
+                    observed_test_level.index.isin(X_test.index)
+                ]
+                per_horizon[h]["events"] = _events_with_lead_time(
+                    observed_test_level, warn_thr, p_warning_test
+                )
+
+    # --- full refit on the ENTIRE record for the deployed artifact ---
+    # This may include/exclude different heads than the eval-phase gate above (the
+    # full record has more labeled rows), so skip reasons are re-derived here --
+    # skipped_heads must reflect what actually ends up in the saved bundle.
+    final_heads: Dict[str, object] = {}
+    for h in horizons:
+        max_col, warn_col, danger_col = (
+            f"max_level_{h}",
+            f"exceed_warn_{h}",
+            f"exceed_danger_{h}",
+        )
+        head_key = f"max_{h}"
+        labeled = Y[max_col].notna()
+        if labeled.sum() >= MIN_ROWS_FOR_HEAD:
+            reg = _safe_fit(
+                _make_regressor(hgb_overrides),
+                X.loc[labeled],
+                Y.loc[labeled, max_col] - X.loc[labeled, "level"],  # rise target
+                head_key,
+                skipped_heads,
+            )
+            final_heads[head_key] = reg
+            if reg is not None:
+                skipped_heads.pop(head_key, None)
+        else:
+            skipped_heads[head_key] = f"only {int(labeled.sum())} labeled rows"
+            final_heads[head_key] = None
+
+        for label_name, col in (("warn", warn_col), ("danger", danger_col)):
+            head_key = f"{label_name}_{h}"
+            train_labeled = Y[col].notna()
+            n_pos = int(Y.loc[train_labeled, col].sum()) if train_labeled.any() else 0
+            if n_pos >= MIN_POSITIVES_FOR_CLASSIFIER:
+                clf = _safe_fit(
+                    _make_classifier(hgb_overrides),
+                    X.loc[train_labeled],
+                    Y.loc[train_labeled, col],
+                    head_key,
+                    skipped_heads,
+                )
+                final_heads[head_key] = clf
+                if clf is not None:
+                    skipped_heads.pop(head_key, None)
+            else:
+                skipped_heads[
+                    head_key
+                ] = f"only {n_pos} positives in train span (< {MIN_POSITIVES_FOR_CLASSIFIER})"
+                final_heads[head_key] = None
+
+    # v3 = rise target + Open-Meteo rain features; v2 = rise target only
+    version_prefix = "hgb-v3" if "rain_24h" in feature_names else "hgb-v2"
+    bundle = {
+        "station_code": station,
+        "model_version": f"{version_prefix}+{_git_short_sha()}",
+        # v2+: regression heads predict the RISE over the current level; the
+        # serving side must add the level back. Old v1 bundles lack this key.
+        "regression_target": "rise",
+        "trained_at": datetime.datetime.now().isoformat(),
+        "sklearn_version": sklearn.__version__,
+        "feature_names": feature_names,
+        "horizons": list(horizons),
+        "thresholds": {"warning": warn_thr, "danger": danger_thr},
+        "heads": final_heads,
+        "sigma": sigma,
+        "skipped_heads": skipped_heads,
+        "train_span": meta["span"],
+        "n_train_rows": meta["n_rows"],
+    }
+    station_metrics = {"status": "trained", "per_horizon": per_horizon}
+    return bundle, station_metrics
+
+
+def train_all(
+    df_long: pd.DataFrame,
+    stations: List[str],
+    horizons: Tuple[int, ...] = HORIZONS,
+    models_dir: Path = Path("models"),
+    skip_eval: bool = False,
+    hgb_overrides: Optional[dict] = None,
+    use_rain: bool = True,
+) -> dict:
+    """Train and save every requested station's models. Returns the metrics.json payload."""
+    models_dir = Path(models_dir)
+    models_dir.mkdir(parents=True, exist_ok=True)
+
+    # Catchment rain (Open-Meteo archive, 2021+). Optional: without it the
+    # models train as v2 (no rain columns) and still serve correctly.
+    rain_series = None
+    if use_rain:
+        try:
+            from . import rain as rain_mod
+
+            rain_series = rain_mod.catchment_mean(rain_mod.load_history())
+        except Exception as error:
+            logger.warning(f"rain history unavailable, training without it: {error}")
+    if rain_series is not None:
+        logger.info(
+            f"rain series: {rain_series.index.min()} .. {rain_series.index.max()}"
+        )
+    version_prefix = "hgb-v3" if rain_series is not None else "hgb-v2"
+    model_version = f"{version_prefix}+{_git_short_sha()}"
+
+    station_results: Dict[str, dict] = {}
+    for station in stations:
+        if station in features.NOT_TRAINABLE:
+            reason = features.NOT_TRAINABLE[station]
+            logger.info(f"{station}: heuristic ({reason})")
+            station_results[station] = {"status": "heuristic", "reason": reason}
+            continue
+        try:
+            bundle, station_metrics = train_station(
+                df_long,
+                station,
+                horizons,
+                skip_eval=skip_eval,
+                hgb_overrides=hgb_overrides,
+                rain=rain_series,
+            )
+            if bundle is None:
+                logger.warning(f"{station}: failed ({station_metrics.get('reason')})")
+                station_results[station] = station_metrics
+                continue
+            joblib.dump(bundle, models_dir / f"flood_{station}.joblib")
+            logger.info(
+                f"{station}: trained, {bundle['n_train_rows']} rows, "
+                f"{len(bundle['skipped_heads'])} heads skipped"
+            )
+            station_results[station] = station_metrics
+        except Exception as error:
+            logger.error(f"{station}: failed with exception: {error}")
+            station_results[station] = {"status": "failed", "reason": str(error)}
+
+    metrics_payload = {
+        "generated_at": datetime.datetime.now().isoformat(),
+        "model_version": model_version,
+        "split": {
+            "train_end": SPLIT_B_TRAIN_END,
+            "test_start": SPLIT_B_TEST_START,
+            "test_end": SPLIT_B_TEST_END,
+        },
+        "stations": station_results,
+    }
+    with open(models_dir / "metrics.json", "w", encoding="utf-8") as handle:
+        json.dump(metrics_payload, handle, indent=2, default=str)
+    return metrics_payload
+
+
+def main(argv: Optional[List[str]] = None) -> None:
+    logging.basicConfig(
+        level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s"
+    )
+
+    parser = argparse.ArgumentParser(
+        description="Train Ping River flood forecast models"
+    )
+    parser.add_argument(
+        "--stations",
+        default="all",
+        help="'all' or a comma-separated list of station codes",
+    )
+    parser.add_argument("--models-dir", default="models")
+    parser.add_argument("--db-url", default=None)
+    parser.add_argument("--api-url", default=DEFAULT_API_URL)
+    parser.add_argument(
+        "--skip-eval",
+        action="store_true",
+        help="Refit-only fast path; skip Split B evaluation",
+    )
+    parser.add_argument(
+        "--start", default=None, help="ISO date; earliest measurement to load"
+    )
+    parser.add_argument(
+        "--end", default=None, help="ISO date; latest measurement to load"
+    )
+    parser.add_argument(
+        "--no-rain",
+        action="store_true",
+        help="train without the Open-Meteo rain features (v2-style bundles)",
+    )
+    args = parser.parse_args(argv)
+
+    if args.stations == "all":
+        stations = list(features.UPSTREAM_LEADS.keys())
+    else:
+        stations = [s.strip() for s in args.stations.split(",") if s.strip()]
+
+    start = datetime.datetime.fromisoformat(args.start) if args.start else None
+    end = datetime.datetime.fromisoformat(args.end) if args.end else None
+
+    logger.info(f"Loading measurements for {len(stations)} stations...")
+    df_long = load_measurements(
+        db_url=resolve_db_url(args.db_url),
+        stations=None,
+        start=start,
+        end=end,
+        api_url=args.api_url,
+    )
+    logger.info(
+        f"Loaded {len(df_long)} rows spanning {df_long['timestamp'].min()} .. {df_long['timestamp'].max()}"
+    )
+
+    metrics_payload = train_all(
+        df_long,
+        stations,
+        models_dir=Path(args.models_dir),
+        skip_eval=args.skip_eval,
+        use_rain=not args.no_rain,
+    )
+    trained = sum(
+        1 for s in metrics_payload["stations"].values() if s["status"] == "trained"
+    )
+    logger.info(
+        f"Done: {trained}/{len(stations)} stations trained. metrics.json written to {args.models_dir}"
+    )
+
+
+if __name__ == "__main__":
+    main()

src/models.py

@@ -5,8 +5,9 @@ Data models for water monitoring system
 
 from dataclasses import dataclass, field
 from datetime import datetime
-from typing import Optional, List, Dict, Any
 from enum import Enum
+from typing import Any, Dict, List, Optional
+
 
 class DatabaseType(Enum):
     SQLITE = "sqlite"
@@ -15,15 +16,18 @@ class DatabaseType(Enum):
     INFLUXDB = "influxdb"
     VICTORIAMETRICS = "victoriametrics"
 
+
 class StationStatus(Enum):
     ACTIVE = "active"
     INACTIVE = "inactive"
     MAINTENANCE = "maintenance"
     ERROR = "error"
 
+
 @dataclass
 class StationInfo:
     """Station information model"""
+
     station_id: int
     station_code: str
     thai_name: str
@@ -33,9 +37,11 @@ class StationInfo:
     geohash: Optional[str] = None
     status: StationStatus = StationStatus.ACTIVE
 
+
 @dataclass
 class WaterMeasurement:
     """Water measurement data model"""
+
     timestamp: datetime
     station_info: StationInfo
     water_level: float
@@ -44,29 +50,31 @@ class WaterMeasurement:
     discharge_unit: str = "cms"
     discharge_percent: Optional[float] = None
     status: StationStatus = StationStatus.ACTIVE
-    
+
     def to_dict(self) -> Dict[str, Any]:
         """Convert to dictionary for database storage"""
         return {
-            'timestamp': self.timestamp,
-            'station_id': self.station_info.station_id,
-            'station_code': self.station_info.station_code,
-            'station_name_en': self.station_info.english_name,
-            'station_name_th': self.station_info.thai_name,
-            'latitude': self.station_info.latitude,
-            'longitude': self.station_info.longitude,
-            'geohash': self.station_info.geohash,
-            'water_level': self.water_level,
-            'water_level_unit': self.water_level_unit,
-            'discharge': self.discharge,
-            'discharge_unit': self.discharge_unit,
-            'discharge_percent': self.discharge_percent,
-            'status': self.status.value
+            "timestamp": self.timestamp,
+            "station_id": self.station_info.station_id,
+            "station_code": self.station_info.station_code,
+            "station_name_en": self.station_info.english_name,
+            "station_name_th": self.station_info.thai_name,
+            "latitude": self.station_info.latitude,
+            "longitude": self.station_info.longitude,
+            "geohash": self.station_info.geohash,
+            "water_level": self.water_level,
+            "water_level_unit": self.water_level_unit,
+            "discharge": self.discharge,
+            "discharge_unit": self.discharge_unit,
+            "discharge_percent": self.discharge_percent,
+            "status": self.status.value,
         }
 
+
 @dataclass
 class DatabaseConfig:
     """Database configuration model"""
+
     db_type: DatabaseType
     connection_string: Optional[str] = None
     host: Optional[str] = None
@@ -76,18 +84,22 @@ class DatabaseConfig:
     password: Optional[str] = None
     additional_params: Dict[str, Any] = field(default_factory=dict)
 
+
 @dataclass
 class ScrapingResult:
     """Result of a scraping operation"""
+
     success: bool
     measurements_count: int
     error_message: Optional[str] = None
     timestamp: datetime = field(default_factory=datetime.now)
     processing_time_seconds: Optional[float] = None
 
+
 @dataclass
 class StationCreateRequest:
     """Request model for creating a new station"""
+
     station_code: str
     thai_name: str
     english_name: str
@@ -96,12 +108,14 @@ class StationCreateRequest:
     geohash: Optional[str] = None
     status: StationStatus = StationStatus.ACTIVE
 
+
 @dataclass
 class StationUpdateRequest:
     """Request model for updating an existing station"""
+
     thai_name: Optional[str] = None
     english_name: Optional[str] = None
     latitude: Optional[float] = None
     longitude: Optional[float] = None
     geohash: Optional[str] = None
-    status: Optional[StationStatus] = None
+    status: Optional[StationStatus] = None

src/postgres_history.py

@@ -0,0 +1,99 @@
+"""Read historical station measurements from PostgreSQL."""
+
+import datetime
+import os
+from typing import Dict, List, Optional, Tuple
+
+from sqlalchemy import create_engine, text
+
+# Stage-discharge rating curves: Q = a * (H - b)^c
+# Key: station_code, Value: (a, b, c)
+# Use linear fallback Q = slope * H if a curve is not defined.
+_RATING_CURVES: Dict[str, Tuple[float, float, float]] = {}
+_DEFAULT_LINEAR_SLOPE = 20.0  # m^3/s per meter
+
+
+def _calculate_discharge(
+    water_level: Optional[float], station_code: str = None
+) -> Optional[float]:
+    """Estimate discharge from water level using a rating curve or linear fallback."""
+    if water_level is None:
+        return None
+
+    curve = _RATING_CURVES.get(station_code)
+    if curve:
+        a, b, c = curve
+        h_excess = water_level - b
+        if h_excess <= 0:
+            return 0.0
+        return round(a * (h_excess**c), 2)
+
+    # Linear fallback: Q = slope * H
+    return round(_DEFAULT_LINEAR_SLOPE * water_level, 2)
+
+
+class PostgresHistory:
+    def __init__(self, connection_string: Optional[str] = None, engine=None):
+        connection_string = connection_string or os.getenv("POSTGRES_CONNECTION_STRING")
+        if engine is None and not connection_string:
+            raise RuntimeError("POSTGRES_CONNECTION_STRING is not configured")
+        self.engine = engine or create_engine(connection_string, pool_pre_ping=True)
+
+    def station_history(
+        self,
+        station_code: str,
+        start: datetime.datetime,
+        end: datetime.datetime,
+        limit: int = 2000,
+    ) -> List[Dict]:
+        if not 1 <= limit <= 100000:
+            raise ValueError("limit must be between 1 and 100000")
+        if start >= end:
+            raise ValueError("start must be before end")
+
+        query = text(
+            """
+            SELECT m.timestamp, s.station_code, m.water_level,
+                   m.discharge, m.discharge_percent
+            FROM water_measurements m
+            JOIN stations s ON m.station_id = s.id
+            WHERE s.station_code = :station_code
+              AND m.timestamp >= :start_time
+              AND m.timestamp <= :end_time
+            ORDER BY m.timestamp ASC
+            LIMIT :limit
+            """
+        )
+        with self.engine.connect() as connection:
+            rows = connection.execute(
+                query,
+                {
+                    "station_code": station_code,
+                    "start_time": start,
+                    "end_time": end,
+                    "limit": limit,
+                },
+            )
+            result = []
+            for row in rows:
+                timestamp = row[0]
+                if isinstance(timestamp, str):
+                    timestamp = datetime.datetime.fromisoformat(timestamp)
+                station_code = row[1]
+                water_level = float(row[2]) if row[2] is not None else None
+                discharge = float(row[3]) if row[3] is not None else None
+                # Estimate discharge from water level if DB value is missing
+                if discharge is None and water_level is not None:
+                    discharge = _calculate_discharge(water_level, station_code)
+                result.append(
+                    {
+                        "timestamp": timestamp,
+                        "station_code": station_code,
+                        "water_level": water_level,
+                        "discharge": discharge,
+                        "discharge_percent": float(row[4])
+                        if row[4] is not None
+                        else None,
+                    }
+                )
+            return result

src/rate_limiter.py

@@ -3,22 +3,23 @@
 Rate limiting utilities for API requests
 """
 
-import time
+import logging
 import threading
-from typing import Dict, Optional
+import time
 from collections import deque
 from datetime import datetime, timedelta
-import logging
+from typing import Dict, Optional
 
 logger = logging.getLogger(__name__)
 
+
 class RateLimiter:
     """Token bucket rate limiter"""
-    
+
     def __init__(self, max_requests: int, time_window_seconds: int):
         """
         Initialize rate limiter
-        
+
         Args:
             max_requests: Maximum number of requests allowed
             time_window_seconds: Time window in seconds
@@ -27,33 +28,33 @@ class RateLimiter:
         self.time_window = time_window_seconds
         self.requests = deque()
         self._lock = threading.Lock()
-    
+
     def is_allowed(self) -> bool:
         """Check if a request is allowed"""
         with self._lock:
             now = time.time()
-            
+
             # Remove old requests outside the time window
             while self.requests and self.requests[0] <= now - self.time_window:
                 self.requests.popleft()
-            
+
             # Check if we can make a new request
             if len(self.requests) < self.max_requests:
                 self.requests.append(now)
                 return True
-            
+
             return False
-    
+
     def wait_time(self) -> float:
         """Get the time to wait before next request is allowed"""
         with self._lock:
             if len(self.requests) < self.max_requests:
                 return 0.0
-            
+
             # Time until the oldest request expires
             oldest_request = self.requests[0]
             return max(0.0, (oldest_request + self.time_window) - time.time())
-    
+
     def wait_if_needed(self):
         """Block until a request is allowed"""
         wait_time = self.wait_time()
@@ -61,13 +62,16 @@ class RateLimiter:
             logger.info(f"Rate limit reached, waiting {wait_time:.2f} seconds")
             time.sleep(wait_time)
 
+
 class AdaptiveRateLimiter:
     """Adaptive rate limiter that adjusts based on response times"""
-    
-    def __init__(self, initial_rate: float = 1.0, min_rate: float = 0.1, max_rate: float = 10.0):
+
+    def __init__(
+        self, initial_rate: float = 1.0, min_rate: float = 0.1, max_rate: float = 10.0
+    ):
         """
         Initialize adaptive rate limiter
-        
+
         Args:
             initial_rate: Initial requests per second
             min_rate: Minimum requests per second
@@ -79,48 +83,51 @@ class AdaptiveRateLimiter:
         self.last_request_time = 0.0
         self.response_times = deque(maxlen=10)
         self._lock = threading.Lock()
-    
+
     def wait_and_record(self, response_time: Optional[float] = None):
         """Wait for rate limit and record response time"""
         with self._lock:
             now = time.time()
-            
+
             # Calculate wait time based on current rate
             time_since_last = now - self.last_request_time
             min_interval = 1.0 / self.current_rate
-            
+
             if time_since_last < min_interval:
                 wait_time = min_interval - time_since_last
                 time.sleep(wait_time)
                 now = time.time()
-            
+
             self.last_request_time = now
-            
+
             # Record response time and adjust rate
             if response_time is not None:
                 self.response_times.append(response_time)
                 self._adjust_rate()
-    
+
     def _adjust_rate(self):
         """Adjust rate based on recent response times"""
         if len(self.response_times) < 3:
             return
-        
+
         avg_response_time = sum(self.response_times) / len(self.response_times)
-        
+
         # Decrease rate if responses are slow
         if avg_response_time > 5.0:  # 5 seconds
             self.current_rate = max(self.min_rate, self.current_rate * 0.8)
-            logger.info(f"Decreased rate to {self.current_rate:.2f} req/s due to slow responses")
-        
+            logger.info(
+                f"Decreased rate to {self.current_rate:.2f} req/s due to slow responses"
+            )
+
         # Increase rate if responses are fast
         elif avg_response_time < 1.0:  # 1 second
             self.current_rate = min(self.max_rate, self.current_rate * 1.1)
             logger.debug(f"Increased rate to {self.current_rate:.2f} req/s")
 
+
 class RequestTracker:
     """Track API request statistics"""
-    
+
     def __init__(self):
         self.total_requests = 0
         self.successful_requests = 0
@@ -129,39 +136,45 @@ class RequestTracker:
         self.last_request_time = None
         self.error_count_by_type = {}
         self._lock = threading.Lock()
-    
-    def record_request(self, success: bool, response_time: float, error_type: Optional[str] = None):
+
+    def record_request(
+        self, success: bool, response_time: float, error_type: Optional[str] = None
+    ):
         """Record a request"""
         with self._lock:
             self.total_requests += 1
             self.total_response_time += response_time
             self.last_request_time = datetime.now()
-            
+
             if success:
                 self.successful_requests += 1
             else:
                 self.failed_requests += 1
                 if error_type:
-                    self.error_count_by_type[error_type] = self.error_count_by_type.get(error_type, 0) + 1
-    
+                    self.error_count_by_type[error_type] = (
+                        self.error_count_by_type.get(error_type, 0) + 1
+                    )
+
     def get_stats(self) -> Dict[str, any]:
         """Get request statistics"""
         with self._lock:
             if self.total_requests == 0:
                 return {
-                    'total_requests': 0,
-                    'success_rate': 0.0,
-                    'average_response_time': 0.0,
-                    'last_request_time': None,
-                    'error_breakdown': {}
+                    "total_requests": 0,
+                    "success_rate": 0.0,
+                    "average_response_time": 0.0,
+                    "last_request_time": None,
+                    "error_breakdown": {},
                 }
-            
+
             return {
-                'total_requests': self.total_requests,
-                'successful_requests': self.successful_requests,
-                'failed_requests': self.failed_requests,
-                'success_rate': self.successful_requests / self.total_requests,
-                'average_response_time': self.total_response_time / self.total_requests,
-                'last_request_time': self.last_request_time.isoformat() if self.last_request_time else None,
-                'error_breakdown': dict(self.error_count_by_type)
-            }
+                "total_requests": self.total_requests,
+                "successful_requests": self.successful_requests,
+                "failed_requests": self.failed_requests,
+                "success_rate": self.successful_requests / self.total_requests,
+                "average_response_time": self.total_response_time / self.total_requests,
+                "last_request_time": self.last_request_time.isoformat()
+                if self.last_request_time
+                else None,
+                "error_breakdown": dict(self.error_count_by_type),
+            }

src/rid_reservoir.py

@@ -0,0 +1,351 @@
+"""Collector for RID large-dam daily status (app.rid.go.th/reservoir).
+
+The Royal Irrigation Department reservoir app exposes an unauthenticated
+JSON API: ``POST https://app.rid.go.th/reservoir/api/dams`` with form field
+``date=YYYY-MM-DD`` (empty = today) returns a daily snapshot of every large
+dam in Thailand — storage, inflow and outflow in MCM — with archive depth
+back to at least 2009. GET returns 404 ("Unknown method."); the POST body
+may be empty but must carry a Content-Length.
+
+The reservoir that matters for P.1 flood forecasting is Mae Ngat Somboon
+Chon (DAM_ID 200103), the only large dam upstream of Chiang Mai: during the
+Oct 2024 record flood it reached 113% of usable capacity with inflow spikes
+of ~19 MCM/day. All dams in the payload are stored (same request cost);
+filtering happens at feature-build time.
+
+See docs/DATA_SOURCES.md for the endpoint catalog.
+"""
+
+import datetime
+import logging
+import time
+from typing import Any, Dict, List, Optional
+
+import requests
+
+logger = logging.getLogger(__name__)
+
+RID_DAMS_URL = "https://app.rid.go.th/reservoir/api/dams"
+MAE_NGAT_DAM_ID = "200103"
+
+
+def _to_float(value: Any) -> Optional[float]:
+    """API numerics arrive as strings ('222.01'), ' - ' placeholders, or None."""
+    if value is None:
+        return None
+    if isinstance(value, str):
+        value = value.replace(",", "").strip()
+        if value in ("", "-"):
+            return None
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def parse_dam_records(payload: Dict) -> List[Dict]:
+    """Flatten the regions/dams payload into per-dam daily rows."""
+    records = []
+    for region in payload.get("regions") or []:
+        for dam in region.get("dams") or []:
+            dam_id = dam.get("DAM_ID")
+            try:
+                date = datetime.date.fromisoformat(dam.get("DMD_Date") or "")
+            except ValueError:
+                continue
+            if not dam_id:
+                continue
+            records.append(
+                {
+                    "dam_id": dam_id,
+                    "region": region.get("region_name"),
+                    "name_th": dam.get("DAM_Name"),
+                    "latitude": _to_float(dam.get("DAM_Lat")),
+                    "longitude": _to_float(dam.get("DAM_Lon")),
+                    "capacity_max_mcm": _to_float(dam.get("DAM_QMax")),
+                    "capacity_normal_mcm": _to_float(dam.get("DAM_QStore")),
+                    "date": date,
+                    "storage_mcm": _to_float(dam.get("DMD_QUse")),
+                    "storage_pct": _to_float(dam.get("PERCENT_DMD_QUse")),
+                    "inflow_mcm": _to_float(dam.get("DMD_Inflow")),
+                    "outflow_mcm": _to_float(dam.get("DMD_Outflow")),
+                    "level_msl": _to_float(dam.get("DMD_Q")),
+                }
+            )
+    return records
+
+
+class RidReservoirClient:
+    """HTTP client for the RID reservoir daily-status API."""
+
+    def __init__(
+        self,
+        url: str = RID_DAMS_URL,
+        session: Optional[requests.Session] = None,
+        timeout: int = 60,
+    ):
+        self.url = url
+        self.session = session or requests.Session()
+        self.timeout = timeout
+
+    def fetch_day(self, date: Optional[datetime.date] = None) -> List[Dict]:
+        """Daily rows for every large dam on `date` (None = today)."""
+        data = {"date": date.isoformat()} if date else {"date": ""}
+        response = self.session.post(
+            self.url,
+            data=data,
+            headers={"X-Requested-With": "XMLHttpRequest"},
+            timeout=self.timeout,
+        )
+        response.raise_for_status()
+        return parse_dam_records(response.json())
+
+
+class RidReservoirStore:
+    """SQL persistence for dam metadata + daily measurements.
+
+    Shares the app's relational database; writes rid_dams (metadata) and
+    rid_reservoir_daily keyed (dam_id, date) — the composite natural PK keeps
+    the table TimescaleDB-hypertable compatible.
+    """
+
+    def __init__(self, connection_string: str, db_type: str):
+        self.db_type = db_type.lower()
+        if self.db_type not in ("sqlite", "postgresql", "mysql"):
+            raise ValueError(
+                f"Reservoir collection requires a SQL database, got '{db_type}'"
+            )
+        self.connection_string = connection_string
+        self.engine = None
+
+    def connect(self) -> bool:
+        try:
+            from sqlalchemy import create_engine
+
+            self.engine = create_engine(self.connection_string, pool_pre_ping=True)
+            self._create_tables()
+            return True
+        except Exception as e:
+            logger.error(f"RidReservoirStore failed to connect: {e}")
+            self.engine = None
+            return False
+
+    def _create_tables(self):
+        from sqlalchemy import text
+
+        ddl = [
+            """
+            CREATE TABLE IF NOT EXISTS rid_dams (
+                dam_id VARCHAR(10) PRIMARY KEY,
+                region VARCHAR(40),
+                name_th VARCHAR(255),
+                latitude NUMERIC(10,6),
+                longitude NUMERIC(10,6),
+                capacity_max_mcm NUMERIC(10,2),
+                capacity_normal_mcm NUMERIC(10,2),
+                updated_at TIMESTAMP
+            )
+            """,
+            """
+            CREATE TABLE IF NOT EXISTS rid_reservoir_daily (
+                dam_id VARCHAR(10) NOT NULL,
+                date DATE NOT NULL,
+                storage_mcm NUMERIC(10,2),
+                storage_pct NUMERIC(6,2),
+                inflow_mcm NUMERIC(10,2),
+                outflow_mcm NUMERIC(10,2),
+                level_msl NUMERIC(8,2),
+                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+                PRIMARY KEY (dam_id, date)
+            )
+            """,
+        ]
+        if self.db_type != "mysql":
+            ddl.append(
+                "CREATE INDEX IF NOT EXISTS idx_rid_reservoir_date "
+                "ON rid_reservoir_daily(date)"
+            )
+        with self.engine.begin() as conn:
+            for statement in ddl:
+                conn.execute(text(statement))
+
+    def _upsert(self, table: str, key_cols: List[str], value_cols: List[str]) -> str:
+        cols = key_cols + value_cols
+        col_list = ", ".join(cols)
+        params = ", ".join(f":{c}" for c in cols)
+        if self.db_type == "sqlite":
+            return f"INSERT OR REPLACE INTO {table} ({col_list}) VALUES ({params})"
+        if self.db_type == "postgresql":
+            updates = ", ".join(f"{c} = EXCLUDED.{c}" for c in value_cols)
+            conflict = ", ".join(key_cols)
+            return (
+                f"INSERT INTO {table} ({col_list}) VALUES ({params}) "
+                f"ON CONFLICT ({conflict}) DO UPDATE SET {updates}"
+            )
+        updates = ", ".join(f"{c} = VALUES({c})" for c in value_cols)
+        return (
+            f"INSERT INTO {table} ({col_list}) VALUES ({params}) "
+            f"ON DUPLICATE KEY UPDATE {updates}"
+        )
+
+    def save(self, records: List[Dict]) -> int:
+        """Upsert one day's dam rows (metadata + measurements); idempotent."""
+        if not records:
+            return 0
+        if not self.engine and not self.connect():
+            return 0
+        from sqlalchemy import text
+
+        dam_cols = [
+            "region",
+            "name_th",
+            "latitude",
+            "longitude",
+            "capacity_max_mcm",
+            "capacity_normal_mcm",
+        ]
+        measure_cols = [
+            "storage_mcm",
+            "storage_pct",
+            "inflow_mcm",
+            "outflow_mcm",
+            "level_msl",
+        ]
+        dam_sql = self._upsert("rid_dams", ["dam_id"], dam_cols + ["updated_at"])
+        measure_sql = self._upsert(
+            "rid_reservoir_daily", ["dam_id", "date"], measure_cols
+        )
+        now = datetime.datetime.now()
+        dams = {}
+        measurements = []
+        for record in records:
+            dam_row = {c: record.get(c) for c in dam_cols}
+            dam_row.update({"dam_id": record["dam_id"], "updated_at": now})
+            dams[record["dam_id"]] = dam_row
+            measure_row = {c: record.get(c) for c in measure_cols}
+            measure_row.update(
+                {"dam_id": record["dam_id"], "date": record["date"]}
+            )
+            measurements.append(measure_row)
+        try:
+            with self.engine.begin() as conn:
+                conn.execute(text(dam_sql), list(dams.values()))
+                conn.execute(text(measure_sql), measurements)
+            return len(measurements)
+        except Exception as e:
+            logger.error(f"RidReservoirStore save failed: {e}")
+            return 0
+
+    def present_dates(
+        self, start: datetime.date, end: datetime.date
+    ) -> "set[datetime.date]":
+        """Dates in [start, end] that already have rows, for backfill skipping."""
+        if not self.engine and not self.connect():
+            return set()
+        from sqlalchemy import text
+
+        with self.engine.begin() as conn:
+            values = conn.execute(
+                text(
+                    "SELECT DISTINCT date FROM rid_reservoir_daily "
+                    "WHERE date >= :start AND date <= :end"
+                ),
+                {"start": start, "end": end},
+            ).fetchall()
+        dates = set()
+        for (value,) in values:
+            if isinstance(value, str):  # sqlite returns ISO strings
+                value = datetime.date.fromisoformat(value[:10])
+            if isinstance(value, datetime.datetime):
+                value = value.date()
+            dates.add(value)
+        return dates
+
+
+class RidReservoirCollector:
+    """Fetch + persist the daily dam snapshot (today and yesterday)."""
+
+    def __init__(self, db_config: Dict, client: Optional[RidReservoirClient] = None):
+        self.client = client or RidReservoirClient()
+        self.store = RidReservoirStore(
+            connection_string=db_config["connection_string"],
+            db_type=db_config["type"],
+        )
+
+    def run_cycle(self) -> int:
+        """Collect today's snapshot plus yesterday's (late daily revisions)."""
+        saved = 0
+        today = datetime.date.today()
+        for date in (None, today - datetime.timedelta(days=1)):
+            try:
+                saved += self.store.save(self.client.fetch_day(date))
+            except Exception as e:
+                logger.error(f"RID reservoir collection failed for {date}: {e}")
+        logger.info(f"RID reservoir collection: {saved} dam-day rows saved")
+        return saved
+
+
+def backfill(
+    store: RidReservoirStore,
+    start: datetime.date,
+    end: Optional[datetime.date] = None,
+    client: Optional[RidReservoirClient] = None,
+    throttle_seconds: float = 0.4,
+) -> int:
+    """Fetch every MISSING day in [start, end]; one polite request per day.
+
+    Only dates absent from rid_reservoir_daily are requested, so a rerun
+    repairs holes left by transient failures instead of resuming past them
+    (the hourly collector writes today's rows immediately, which makes any
+    newest-row cursor useless as a resume point). A save that persists
+    nothing counts as a failure too — a broken DB must not burn thousands
+    of requests against the RID API.
+    """
+    client = client or RidReservoirClient()
+    end = end or datetime.date.today()
+    if not store.engine and not store.connect():
+        logger.error("backfill aborted: database connection failed")
+        return 0
+    span = [
+        start + datetime.timedelta(days=i) for i in range((end - start).days + 1)
+    ]
+    present = store.present_dates(start, end)
+    targets = [d for d in span if d not in present]
+    logger.info(
+        f"backfill: {len(targets)} of {len(span)} days missing in [{start}, {end}]"
+    )
+    total = 0
+    failures = 0
+    for i, date in enumerate(targets):
+        try:
+            records = client.fetch_day(date)
+            saved = store.save(records)
+            if records and not saved:
+                raise RuntimeError("database save persisted 0 rows")
+            total += saved
+            failures = 0
+        except Exception as e:
+            failures += 1
+            logger.warning(f"backfill {date} failed ({failures} in a row): {e}")
+            if failures >= 5:
+                logger.error("5 consecutive failures — aborting backfill")
+                break
+        if i % 100 == 0:
+            logger.info(f"backfill progress: {date} ({total} rows)")
+        time.sleep(throttle_seconds)
+    return total
+
+
+def create_collector_from_config() -> Optional[RidReservoirCollector]:
+    """Build a collector from app Config; None when disabled or non-SQL DB."""
+    from .config import Config
+
+    if not Config.ENABLE_RESERVOIR_COLLECTION:
+        return None
+    db_config = Config.get_database_config()
+    if db_config["type"] not in ("sqlite", "postgresql", "mysql"):
+        logger.warning(
+            f"Reservoir collection skipped: DB_TYPE '{db_config['type']}' is not SQL"
+        )
+        return None
+    return RidReservoirCollector(db_config)

src/schemas.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+"""Pydantic request/response schemas for the water monitoring web API."""
+
+from datetime import datetime
+from typing import Any, Dict, Optional
+
+from pydantic import BaseModel, Field
+
+
+class StationResponse(BaseModel):
+    station_id: int
+    station_code: str
+    thai_name: str
+    english_name: str
+    latitude: Optional[float] = None
+    longitude: Optional[float] = None
+    geohash: Optional[str] = None
+    status: str = "active"
+
+
+class StationCreateModel(BaseModel):
+    station_code: str = Field(..., description="Station code (e.g., P.1, P.20)")
+    thai_name: str = Field(..., description="Thai name of the station")
+    english_name: str = Field(..., description="English name of the station")
+    latitude: Optional[float] = Field(
+        None, ge=-90, le=90, description="Latitude coordinate"
+    )
+    longitude: Optional[float] = Field(
+        None, ge=-180, le=180, description="Longitude coordinate"
+    )
+    geohash: Optional[str] = Field(None, description="Geohash for the location")
+    status: str = Field("active", description="Station status")
+
+
+class StationUpdateModel(BaseModel):
+    thai_name: Optional[str] = Field(None, description="Thai name of the station")
+    english_name: Optional[str] = Field(None, description="English name of the station")
+    latitude: Optional[float] = Field(
+        None, ge=-90, le=90, description="Latitude coordinate"
+    )
+    longitude: Optional[float] = Field(
+        None, ge=-180, le=180, description="Longitude coordinate"
+    )
+    geohash: Optional[str] = Field(None, description="Geohash for the location")
+    status: Optional[str] = Field(None, description="Station status")
+
+
+class MeasurementResponse(BaseModel):
+    timestamp: datetime
+    station_code: str
+    station_name_en: str
+    station_name_th: str
+    water_level: float
+    discharge: Optional[float] = None
+    discharge_percent: Optional[float] = None
+    status: str = "active"
+
+
+class HealthResponse(BaseModel):
+    overall_status: str
+    timestamp: str
+    checks: Dict[str, Dict[str, Any]]
+
+
+class MetricsResponse(BaseModel):
+    counters: Dict[str, float]
+    gauges: Dict[str, float]
+    histograms: Dict[str, Dict[str, float]]
+
+
+class ScrapingStatusResponse(BaseModel):
+    is_running: bool
+    last_run: Optional[datetime] = None
+    next_run: Optional[datetime] = None
+    total_runs: int = 0
+    successful_runs: int = 0
+    failed_runs: int = 0

src/static/custom-markers.json

@@ -0,0 +1,7 @@
+[
+  { "name": "newedge", "icon": "🏢", "lat": 18.77208394455051, "lon": 98.98366920482054 },
+  { "name": "Sunflower", "icon": "🏠", "lat": 18.775240304156092, "lon": 98.98373200474349 },
+  { "name": "white house", "icon": "🏠", "lat": 18.77589874066847, "lon": 98.98363500976409 },
+  { "name": "casa mia / b4l", "icon": "🏠", "lat": 18.76974018178173, "lon": 98.9877579642922 },
+  { "name": "baan boe", "icon": "🏠", "lat": 18.77410496738743, "lon": 98.98474626480406 }
+]

src/static/llms.txt

@@ -0,0 +1,34 @@
+# Ping River Live Monitor
+
+> Live water levels, discharge, rainfall and machine-learning flood forecasts for
+> the Ping River basin in Northern Thailand, centered on Chiang Mai (gauge P.1,
+> Nawarat Bridge). Open-source, hourly data since 2018, operated by B4L
+> (https://buildfor.life).
+
+Flood context: Chiang Mai city flooding begins when P.1 reaches 3.70 m
+(official inundation stage 1); the record peak was 5.30 m on 2024-10-05.
+
+## Data
+
+- Hourly water level (m) and discharge (m³/s) from 16 Royal Irrigation
+  Department (RID) telemetry gauges in the Ping basin, 2018-present.
+- Hourly rainfall and water level for 400+ ThaiWater/HII stations (Ping basin),
+  including historical water levels back to 2019.
+- ML flood-risk forecasts (probability of warning/danger level within 6/12/24 h)
+  per station, plus Chiang Mai inundation-stage probabilities.
+
+## API (no key required)
+
+- https://water.buildfor.life/docs : interactive OpenAPI reference
+- /stations : station metadata (code, names, coordinates)
+- /measurements/latest?limit=N : latest readings
+- /measurements/history/{station_code}?hours=N or ?start=YYYY-MM-DD&end=YYYY-MM-DD : history
+- /forecast : flood-risk forecasts for all stations
+- /api/stats : database coverage statistics
+- /api/hii/rainfall/latest : latest rainfall per ThaiWater/HII station
+- /api/hii/waterlevel/latest : latest water level per ThaiWater/HII station (m MSL)
+
+## Source
+
+- https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor : code, docs,
+  and the full data-source catalog (docs/DATA_SOURCES.md).

src/static/robots.txt

@@ -0,0 +1,6 @@
+User-agent: *
+Allow: /
+Disallow: /metrics
+Disallow: /scrape/
+
+Sitemap: https://water.buildfor.life/sitemap.xml

src/static/sitemap.xml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <url>
+    <loc>https://water.buildfor.life/</loc>
+    <changefreq>hourly</changefreq>
+    <priority>1.0</priority>
+  </url>
+  <url>
+    <loc>https://water.buildfor.life/docs</loc>
+    <changefreq>monthly</changefreq>
+    <priority>0.5</priority>
+  </url>
+</urlset>

src/thaiwater.py

@@ -0,0 +1,61 @@
+"""Client for ThaiWater's public water-level sensor feed."""
+
+from typing import Dict, List, Optional
+
+import requests
+
+
+class ThaiWaterClient:
+    API_URL = "https://twa-api-public.thaiwater.net/v2/waterlevel"
+
+    def __init__(self, session=None, api_key: Optional[str] = None, timeout: int = 30):
+        self.session = session or requests.Session()
+        self.api_key = api_key
+        self.timeout = timeout
+
+    def fetch_ping_sensors(self) -> List[Dict]:
+        if not self.api_key:
+            raise RuntimeError("THAIWATER_API_KEY is not configured")
+
+        response = self.session.get(
+            self.API_URL,
+            headers={"Accept-Language": "en", "x-api-key": self.api_key},
+            timeout=self.timeout,
+        )
+        response.raise_for_status()
+        return self._parse_ping_features(response.json())
+
+    @staticmethod
+    def _parse_ping_features(payload: Dict) -> List[Dict]:
+        sensors = []
+        for collection in payload.get("data", {}).values():
+            for feature in collection.get("features", []):
+                properties = feature.get("properties") or {}
+                basin = properties.get("basin") or {}
+                if basin.get("basin") != "Ping":
+                    continue
+
+                geometry = feature.get("geometry") or {}
+                coordinates = geometry.get("coordinates") or []
+                if len(coordinates) < 2:
+                    continue
+
+                station = properties.get("station") or {}
+                station_code = station.get("stationCode", "")
+                sensors.append(
+                    {
+                        "id": f"thaiwater:{properties.get('id')}",
+                        "station_code": station_code.split("-", 1)[-1],
+                        "station_name": station.get("station"),
+                        "latitude": coordinates[1],
+                        "longitude": coordinates[0],
+                        "timestamp": properties.get("waterlevelDatetime"),
+                        "water_level_msl": properties.get("waterlevelMsl"),
+                        "bank_percent": properties.get("storagePercent"),
+                        "distance_to_bank": properties.get("diffWlBank"),
+                        "river_name": properties.get("riverName"),
+                        "agency": (properties.get("agency") or {}).get("agencyShort"),
+                        "source": "ThaiWater",
+                    }
+                )
+        return sensors

src/validators.py

@@ -3,114 +3,142 @@
 Data validation utilities for water monitoring system
 """
 
-from typing import List, Dict, Any, Optional
-from datetime import datetime
 import logging
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+
 from .exceptions import DataValidationError
-from .models import WaterMeasurement, StationInfo
+from .models import StationInfo, WaterMeasurement
 
 logger = logging.getLogger(__name__)
 
+
 class DataValidator:
     """Validates water measurement data"""
-    
+
     # Reasonable ranges for water measurements
     WATER_LEVEL_MIN = -10.0  # meters
-    WATER_LEVEL_MAX = 50.0   # meters
-    DISCHARGE_MIN = 0.0      # cms
+    WATER_LEVEL_MAX = 50.0  # meters
+    DISCHARGE_MIN = 0.0  # cms
     DISCHARGE_MAX = 10000.0  # cms
     DISCHARGE_PERCENT_MIN = 0.0
-    DISCHARGE_PERCENT_MAX = 200.0  # Allow some overflow
-    
+    # % of channel capacity. Major floods genuinely exceed 200% (Oct 2024 peaked
+    # at 226%); values beyond 500% are treated as data errors.
+    DISCHARGE_PERCENT_MAX = 500.0
+
     @classmethod
     def validate_measurement(cls, measurement: Dict[str, Any]) -> bool:
         """Validate a single measurement"""
         try:
-            # Check required fields
-            required_fields = ['timestamp', 'station_id', 'water_level', 'discharge']
+            # Check required fields (discharge is now optional)
+            required_fields = ["timestamp", "station_id", "water_level"]
             for field in required_fields:
                 if field not in measurement:
                     logger.warning(f"Missing required field: {field}")
                     return False
-            
+
             # Validate timestamp
-            if not isinstance(measurement['timestamp'], datetime):
-                logger.warning(f"Invalid timestamp type: {type(measurement['timestamp'])}")
+            if not isinstance(measurement["timestamp"], datetime):
+                logger.warning(
+                    f"Invalid timestamp type: {type(measurement['timestamp'])}"
+                )
                 return False
-            
-            # Validate water level
-            water_level = float(measurement['water_level'])
+
+            # Validate water level (required)
+            if measurement["water_level"] is None:
+                logger.warning("Water level cannot be None")
+                return False
+            water_level = float(measurement["water_level"])
             if not (cls.WATER_LEVEL_MIN <= water_level <= cls.WATER_LEVEL_MAX):
                 logger.warning(f"Water level out of range: {water_level}")
                 return False
-            
-            # Validate discharge
-            discharge = float(measurement['discharge'])
-            if not (cls.DISCHARGE_MIN <= discharge <= cls.DISCHARGE_MAX):
-                logger.warning(f"Discharge out of range: {discharge}")
-                return False
-            
-            # Validate discharge percent if present
-            if measurement.get('discharge_percent') is not None:
-                discharge_percent = float(measurement['discharge_percent'])
-                if not (cls.DISCHARGE_PERCENT_MIN <= discharge_percent <= cls.DISCHARGE_PERCENT_MAX):
-                    logger.warning(f"Discharge percent out of range: {discharge_percent}")
+
+            # Validate discharge (optional - can be None)
+            discharge_value = measurement.get("discharge")
+            if discharge_value is not None:
+                discharge = float(discharge_value)
+                if not (cls.DISCHARGE_MIN <= discharge <= cls.DISCHARGE_MAX):
+                    logger.warning(f"Discharge out of range: {discharge}")
                     return False
-            
+
+            # Validate discharge percent if present. This is an auxiliary field:
+            # an implausible value must not cost us the water level and discharge
+            # (rejecting rows here silently deleted the Oct 2024 flood peaks), so
+            # out-of-range percents are nulled and the measurement is kept.
+            if measurement.get("discharge_percent") is not None:
+                discharge_percent = float(measurement["discharge_percent"])
+                if not (
+                    cls.DISCHARGE_PERCENT_MIN
+                    <= discharge_percent
+                    <= cls.DISCHARGE_PERCENT_MAX
+                ):
+                    logger.warning(
+                        f"Discharge percent out of range ({discharge_percent}); "
+                        "keeping measurement with discharge_percent=None"
+                    )
+                    measurement["discharge_percent"] = None
+
             # Validate station ID
-            station_id = measurement['station_id']
+            station_id = measurement["station_id"]
             if not isinstance(station_id, int) or station_id < 1 or station_id > 16:
                 logger.warning(f"Invalid station ID: {station_id}")
                 return False
-            
+
             return True
-            
+
         except (ValueError, TypeError) as e:
             logger.warning(f"Data validation error: {e}")
             return False
-    
+
     @classmethod
-    def validate_measurements(cls, measurements: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    def validate_measurements(
+        cls, measurements: List[Dict[str, Any]]
+    ) -> List[Dict[str, Any]]:
         """Validate and filter a list of measurements"""
         valid_measurements = []
         invalid_count = 0
-        
+
         for measurement in measurements:
             if cls.validate_measurement(measurement):
                 valid_measurements.append(measurement)
             else:
                 invalid_count += 1
-        
+
         if invalid_count > 0:
             logger.warning(f"Filtered out {invalid_count} invalid measurements")
-        
+
         return valid_measurements
-    
+
     @classmethod
     def validate_station_info(cls, station_info: Dict[str, Any]) -> bool:
         """Validate station information"""
         try:
-            required_fields = ['station_id', 'station_code', 'thai_name', 'english_name']
+            required_fields = [
+                "station_id",
+                "station_code",
+                "thai_name",
+                "english_name",
+            ]
             for field in required_fields:
                 if field not in station_info or not station_info[field]:
                     logger.warning(f"Missing or empty station field: {field}")
                     return False
-            
+
             # Validate coordinates if present
-            if station_info.get('latitude') is not None:
-                lat = float(station_info['latitude'])
+            if station_info.get("latitude") is not None:
+                lat = float(station_info["latitude"])
                 if not (-90 <= lat <= 90):
                     logger.warning(f"Invalid latitude: {lat}")
                     return False
-            
-            if station_info.get('longitude') is not None:
-                lon = float(station_info['longitude'])
+
+            if station_info.get("longitude") is not None:
+                lon = float(station_info["longitude"])
                 if not (-180 <= lon <= 180):
                     logger.warning(f"Invalid longitude: {lon}")
                     return False
-            
+
             return True
-            
+
         except (ValueError, TypeError) as e:
             logger.warning(f"Station validation error: {e}")
-            return False
+            return False

tests/conftest.py

@@ -0,0 +1,12 @@
+"""Shared pytest configuration.
+
+Ensures the repository root is on sys.path so tests can import the ``src``
+package regardless of the working directory pytest is invoked from.
+"""
+
+import os
+import sys
+
+REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+if REPO_ROOT not in sys.path:
+    sys.path.insert(0, REPO_ROOT)

tests/test_alerting.py

@@ -0,0 +1,383 @@
+#!/usr/bin/env python3
+"""
+Comprehensive tests for the alerting system
+Tests both zone-based and rate-of-change alerts
+"""
+
+import sys
+import os
+import datetime
+import sqlite3
+import time
+import gc
+
+# Add src directory to path
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from src.alerting import WaterLevelAlertSystem, AlertLevel
+from src.database_adapters import create_database_adapter
+
+
+def setup_test_database(test_name='default'):
+    """Create a test database with sample data"""
+    db_path = f'test_alerts_{test_name}.db'
+
+    # Remove existing test database
+    if os.path.exists(db_path):
+        try:
+            os.remove(db_path)
+        except PermissionError:
+            # If locked, use a different name with timestamp
+            import random
+            db_path = f'test_alerts_{test_name}_{random.randint(1000, 9999)}.db'
+
+    # Create new database
+    conn = sqlite3.connect(db_path)
+    cursor = conn.cursor()
+
+    # Create stations table
+    cursor.execute("""
+    CREATE TABLE stations (
+        id INTEGER PRIMARY KEY,
+        station_code TEXT NOT NULL UNIQUE,
+        english_name TEXT,
+        thai_name TEXT,
+        latitude REAL,
+        longitude REAL,
+        basin TEXT,
+        province TEXT,
+        status TEXT DEFAULT 'active',
+        created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+    )
+    """)
+
+    # Create water_measurements table
+    cursor.execute("""
+    CREATE TABLE water_measurements (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        timestamp DATETIME NOT NULL,
+        station_id INTEGER NOT NULL,
+        water_level REAL NOT NULL,
+        discharge REAL,
+        discharge_percent REAL,
+        status TEXT DEFAULT 'active',
+        created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+        FOREIGN KEY (station_id) REFERENCES stations (id)
+    )
+    """)
+
+    # Insert P.1 station (id=8 to match existing data)
+    cursor.execute("""
+    INSERT INTO stations (id, station_code, english_name, thai_name, basin, province)
+    VALUES (8, 'P.1', 'Nawarat Bridge', 'สะพานนวรัฐ', 'Ping', 'Chiang Mai')
+    """)
+
+    conn.commit()
+    conn.close()
+
+    return db_path
+
+
+def test_zone_level_alerts():
+    """Test that zone-based alerts trigger correctly"""
+    print("="*70)
+    print("TEST 1: Zone-Based Water Level Alerts")
+    print("="*70)
+
+    db_path = setup_test_database('zone_tests')
+
+    # Test cases for P.1 zone thresholds
+    test_cases = [
+        (2.5, None, "Below all zones"),
+        (3.7, AlertLevel.INFO, "Zone 1"),
+        (3.9, AlertLevel.INFO, "Zone 2"),
+        (4.0, AlertLevel.WARNING, "Zone 3"),
+        (4.2, AlertLevel.WARNING, "Zone 5"),
+        (4.3, AlertLevel.CRITICAL, "Zone 6"),
+        (4.6, AlertLevel.CRITICAL, "Zone 7"),
+        (4.8, AlertLevel.EMERGENCY, "Zone 8/NewEdge"),
+        (5.0, AlertLevel.EMERGENCY, "Above all zones"),
+    ]
+
+    print("\nTesting P.1 (Nawarat Bridge) zone thresholds:")
+    print("-" * 70)
+
+    passed = 0
+    failed = 0
+
+    for water_level, expected_level, zone_description in test_cases:
+        # Insert test data
+        conn = sqlite3.connect(db_path)
+        cursor = conn.cursor()
+        cursor.execute("DELETE FROM water_measurements")
+
+        current_time = datetime.datetime.now()
+        cursor.execute("""
+        INSERT INTO water_measurements (timestamp, station_id, water_level, discharge)
+        VALUES (?, 8, ?, 350.0)
+        """, (current_time, water_level))
+
+        conn.commit()
+        conn.close()
+
+        # Check alerts
+        alerting = WaterLevelAlertSystem()
+        alerting.db_adapter = create_database_adapter('sqlite', connection_string=f'sqlite:///{db_path}')
+        alerting.db_adapter.connect()
+
+        alerts = alerting.check_water_levels()
+
+        # Verify result
+        if expected_level is None:
+            # Should not trigger any alert
+            if len(alerts) == 0:
+                print(f"[PASS] {water_level:.1f}m: {zone_description} - No alert")
+                passed += 1
+            else:
+                print(f"[FAIL] {water_level:.1f}m: {zone_description} - Unexpected alert")
+                failed += 1
+        else:
+            # Should trigger alert with specific level
+            if len(alerts) > 0 and alerts[0].level == expected_level:
+                print(f"[PASS] {water_level:.1f}m: {zone_description} - {expected_level.value.upper()} alert")
+                passed += 1
+            elif len(alerts) == 0:
+                print(f"[FAIL] {water_level:.1f}m: {zone_description} - No alert triggered")
+                failed += 1
+            else:
+                print(f"[FAIL] {water_level:.1f}m: {zone_description} - Wrong alert level: {alerts[0].level.value}")
+                failed += 1
+
+    print("-" * 70)
+    print(f"Zone Alert Tests: {passed} passed, {failed} failed")
+
+    # Cleanup - force garbage collection and wait briefly before removing file
+    gc.collect()
+    time.sleep(0.5)
+    try:
+        os.remove(db_path)
+    except PermissionError:
+        print(f"Warning: Could not remove test database {db_path}")
+
+    return failed == 0
+
+
+def test_rate_of_change_alerts():
+    """Test that rate-of-change alerts trigger correctly"""
+    print("\n" + "="*70)
+    print("TEST 2: Rate-of-Change Water Level Alerts")
+    print("="*70)
+
+    db_path = setup_test_database('rate_tests')
+
+    # Test cases: (initial_level, final_level, hours_elapsed, expected_alert_level, description)
+    test_cases = [
+        (3.0, 3.1, 3.0, None, "Slow rise (0.03m/h)"),
+        (3.0, 3.5, 3.0, AlertLevel.WARNING, "Moderate rise (0.17m/h)"),
+        (3.0, 3.8, 3.0, AlertLevel.CRITICAL, "Rapid rise (0.27m/h)"),
+        (3.0, 4.2, 3.0, AlertLevel.EMERGENCY, "Very rapid rise (0.40m/h)"),
+        (4.0, 3.5, 3.0, None, "Falling water (negative rate)"),
+    ]
+
+    print("\nTesting P.1 rate-of-change thresholds:")
+    print("  Warning: 0.15 m/h (15 cm/h)")
+    print("  Critical: 0.25 m/h (25 cm/h)")
+    print("  Emergency: 0.40 m/h (40 cm/h)")
+    print("-" * 70)
+
+    passed = 0
+    failed = 0
+
+    for initial_level, final_level, hours, expected_level, description in test_cases:
+        # Insert test data simulating water level change over time
+        conn = sqlite3.connect(db_path)
+        cursor = conn.cursor()
+        cursor.execute("DELETE FROM water_measurements")
+
+        current_time = datetime.datetime.now()
+        start_time = current_time - datetime.timedelta(hours=hours)
+
+        # Insert initial measurement
+        cursor.execute("""
+        INSERT INTO water_measurements (timestamp, station_id, water_level, discharge)
+        VALUES (?, 8, ?, 350.0)
+        """, (start_time, initial_level))
+
+        # Insert final measurement
+        cursor.execute("""
+        INSERT INTO water_measurements (timestamp, station_id, water_level, discharge)
+        VALUES (?, 8, ?, 380.0)
+        """, (current_time, final_level))
+
+        conn.commit()
+        conn.close()
+
+        # Check rate-of-change alerts
+        alerting = WaterLevelAlertSystem()
+        alerting.db_adapter = create_database_adapter('sqlite', connection_string=f'sqlite:///{db_path}')
+        alerting.db_adapter.connect()
+
+        rate_alerts = alerting.check_rate_of_change(lookback_hours=int(hours) + 1)
+
+        # Calculate actual rate for display
+        level_change = final_level - initial_level
+        rate = level_change / hours if hours > 0 else 0
+
+        # Verify result
+        if expected_level is None:
+            # Should not trigger any alert
+            if len(rate_alerts) == 0:
+                print(f"[PASS] {rate:+.2f}m/h: {description} - No alert")
+                passed += 1
+            else:
+                print(f"[FAIL] {rate:+.2f}m/h: {description} - Unexpected alert")
+                print(f"   Alert: {rate_alerts[0].alert_type} - {rate_alerts[0].level.value}")
+                failed += 1
+        else:
+            # Should trigger alert with specific level
+            if len(rate_alerts) > 0 and rate_alerts[0].level == expected_level:
+                print(f"[PASS] {rate:+.2f}m/h: {description} - {expected_level.value.upper()} alert")
+                print(f"   Message: {rate_alerts[0].message}")
+                passed += 1
+            elif len(rate_alerts) == 0:
+                print(f"[FAIL] {rate:+.2f}m/h: {description} - No alert triggered")
+                failed += 1
+            else:
+                print(f"[FAIL] {rate:+.2f}m/h: {description} - Wrong alert level: {rate_alerts[0].level.value}")
+                failed += 1
+
+    print("-" * 70)
+    print(f"Rate-of-Change Tests: {passed} passed, {failed} failed")
+
+    # Cleanup - force garbage collection and wait briefly before removing file
+    gc.collect()
+    time.sleep(0.5)
+    try:
+        os.remove(db_path)
+    except PermissionError:
+        print(f"Warning: Could not remove test database {db_path}")
+
+    return failed == 0
+
+
+def test_combined_alerts():
+    """Test scenario where both zone and rate-of-change alerts trigger"""
+    print("\n" + "="*70)
+    print("TEST 3: Combined Zone + Rate-of-Change Alerts")
+    print("="*70)
+
+    db_path = setup_test_database('combined_tests')
+
+    print("\nScenario: Water rising rapidly from 3.5m to 4.5m over 3 hours")
+    print("  Expected: Both Zone 7 alert AND Critical rate-of-change alert")
+    print("-" * 70)
+
+    # Insert test data
+    conn = sqlite3.connect(db_path)
+    cursor = conn.cursor()
+
+    current_time = datetime.datetime.now()
+    start_time = current_time - datetime.timedelta(hours=3)
+
+    # Water rising from 3.5m to 4.5m over 3 hours (0.33 m/h - Critical rate)
+    cursor.execute("""
+    INSERT INTO water_measurements (timestamp, station_id, water_level, discharge)
+    VALUES (?, 8, 3.5, 350.0)
+    """, (start_time,))
+
+    cursor.execute("""
+    INSERT INTO water_measurements (timestamp, station_id, water_level, discharge)
+    VALUES (?, 8, 4.5, 450.0)
+    """, (current_time,))
+
+    conn.commit()
+    conn.close()
+
+    # Check both types of alerts
+    alerting = WaterLevelAlertSystem()
+    alerting.db_adapter = create_database_adapter('sqlite', connection_string=f'sqlite:///{db_path}')
+    alerting.db_adapter.connect()
+
+    zone_alerts = alerting.check_water_levels()
+    rate_alerts = alerting.check_rate_of_change(lookback_hours=4)
+
+    all_alerts = zone_alerts + rate_alerts
+
+    print(f"\nTotal alerts triggered: {len(all_alerts)}")
+
+    zone_alert_found = False
+    rate_alert_found = False
+
+    for alert in all_alerts:
+        print(f"\n  Alert Type: {alert.alert_type}")
+        print(f"     Severity: {alert.level.value.upper()}")
+        print(f"     Water Level: {alert.water_level:.2f}m")
+        if alert.message:
+            print(f"     Details: {alert.message}")
+
+        if "Zone" in alert.alert_type:
+            zone_alert_found = True
+        if "Rise" in alert.alert_type or "rate" in alert.alert_type.lower():
+            rate_alert_found = True
+
+    print("-" * 70)
+
+    if zone_alert_found and rate_alert_found:
+        print("[PASS] Combined Alert Test - Both alert types triggered")
+        success = True
+    else:
+        print("[FAIL] Combined Alert Test")
+        if not zone_alert_found:
+            print("   Missing: Zone-based alert")
+        if not rate_alert_found:
+            print("   Missing: Rate-of-change alert")
+        success = False
+
+    # Cleanup - force garbage collection and wait briefly before removing file
+    gc.collect()
+    time.sleep(0.5)
+    try:
+        os.remove(db_path)
+    except PermissionError:
+        print(f"Warning: Could not remove test database {db_path}")
+
+    return success
+
+
+def main():
+    """Run all alert tests"""
+    print("\n" + "="*70)
+    print("WATER LEVEL ALERTING SYSTEM - COMPREHENSIVE TESTS")
+    print("="*70)
+
+    results = []
+
+    # Run tests
+    results.append(("Zone-Based Alerts", test_zone_level_alerts()))
+    results.append(("Rate-of-Change Alerts", test_rate_of_change_alerts()))
+    results.append(("Combined Alerts", test_combined_alerts()))
+
+    # Summary
+    print("\n" + "="*70)
+    print("TEST SUMMARY")
+    print("="*70)
+
+    all_passed = True
+    for test_name, passed in results:
+        status = "PASS" if passed else "FAIL"
+        print(f"{test_name}: [{status}]")
+        if not passed:
+            all_passed = False
+
+    print("="*70)
+
+    if all_passed:
+        print("\nAll tests PASSED!")
+        return 0
+    else:
+        print("\nSome tests FAILED!")
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/test_dashboard.py

@@ -0,0 +1,89 @@
+from pathlib import Path
+
+
+DASHBOARD_PATH = Path(__file__).parents[1] / "src" / "static" / "dashboard.html"
+
+
+def test_dashboard_contains_live_map_and_flow_visualization():
+    html = DASHBOARD_PATH.read_text(encoding="utf-8")
+
+    assert "id=\"station-map\"" in html
+    assert "id=\"river-flow\"" in html
+    assert "fetch('/stations')" in html or 'fetch("/stations")' in html
+    assert "fetch('/measurements/latest" in html or 'fetch(\"/measurements/latest' in html
+    assert "leaflet" in html.lower()
+
+
+def test_dashboard_explains_flow_legend_and_refresh():
+    html = DASHBOARD_PATH.read_text(encoding="utf-8")
+
+    assert "Flow status" in html
+    assert "Last updated" in html
+    assert "Refresh" in html
+
+
+def test_dashboard_uses_mapped_river_network_instead_of_station_connections():
+    html = DASHBOARD_PATH.read_text(encoding="utf-8")
+    river_network = DASHBOARD_PATH.with_name("ping-river-network.geojson")
+
+    assert river_network.exists()
+    assert "fetch('/static/ping-river-network.geojson')" in html
+    assert "mainBasin.map" not in html
+
+
+def test_dashboard_loads_additional_thaiwater_sensors():
+    html = DASHBOARD_PATH.read_text(encoding="utf-8")
+
+    assert "fetch('/sensors/thaiwater')" in html
+    assert "Additional basin stations" in html
+    assert 'id="station-search"' in html
+    assert "applyStationSearch" in html
+
+
+def test_dashboard_has_seo_and_indexing_files():
+    html = DASHBOARD_PATH.read_text(encoding="utf-8")
+    static_dir = DASHBOARD_PATH.parent
+
+    assert '<meta name="description"' in html
+    assert '<link rel="canonical" href="https://water.buildfor.life/">' in html
+    assert 'property="og:title"' in html
+    assert 'rel="icon"' in html
+
+    robots = (static_dir / "robots.txt").read_text(encoding="utf-8")
+    assert "Sitemap: https://water.buildfor.life/sitemap.xml" in robots
+    assert "<loc>https://water.buildfor.life/</loc>" in (static_dir / "sitemap.xml").read_text(encoding="utf-8")
+    assert "Ping River Live Monitor" in (static_dir / "llms.txt").read_text(encoding="utf-8")
+
+    from src import web_api
+
+    routes = {route.path for route in web_api.app.routes}
+    assert {"/robots.txt", "/llms.txt", "/sitemap.xml"} <= routes
+
+
+def test_dashboard_shows_hii_rainfall_layer():
+    html = DASHBOARD_PATH.read_text(encoding="utf-8")
+
+    assert "fetch('/api/hii/rainfall/latest')" in html
+    assert "fetch('/api/hii/waterlevel/latest')" in html
+    assert "renderRainLayer" in html
+    assert "rain-toggle" in html
+    assert "Rainfall · 24 h" in html
+    # TMD rain classes on the legend
+    assert "Heavy 35–90" in html
+    assert "Extreme &gt; 150" in html
+
+
+def test_dashboard_loads_station_history_chart():
+    html = DASHBOARD_PATH.read_text(encoding="utf-8")
+
+    assert "Station history" in html
+    assert "/api/forecast/history/" in html
+    assert "Model 24 h peak (as issued)" in html
+    assert "PostgreSQL" not in html
+    assert "/measurements/history/" in html
+    assert "history-chart" in html
+    # Date-range picker alongside the quick-range dropdown
+    assert 'id="history-start"' in html
+    assert 'id="history-end"' in html
+    assert "Last 7 days" in html
+    assert "Last 90 days" in html

tests/test_flood_forecast.py

@@ -0,0 +1,246 @@
+"""Tests for the flood forecast ML package. Synthetic data only -- no DB/network."""
+
+import datetime
+from typing import Dict, List, Optional
+
+import joblib
+import numpy as np
+import pandas as pd
+import pytest
+
+from src.ml import features, predict, train
+
+
+def make_synth(
+    n_hours: int,
+    stations: List[str],
+    seed: int = 0,
+    start: str = "2020-01-01",
+    pulses: Optional[Dict[str, List[tuple]]] = None,
+    missing_patches: Optional[Dict[str, List[tuple]]] = None,
+) -> pd.DataFrame:
+    """Generate a synthetic long measurement frame with smooth levels, flood pulses,
+    and optional missing patches, for `stations` over `n_hours` hourly steps.
+
+    pulses: {station: [(start_hour, width_hours, peak_add), ...]}
+    missing_patches: {station: [(start_hour, length_hours), ...]}
+    """
+    rng = np.random.default_rng(seed)
+    idx = pd.date_range(start, periods=n_hours, freq="h")
+    rows = []
+    for station in stations:
+        base = 1.5 + 0.1 * np.sin(np.linspace(0, 6 * np.pi, n_hours))
+        noise = rng.normal(0, 0.02, n_hours)
+        level = base + noise
+        for pulse_start, width, peak_add in (pulses or {}).get(station, []):
+            t = np.arange(n_hours)
+            bump = peak_add * np.exp(-0.5 * ((t - (pulse_start + width / 2)) / (width / 4)) ** 2)
+            level = level + bump
+        discharge = 20.0 * level + rng.normal(0, 1.0, n_hours)
+
+        missing = np.zeros(n_hours, dtype=bool)
+        for patch_start, length in (missing_patches or {}).get(station, []):
+            missing[patch_start : patch_start + length] = True
+
+        for i in range(n_hours):
+            if missing[i]:
+                continue
+            rows.append(
+                {
+                    "timestamp": idx[i],
+                    "station_code": station,
+                    "water_level": round(float(level[i]), 3),
+                    "discharge": round(float(discharge[i]), 2),
+                }
+            )
+    return pd.DataFrame(rows)
+
+
+def test_no_future_leakage():
+    stations = ["P.1", "P.20"]
+    df_a = make_synth(200, stations, seed=1, pulses={"P.1": [(150, 10, 3.0)]})
+    grid_a = features.make_hourly_grid(df_a)
+    feat_a = features.build_features(grid_a, "P.1")
+
+    t0 = grid_a.observed.index[120]
+
+    df_b = df_a.copy()
+    future_mask = df_b["timestamp"] > t0
+    df_b.loc[future_mask, "water_level"] = df_b.loc[future_mask, "water_level"] + 50.0
+    df_b.loc[future_mask, "discharge"] = df_b.loc[future_mask, "discharge"] + 500.0
+    grid_b = features.make_hourly_grid(df_b)
+    feat_b = features.build_features(grid_b, "P.1")
+
+    past_a = feat_a.loc[feat_a.index <= t0]
+    past_b = feat_b.loc[feat_b.index <= t0]
+    pd.testing.assert_frame_equal(past_a, past_b)
+
+
+def test_label_alignment():
+    idx = pd.date_range("2020-01-01", periods=12, freq="h")
+    warn_thr, _danger_thr = features.get_thresholds("P.1")
+    peak = warn_thr + 0.3
+    levels = [1.0, 1.0, 1.0, 1.0, 1.0, peak, peak, 1.0, 1.0, 1.0, 1.0, 1.0]
+    df = pd.DataFrame(
+        {
+            "timestamp": idx,
+            "station_code": "P.1",
+            "water_level": levels,
+            "discharge": [20.0 * lvl for lvl in levels],
+        }
+    )
+    grid = features.make_hourly_grid(df)
+    labels = features.build_labels(grid, "P.1", horizons=(6,))
+
+    # Level crosses the warning threshold at t=5. A 6h forward window (t, t+6]
+    # first includes t=5 for t=0 .. t=4 (inclusive), so exceed_warn_6 should be
+    # 1 for t=0..4 and not (necessarily) for later rows in this hand-built series.
+    for t in range(5):
+        assert labels["exceed_warn_6"].iloc[t] == 1.0, f"t={t} expected warn exceedance"
+
+    # max_level_6 at t=0 covers hours 1..6 -> includes the peak.
+    assert labels["max_level_6"].iloc[0] == pytest.approx(peak)
+
+
+def test_label_coverage_gate():
+    n = 40
+    idx = pd.date_range("2020-01-01", periods=n, freq="h")
+    levels = [1.0] * n
+    df = pd.DataFrame(
+        {"timestamp": idx, "station_code": "P.1", "water_level": levels, "discharge": [20.0] * n}
+    )
+    # Drop 70% of a future window (hours 21..26) for the row at t=20, no exceedance in it.
+    df_missing = df[~df["timestamp"].isin(idx[21:26])].copy()
+    grid = features.make_hourly_grid(df_missing)
+    labels = features.build_labels(grid, "P.1", horizons=(6,))
+    t20 = idx[20]
+    assert pd.isna(labels.loc[t20, "exceed_warn_6"])
+
+    # Same sparse window, but WITH an observed exceedance inside it -> must be 1, not NaN.
+    df_with_peak = df_missing.copy()
+    peak_row = pd.DataFrame(
+        [{"timestamp": idx[22], "station_code": "P.1", "water_level": 5.0, "discharge": 100.0}]
+    )
+    df_with_peak = pd.concat([df_with_peak, peak_row], ignore_index=True)
+    grid2 = features.make_hourly_grid(df_with_peak)
+    labels2 = features.build_labels(grid2, "P.1", horizons=(6,))
+    assert labels2.loc[t20, "exceed_warn_6"] == 1.0
+
+
+def test_ffill_and_staleness():
+    n = 20
+    idx = pd.date_range("2020-01-01", periods=n, freq="h")
+    df = pd.DataFrame(
+        {
+            "timestamp": idx,
+            "station_code": "P.1",
+            "water_level": [1.0 + 0.01 * i for i in range(n)],
+            "discharge": [20.0] * n,
+        }
+    )
+    # Small gap: drop hours 5,6 (2h gap).
+    df_small_gap = df[~df["timestamp"].isin(idx[5:7])].copy()
+    grid = features.make_hourly_grid(df_small_gap)
+    feat = features.build_features(grid, "P.1")
+    assert feat.loc[idx[5], "obs_age_h"] == pytest.approx(1.0)
+    assert feat.loc[idx[6], "obs_age_h"] == pytest.approx(2.0)
+
+    # Large gap: drop hours 5..9 (5h gap) -> rows with age>3 dropped (NaN).
+    df_big_gap = df[~df["timestamp"].isin(idx[5:10])].copy()
+    grid2 = features.make_hourly_grid(df_big_gap)
+    feat2 = features.build_features(grid2, "P.1")
+    assert feat2.loc[idx[8], "obs_age_h"] != feat2.loc[idx[8], "obs_age_h"]  # NaN
+    assert feat2.loc[idx[9], "obs_age_h"] != feat2.loc[idx[9], "obs_age_h"]  # NaN
+    assert feat2.loc[idx[7], "obs_age_h"] == pytest.approx(3.0)
+
+
+_FORECAST_KEYS = {
+    "station_code",
+    "horizon_hours",
+    "p_warning",
+    "p_danger",
+    "predicted_max_level",
+    "current_level",
+    "as_of",
+    "model_version",
+    "trained_at",
+    "source",
+    "threshold_warning",
+    "threshold_danger",
+}
+
+
+def _assert_valid_forecast_row(row: dict) -> None:
+    # "stages" is optional: model rows for stations in features.FLOOD_STAGES carry
+    # per-inundation-stage exceedance probabilities (currently P.1 only).
+    assert _FORECAST_KEYS <= set(row.keys())
+    assert set(row.keys()) - _FORECAST_KEYS <= {"stages"}
+    assert 0.0 <= row["p_warning"] <= 1.0
+    assert 0.0 <= row["p_danger"] <= 1.0
+    assert row["p_danger"] <= row["p_warning"]
+    assert row["predicted_max_level"] >= row["current_level"]
+    assert row["source"] in ("model", "heuristic")
+    for stage in row.get("stages", []):
+        assert 0.0 <= stage["p_exceed"] <= 1.0
+        assert stage["level"] > 0
+
+
+def test_train_smoke_and_roundtrip(tmp_path):
+    # Include every station P.1's feature set actually references (its UPSTREAM_LEADS)
+    # so no upstream column is entirely NaN -- HistGradientBoosting's binning step
+    # cannot fit a fully-degenerate column (see train._safe_fit).
+    upstream = [code for code, _lead in features.UPSTREAM_LEADS["P.1"]]
+    data_stations = ["P.1"] + upstream
+    target_stations = ["P.1", "P.20"]
+    n = 700
+    pulses = {station: [(start, 20, 2.0) for start in range(50, n - 50, 110)] for station in data_stations}
+    df = make_synth(n, data_stations, seed=7, pulses=pulses)
+
+    metrics = train.train_all(
+        df, target_stations, models_dir=tmp_path, skip_eval=True, hgb_overrides={"max_iter": 20}, use_rain=False
+    )
+    assert metrics["stations"]["P.1"]["status"] == "trained"
+    assert metrics["stations"]["P.20"]["status"] == "trained"
+    assert (tmp_path / "flood_P.1.joblib").exists()
+    assert (tmp_path / "metrics.json").exists()
+
+    readings_by_station = {
+        code: group[["timestamp", "water_level", "discharge"]].to_dict("records")
+        for code, group in df.groupby("station_code")
+        if code in target_stations
+    }
+    now = df["timestamp"].max()
+    forecasts = predict.get_forecasts(readings_by_station, models_dir=tmp_path, now=now)
+
+    assert len(forecasts) > 0
+    for row in forecasts:
+        _assert_valid_forecast_row(row)
+    assert any(row["source"] == "model" for row in forecasts)
+
+
+def test_heuristic_fallback(tmp_path):
+    df = make_synth(50, ["P.1"], seed=3)
+    readings_by_station = {"P.1": df[["timestamp", "water_level", "discharge"]].to_dict("records")}
+    now = df["timestamp"].max()
+
+    forecasts = predict.get_forecasts(readings_by_station, models_dir=tmp_path, now=now)
+
+    assert len(forecasts) == len(predict.DEFAULT_HORIZONS)
+    for row in forecasts:
+        _assert_valid_forecast_row(row)
+        assert row["source"] == "heuristic"
+        assert row["model_version"] == "heuristic-v1"
+        assert row["trained_at"] is None
+
+
+def test_feature_name_stability(tmp_path):
+    upstream = [code for code, _lead in features.UPSTREAM_LEADS["P.1"]]
+    data_stations = ["P.1"] + upstream
+    df = make_synth(300, data_stations, seed=11, pulses={"P.1": [(100, 20, 2.0)]})
+    train.train_all(df, ["P.1"], models_dir=tmp_path, skip_eval=True, hgb_overrides={"max_iter": 10}, use_rain=False)
+    # Safe: loading the bundle this same test just wrote to tmp_path, not an external file.
+    bundle = joblib.load(tmp_path / "flood_P.1.joblib")
+
+    grid = features.make_hourly_grid(df)
+    fresh_columns = list(features.build_features(grid, "P.1").columns)
+    assert fresh_columns == bundle["feature_names"]

tests/test_forecast_history.py

@@ -0,0 +1,99 @@
+"""Tests for the issued-forecast archive (store + API endpoint)."""
+
+import asyncio
+import datetime
+
+import pytest
+
+from src.forecast_history import ForecastHistoryStore
+
+
+def _rows(as_of="2026-08-12T10:00:00"):
+    return [
+        {
+            "as_of": as_of,
+            "station_code": "P.1",
+            "horizon_hours": h,
+            "predicted_max_level": 2.8 + h / 100,
+            "p_warning": 0.02,
+            "p_danger": 0.001,
+            "current_level": 2.76,
+            "model_version": "hgb-v1+test",
+            "trained_at": "2026-08-12T10:52:00",
+            "source": "model",
+        }
+        for h in (6, 12, 24)
+    ]
+
+
+class TestForecastHistoryStore:
+    @pytest.fixture
+    def store(self, tmp_path):
+        store = ForecastHistoryStore(f"sqlite:///{tmp_path}/fh.db", "sqlite")
+        assert store.connect()
+        return store
+
+    def test_rejects_non_sql(self):
+        with pytest.raises(ValueError):
+            ForecastHistoryStore("http://x", "victoriametrics")
+
+    def test_roundtrip_and_upsert(self, store):
+        assert store.save_rows(_rows()) == 3
+        # Same as_of again -> upsert, still 3 rows
+        assert store.save_rows(_rows()) == 3
+        rows = store.fetch("P.1")
+        assert len(rows) == 3
+        assert [r["horizon_hours"] for r in rows] == [6, 12, 24]
+        assert rows[2]["predicted_max_level"] == pytest.approx(3.04)
+        assert rows[0]["model_version"] == "hgb-v1+test"
+
+    def test_fetch_filters(self, store):
+        store.save_rows(_rows("2026-08-12T10:00:00"))
+        store.save_rows(_rows("2026-08-12T11:00:00"))
+        only_24 = store.fetch("P.1", horizon_hours=24)
+        assert len(only_24) == 2
+        assert all(r["horizon_hours"] == 24 for r in only_24)
+        windowed = store.fetch(
+            "P.1",
+            start=datetime.datetime(2026, 8, 12, 10, 30),
+            end=datetime.datetime(2026, 8, 12, 12, 0),
+        )
+        assert len(windowed) == 3  # only the 11:00 issue
+        assert store.fetch("P.99") == []
+
+    def test_skips_malformed_rows(self, store):
+        rows = _rows() + [{"station_code": None, "as_of": None}]
+        assert store.save_rows(rows) == 3
+
+
+class TestForecastHistoryEndpoint:
+    def test_endpoint_returns_rows(self, tmp_path, monkeypatch):
+        from src import web_api
+
+        store = ForecastHistoryStore(f"sqlite:///{tmp_path}/api-fh.db", "sqlite")
+        assert store.connect()
+        now = datetime.datetime.now().replace(minute=0, second=0, microsecond=0)
+        store.save_rows(_rows(now.isoformat()))
+        monkeypatch.setitem(web_api.app_state, "forecast_store", store)
+
+        rows = asyncio.run(
+            web_api.get_forecast_history(
+                "P.1", hours=48, start=None, end=None, horizon=24
+            )
+        )
+        assert len(rows) == 1
+        assert rows[0]["horizon_hours"] == 24
+        assert rows[0]["station_code"] == "P.1"
+
+    def test_endpoint_without_store(self, monkeypatch):
+        from src import web_api
+
+        monkeypatch.setitem(web_api.app_state, "forecast_store", None)
+        assert (
+            asyncio.run(
+                web_api.get_forecast_history(
+                    "P.1", hours=168, start=None, end=None, horizon=None
+                )
+            )
+            == []
+        )

tests/test_hii_collector.py

@@ -0,0 +1,465 @@
+"""Tests for the HII/ThaiWater api-v3 collector (parsing + persistence)."""
+
+import datetime
+
+import pytest
+
+from src.hii_backfill import chunk_date_range, parse_graph_rows, select_stations
+from src.hii_collector import (
+    HiiStore,
+    parse_rain_records,
+    parse_waterlevel_records,
+    rid_code_from_oldcode,
+)
+
+
+def _rain_payload():
+    return {
+        "result": "OK",
+        "data": [
+            {
+                "id": 306091240,
+                "rain_1h": 0,
+                "rain_24h": "49.6",
+                "rainfall_datetime": "2026-08-11 13:00",
+                "agency": {"agency_shortname": {"en": "HII", "th": "สสน."}},
+                "basin": {"basin_code": 6, "basin_name": {"en": "Ping Basin"}},
+                "station": {
+                    "id": 418,
+                    "sub_basin_id": "0604",
+                    "tele_station_lat": 19.12207,
+                    "tele_station_long": 98.94447,
+                    "tele_station_name": {"en": "Chiang Mai 5", "th": "แม่แตง"},
+                    "tele_station_oldcode": "CHM005",
+                },
+            },
+            # Wrong basin -> filtered out
+            {
+                "id": 2,
+                "rain_24h": 10,
+                "rainfall_datetime": "2026-08-11 13:00",
+                "basin": {"basin_code": 7},
+                "station": {"id": 99},
+            },
+            # No timestamp -> skipped
+            {
+                "id": 3,
+                "rain_24h": 5,
+                "rainfall_datetime": None,
+                "basin": {"basin_code": 6},
+                "station": {"id": 100},
+            },
+        ],
+    }
+
+
+def _waterlevel_payload():
+    return {
+        "waterlevel_data": {
+            "result": "OK",
+            "data": [
+                {
+                    "id": 1286124160,
+                    "waterlevel_datetime": "2026-08-11 13:00",
+                    "waterlevel_m": None,
+                    "waterlevel_msl": "303.27",
+                    "discharge": "335.00",
+                    "flow_rate": None,
+                    "storage_percent": "81.21",
+                    "situation_level": 4,
+                    "diff_wl_bank": "0.93",
+                    "river_name": "แม่น้ำปิง",
+                    "agency": {"agency_shortname": {"en": "RID"}},
+                    "basin": {"basin_code": 6},
+                    "station": {
+                        "id": 3226,
+                        "tele_station_lat": 18.786961,
+                        "tele_station_long": 99.005089,
+                        "tele_station_name": {"th": "สะพานนวรัฐ"},
+                        "tele_station_oldcode": "P.1",
+                        "offset": 300.5,
+                        "ground_level": 299.25,
+                        "min_bank": 304.2,
+                        "critical_level_msl": 304.2,
+                        "critical_level_m": 3.7,
+                        "qmax": 425,
+                        "is_key_station": True,
+                    },
+                },
+                {
+                    "id": 2,
+                    "waterlevel_datetime": "2026-08-11 13:00",
+                    "waterlevel_msl": "200.0",
+                    "basin": {"basin_code": 6},
+                    "station": {"id": 4000, "tele_station_oldcode": "ridhydro_P.67"},
+                },
+                # Wrong basin -> filtered out
+                {
+                    "id": 3,
+                    "waterlevel_datetime": "2026-08-11 13:00",
+                    "basin": {"basin_code": 10},
+                    "station": {"id": 5000},
+                },
+            ],
+        }
+    }
+
+
+class TestRidCodeNormalization:
+    def test_plain_code(self):
+        assert rid_code_from_oldcode("P.1") == "P.1"
+
+    def test_ridhydro_prefix(self):
+        assert rid_code_from_oldcode("ridhydro_P.67") == "P.67"
+
+    def test_letter_suffix(self):
+        assert rid_code_from_oldcode("ridhydro_P.4A") == "P.4A"
+
+    def test_non_rid_codes(self):
+        assert rid_code_from_oldcode("CHM005") is None
+        assert rid_code_from_oldcode("ridtele_TUP.14") is None
+        assert rid_code_from_oldcode(None) is None
+
+
+class TestParseRain:
+    def test_filters_and_parses(self):
+        records = parse_rain_records(_rain_payload())
+        assert len(records) == 1
+        r = records[0]
+        assert r["station_id"] == 418
+        assert r["oldcode"] == "CHM005"
+        assert r["name_en"] == "Chiang Mai 5"
+        assert r["rain_1h"] == 0.0
+        assert r["rain_24h"] == 49.6
+        assert r["timestamp"] == datetime.datetime(2026, 8, 11, 13, 0)
+        assert r["agency"] == "HII"
+
+    def test_empty_payload(self):
+        assert parse_rain_records({}) == []
+
+
+class TestParseWaterlevel:
+    def test_filters_and_parses(self):
+        records = parse_waterlevel_records(_waterlevel_payload())
+        assert len(records) == 2
+        p1 = records[0]
+        assert p1["station_id"] == 3226
+        assert p1["rid_code"] == "P.1"
+        assert p1["wl_msl"] == 303.27
+        assert p1["discharge"] == 335.0
+        assert p1["flow_rate"] is None
+        assert p1["storage_percent"] == 81.21
+        assert p1["situation_level"] == 4
+        assert p1["offset_msl"] == 300.5
+        assert p1["is_key_station"] is True
+        # MSL minus station offset recovers the familiar gauge level
+        assert p1["wl_msl"] - p1["offset_msl"] == pytest.approx(2.77)
+        assert records[1]["rid_code"] == "P.67"
+
+    def test_empty_payload(self):
+        assert parse_waterlevel_records({}) == []
+
+
+class TestHiiStore:
+    @pytest.fixture
+    def store(self, tmp_path):
+        store = HiiStore(f"sqlite:///{tmp_path}/hii_test.db", "sqlite")
+        assert store.connect()
+        return store
+
+    def test_rejects_non_sql_backend(self):
+        with pytest.raises(ValueError):
+            HiiStore("http://localhost:8428", "victoriametrics")
+
+    def test_rain_roundtrip_and_upsert(self, store):
+        records = parse_rain_records(_rain_payload())
+        assert store.save_rain(records) == 1
+        # Same snapshot again -> upsert, still one row
+        assert store.save_rain(records) == 1
+
+        from sqlalchemy import text
+
+        with store.engine.connect() as conn:
+            rows = conn.execute(text("SELECT COUNT(*) FROM hii_rainfall")).scalar()
+            stations = conn.execute(
+                text("SELECT oldcode FROM hii_rain_stations")
+            ).fetchall()
+        assert rows == 1
+        assert stations == [("CHM005",)]
+
+    def test_waterlevel_roundtrip(self, store):
+        records = parse_waterlevel_records(_waterlevel_payload())
+        assert store.save_waterlevel(records) == 2
+
+        from sqlalchemy import text
+
+        with store.engine.connect() as conn:
+            row = conn.execute(
+                text(
+                    "SELECT s.rid_code, m.wl_msl, m.situation_level "
+                    "FROM hii_waterlevel m JOIN hii_wl_stations s ON s.id = m.station_id "
+                    "WHERE s.oldcode = 'P.1'"
+                )
+            ).fetchone()
+        assert row is not None
+        assert row[0] == "P.1"
+        assert float(row[1]) == 303.27
+        assert row[2] == 4
+
+    def test_save_empty(self, store):
+        assert store.save_rain([]) == 0
+
+    def test_history_upsert_preserves_snapshot_columns(self, store):
+        # A live snapshot row exists with extra columns populated
+        store.save_waterlevel(parse_waterlevel_records(_waterlevel_payload()))
+        # Backfill collides on the same (station, timestamp) with new values
+        rows = [
+            {
+                "timestamp": datetime.datetime(2026, 8, 11, 13, 0),
+                "wl_msl": 303.30,
+                "discharge": 340.0,
+            },
+            {
+                "timestamp": datetime.datetime(2019, 8, 1, 1, 0),
+                "wl_msl": 301.71,
+                "discharge": 11.7,
+            },
+        ]
+        assert store.save_waterlevel_history(3226, rows) == 2
+
+        from sqlalchemy import text
+
+        with store.engine.connect() as conn:
+            collided = conn.execute(
+                text(
+                    "SELECT wl_msl, discharge, storage_percent, situation_level "
+                    "FROM hii_waterlevel WHERE station_id = 3226 "
+                    "AND timestamp = '2026-08-11 13:00:00'"
+                )
+            ).fetchone()
+            historical = conn.execute(
+                text(
+                    "SELECT wl_msl FROM hii_waterlevel WHERE station_id = 3226 "
+                    "AND timestamp = '2019-08-01 01:00:00'"
+                )
+            ).fetchone()
+        # wl_msl/discharge updated, snapshot-only columns untouched
+        assert float(collided[0]) == 303.30
+        assert float(collided[1]) == 340.0
+        assert float(collided[2]) == 81.21
+        assert collided[3] == 4
+        assert float(historical[0]) == 301.71
+
+
+class TestParseGraphRows:
+    def test_parses_and_skips_empty(self):
+        payload = {
+            "data": {
+                "graph_data": [
+                    {"datetime": "2024-10-05 12:00", "value": 305.8, "discharge": 656},
+                    {"datetime": "2024-10-05 13:00", "value": None, "discharge": None},
+                    {"datetime": None, "value": 300.0, "discharge": 1},
+                ]
+            }
+        }
+        rows = parse_graph_rows(payload)
+        assert rows == [
+            {
+                "timestamp": datetime.datetime(2024, 10, 5, 12, 0),
+                "wl_msl": 305.8,
+                "discharge": 656.0,
+            }
+        ]
+
+    def test_empty_payload(self):
+        assert parse_graph_rows({}) == []
+
+
+class TestHiiApiEndpoints:
+    """Call the endpoint coroutines directly (the venv's httpx/starlette
+    combination is incompatible with TestClient)."""
+
+    @pytest.fixture
+    def web_api(self, tmp_path, monkeypatch):
+        from src import web_api
+        from src.hii_collector import HiiCollector
+
+        collector = HiiCollector(
+            {"type": "sqlite", "connection_string": f"sqlite:///{tmp_path}/api.db"}
+        )
+        now = datetime.datetime.now().replace(microsecond=0)
+        rain = parse_rain_records(_rain_payload())
+        wl = parse_waterlevel_records(_waterlevel_payload())
+        for record in rain + wl:
+            record["timestamp"] = now
+        assert collector.store.save_rain(rain) == 1
+        assert collector.store.save_waterlevel(wl) == 2
+        monkeypatch.setitem(web_api.app_state, "hii_collector", collector)
+        web_api.HII_CACHE.clear()  # response cache would leak across tests
+        web_api._REFRESH_IN_FLIGHT.clear()
+        return web_api
+
+    @staticmethod
+    def _get(web_api_module, endpoint, **kwargs):
+        import asyncio
+
+        from fastapi import Response
+
+        response = Response()
+        rows = asyncio.run(
+            getattr(web_api_module, endpoint)(response=response, **kwargs)
+        )
+        return rows, response
+
+    def test_rainfall_latest(self, web_api):
+        rows, _ = self._get(web_api, "get_hii_rainfall_latest", hours=26)
+        assert len(rows) == 1
+        assert rows[0]["oldcode"] == "CHM005"
+        assert rows[0]["rain_24h"] == 49.6
+        assert rows[0]["latitude"] == pytest.approx(19.12207)
+
+    def test_waterlevel_latest(self, web_api):
+        rows, _ = self._get(web_api, "get_hii_waterlevel_latest", hours=26)
+        assert len(rows) == 2
+        p1 = next(r for r in rows if r["oldcode"] == "P.1")
+        assert p1["rid_code"] == "P.1"
+        assert p1["wl_msl"] == 303.27
+        assert p1["offset_msl"] == 300.5
+        assert p1["situation_level"] == 4
+
+    def test_empty_when_collector_disabled(self, monkeypatch):
+        from src import web_api
+
+        monkeypatch.setitem(web_api.app_state, "hii_collector", None)
+        web_api.HII_CACHE.clear()
+        web_api._REFRESH_IN_FLIGHT.clear()
+        assert self._get(web_api, "get_hii_rainfall_latest", hours=26)[0] == []
+        assert self._get(web_api, "get_hii_waterlevel_latest", hours=26)[0] == []
+
+    def test_latest_responses_are_cached(self, web_api, monkeypatch):
+        calls = {"n": 0}
+        real = web_api._hii_rows
+
+        def counting(sql, params):
+            calls["n"] += 1
+            return real(sql, params)
+
+        monkeypatch.setattr(web_api, "_hii_rows", counting)
+        first, _ = self._get(web_api, "get_hii_rainfall_latest", hours=26)
+        second, _ = self._get(web_api, "get_hii_rainfall_latest", hours=26)
+        assert first == second and len(first) == 1
+        assert calls["n"] == 1  # second call served from the TTL cache
+        # different hours -> different cache key -> fresh query
+        self._get(web_api, "get_hii_rainfall_latest", hours=48)
+        assert calls["n"] == 2
+
+    def test_stale_served_on_recompute_failure(self, web_api, monkeypatch):
+        # Prime the cache, expire it, break the DB: the stale copy is served
+        # and flagged via the X-Data-Stale header.
+        good, response = self._get(web_api, "get_hii_rainfall_latest", hours=26)
+        assert good and "x-data-stale" not in response.headers
+
+        from src.config import Config
+
+        monkeypatch.setattr(Config, "HII_CACHE_TTL_SECONDS", 0)
+
+        def broken(sql, params):
+            raise RuntimeError("db unreachable")
+
+        monkeypatch.setattr(web_api, "_hii_rows", broken)
+        rows, response = self._get(web_api, "get_hii_rainfall_latest", hours=26)
+        assert rows == good
+        assert response.headers["X-Data-Stale"] == "true"
+
+    def test_empty_results_are_not_cached(self, tmp_path, monkeypatch):
+        from src import web_api
+        from src.hii_collector import HiiCollector
+
+        collector = HiiCollector(
+            {"type": "sqlite", "connection_string": f"sqlite:///{tmp_path}/empty.db"}
+        )
+        monkeypatch.setitem(web_api.app_state, "hii_collector", collector)
+        web_api.HII_CACHE.clear()
+        web_api._REFRESH_IN_FLIGHT.clear()
+        assert self._get(web_api, "get_hii_rainfall_latest", hours=26)[0] == []
+        assert web_api.HII_CACHE == {}  # empty response left uncached
+
+    def test_measurements_latest_cached_and_stale(self, monkeypatch):
+        from types import SimpleNamespace
+
+        from src import web_api
+        from src.config import Config
+
+        calls = {"n": 0}
+
+        def get_latest_data(limit=100):
+            calls["n"] += 1
+            return [
+                {
+                    "timestamp": "2026-08-12 10:00:00",
+                    "station_code": "P.1",
+                    "station_name_en": "Nawarat Bridge",
+                    "station_name_th": "สะพานนวรัฐ",
+                    "water_level": 2.76,
+                    "discharge": 331.0,
+                    "discharge_percent": 77.9,
+                }
+            ]
+
+        scraper = SimpleNamespace(db_adapter=True, get_latest_data=get_latest_data)
+        monkeypatch.setitem(web_api.app_state, "scraper", scraper)
+        web_api.LATEST_CACHE.clear()
+        web_api._REFRESH_IN_FLIGHT.clear()
+
+        rows, response = self._get(web_api, "get_latest_measurements", limit=500)
+        rows2, _ = self._get(web_api, "get_latest_measurements", limit=500)
+        assert calls["n"] == 1  # second call cached
+        assert rows2[0].station_code == "P.1"
+
+        # DB failure after expiry -> stale copy + header
+        monkeypatch.setattr(Config, "LATEST_CACHE_TTL_SECONDS", 0)
+
+        def broken(limit=100):
+            raise RuntimeError("db unreachable")
+
+        scraper.get_latest_data = broken
+        rows3, response = self._get(web_api, "get_latest_measurements", limit=500)
+        assert rows3[0].water_level == 2.76
+        assert response.headers["X-Data-Stale"] == "true"
+
+
+class TestBackfillHelpers:
+    def test_chunk_date_range(self):
+        chunks = chunk_date_range(
+            datetime.date(2024, 1, 1), datetime.date(2024, 3, 1), 31
+        )
+        assert chunks[0] == (datetime.date(2024, 1, 1), datetime.date(2024, 1, 31))
+        assert chunks[-1][1] == datetime.date(2024, 3, 1)
+        # Contiguous, no overlap
+        for (_, prev_end), (next_start, _) in zip(chunks, chunks[1:]):
+            assert next_start == prev_end + datetime.timedelta(days=1)
+
+    def test_chunk_single_day(self):
+        d = datetime.date(2024, 1, 1)
+        assert chunk_date_range(d, d, 365) == [(d, d)]
+
+    def test_select_default_keeps_rid_and_key_stations(self):
+        records = [
+            {"station_id": 1, "rid_code": "P.1", "is_key_station": True},
+            {"station_id": 2, "rid_code": None, "is_key_station": False},
+            {"station_id": 3, "rid_code": None, "is_key_station": True},
+        ]
+        assert [r["station_id"] for r in select_stations(records)] == [1, 3]
+
+    def test_select_by_code_matches_rid_code_and_oldcode(self):
+        records = [
+            {"station_id": 1, "rid_code": "P.1", "oldcode": "ridhydro_P.1"},
+            {"station_id": 2, "rid_code": None, "oldcode": "CHM004"},
+            {"station_id": 3, "rid_code": "P.67", "oldcode": "P.67"},
+        ]
+        selected = select_stations(records, codes=["p.1", "chm004"])
+        assert [r["station_id"] for r in selected] == [1, 2]
+
+    def test_select_all(self):
+        records = [{"station_id": 1}, {"station_id": 2}]
+        assert select_stations(records, all_stations=True) == records

tests/test_hii_fill.py

@@ -0,0 +1,124 @@
+"""Tests for the in-memory HII gap-fill in the ML data loader."""
+
+import datetime
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from src.hii_collector import HiiStore
+from src.ml.data import fill_from_hii
+
+START = datetime.datetime(2024, 9, 1, 0, 0)
+
+
+def _hours(n, offset=0):
+    return [START + datetime.timedelta(hours=offset + i) for i in range(n)]
+
+
+def _base_frame(code="P.20", n=200, level=2.0):
+    return pd.DataFrame(
+        {
+            "timestamp": _hours(n),
+            "station_code": code,
+            "water_level": level,
+            "discharge": 100.0,
+        }
+    )
+
+
+@pytest.fixture
+def hii_db(tmp_path):
+    """SQLite DB with hii_* tables; returns (db_url, store)."""
+    db_url = f"sqlite:///{tmp_path}/hii_fill.db"
+    store = HiiStore(db_url, "sqlite")
+    assert store.connect()
+    return db_url, store
+
+def _seed_mirror(store, station_id, rid_code, hours, wl_msl, discharge=250.0):
+    from sqlalchemy import text
+
+    with store.engine.begin() as conn:
+        conn.execute(
+            text("INSERT INTO hii_wl_stations (id, rid_code) VALUES (:i, :c)"),
+            {"i": station_id, "c": rid_code},
+        )
+        conn.execute(
+            text(
+                "INSERT INTO hii_waterlevel (station_id, timestamp, wl_msl, discharge) "
+                "VALUES (:i, :t, :w, :d)"
+            ),
+            [
+                {"i": station_id, "t": t, "w": w, "d": discharge}
+                for t, w in zip(hours, wl_msl)
+            ],
+        )
+
+
+class TestFillFromHii:
+    def test_exact_mirror_fills_missing_hours(self, hii_db):
+        db_url, store = hii_db
+        base = _base_frame("P.20", n=200, level=2.0)
+        # Mirror covers the base window plus 48 extra hours, at MSL offset +300
+        _seed_mirror(store, 1, "P.20", _hours(248), [302.0] * 248)
+
+        filled = fill_from_hii(base, db_url, min_overlap_hours=168)
+        p20 = filled[filled["station_code"] == "P.20"]
+        assert len(p20) == 248
+        new_rows = p20[p20["timestamp"] >= START + datetime.timedelta(hours=200)]
+        assert len(new_rows) == 48
+        # MSL converted back to gauge datum via the overlap-derived offset
+        assert new_rows["water_level"].round(6).eq(2.0).all()
+        # Exact mirrors copy discharge
+        assert new_rows["discharge"].eq(250.0).all()
+
+    def test_p81_bias_corrected_without_discharge(self, hii_db):
+        db_url, store = hii_db
+        base = _base_frame("P.81", n=200, level=1.5)
+        # Biased mirror: offset 310.2 (not a published offset — derived only)
+        _seed_mirror(store, 2, "P.81", _hours(230), [311.7] * 230)
+
+        filled = fill_from_hii(base, db_url, min_overlap_hours=168)
+        p81 = filled[filled["station_code"] == "P.81"]
+        new_rows = p81[p81["timestamp"] >= START + datetime.timedelta(hours=200)]
+        assert len(new_rows) == 30
+        assert new_rows["water_level"].round(6).eq(1.5).all()
+        assert new_rows["discharge"].isna().all()
+
+    def test_insufficient_overlap_skips_station(self, hii_db):
+        db_url, store = hii_db
+        base = _base_frame("P.20", n=50)  # only 50 shared hours
+        _seed_mirror(store, 1, "P.20", _hours(100), [302.0] * 100)
+
+        filled = fill_from_hii(base, db_url, min_overlap_hours=168)
+        assert len(filled) == len(base)
+
+    def test_nan_level_rows_are_replaced(self, hii_db):
+        db_url, store = hii_db
+        base = _base_frame("P.20", n=200, level=2.0)
+        base.loc[10, "water_level"] = np.nan
+        _seed_mirror(store, 1, "P.20", _hours(200), [302.5] * 200)
+
+        filled = fill_from_hii(base, db_url, min_overlap_hours=168)
+        p20 = filled[filled["station_code"] == "P.20"]
+        assert len(p20) == 200  # no duplicate hour
+        replaced = p20[p20["timestamp"] == START + datetime.timedelta(hours=10)]
+        assert replaced["water_level"].round(6).eq(2.0).all()
+
+    def test_never_merged_station_untouched(self, hii_db):
+        db_url, store = hii_db
+        base = _base_frame("P.76", n=200, level=1.0)
+        _seed_mirror(store, 3, "P.76", _hours(300), [301.0] * 300)
+
+        filled = fill_from_hii(base, db_url, min_overlap_hours=1)
+        assert len(filled) == len(base)  # P.76 not in HII_FILL_STATIONS
+
+    def test_missing_tables_degrade_gracefully(self, tmp_path):
+        base = _base_frame("P.20")
+        filled = fill_from_hii(base, f"sqlite:///{tmp_path}/empty.db")
+        assert filled.equals(base)
+
+    def test_no_station_overlap_returns_input(self, hii_db):
+        db_url, _ = hii_db
+        base = _base_frame("P.5")  # not fillable
+        assert fill_from_hii(base, db_url).equals(base)

tests/test_matrix_formatting.py

@@ -0,0 +1,98 @@
+"""Assert-based tests for Matrix message formatting.
+
+Matrix clients only render formatting from an HTML ``formatted_body``; Markdown
+in the plain ``body`` shows as literal characters. These tests lock in that the
+notifier emits real HTML plus a clean plain-text fallback, and that untrusted
+station data is HTML-escaped.
+"""
+
+import datetime
+
+from src.alerting import AlertLevel, MatrixNotifier, WaterAlert, markdown_to_matrix_html, strip_markdown
+
+
+class _FakeResponse:
+    def raise_for_status(self):
+        pass
+
+    def json(self):
+        return {"event_id": "$test"}
+
+
+def _notifier_capturing(captured):
+    """A MatrixNotifier whose HTTP PUT records the JSON payload into ``captured``."""
+    notifier = MatrixNotifier("https://hs.example", "token", "!room:hs.example")
+
+    def fake_put(url, headers=None, json=None, timeout=None):
+        captured.update(json)
+        return _FakeResponse()
+
+    notifier.session.put = fake_put
+    return notifier
+
+
+def test_bold_becomes_strong():
+    assert markdown_to_matrix_html("**hi**") == "<strong>hi</strong>"
+
+
+def test_url_is_linkified():
+    out = markdown_to_matrix_html("see https://x.example/z")
+    assert '<a href="https://x.example/z">https://x.example/z</a>' in out
+
+
+def test_newlines_become_br():
+    assert markdown_to_matrix_html("a\nb") == "a<br/>b"
+
+
+def test_html_is_escaped():
+    out = markdown_to_matrix_html("<script> & 'stuff'")
+    assert "&lt;script&gt;" in out
+    assert "&amp;" in out
+    assert "<script>" not in out
+
+
+def test_strip_markdown_removes_bold_markers():
+    assert strip_markdown("**WATER LEVEL ALERT**") == "WATER LEVEL ALERT"
+
+
+def test_send_message_sends_html_and_plain_fallback():
+    captured = {}
+    notifier = _notifier_capturing(captured)
+
+    assert notifier.send_message("**hi** http://x.example") is True
+    assert captured["format"] == "org.matrix.custom.html"
+    assert "<strong>hi</strong>" in captured["formatted_body"]
+    # Plain body has the markdown markers stripped.
+    assert captured["body"] == "hi http://x.example"
+
+
+def test_send_message_plain_when_markdown_disabled():
+    captured = {}
+    notifier = _notifier_capturing(captured)
+
+    assert notifier.send_message("**raw**", markdown=False) is True
+    assert "formatted_body" not in captured
+    assert captured["body"] == "**raw**"
+
+
+def test_send_alert_renders_alert_fields():
+    captured = {}
+    notifier = _notifier_capturing(captured)
+    alert = WaterAlert(
+        station_code="P.1",
+        station_name="สะพานนวรัฐ",
+        alert_type="Zone 7 - Critical",
+        level=AlertLevel.CRITICAL,
+        water_level=4.62,
+        threshold=4.60,
+        discharge=612.0,
+        timestamp=datetime.datetime(2026, 7, 22, 14, 30, 0),
+    )
+
+    assert notifier.send_alert(alert) is True
+    html = captured["formatted_body"]
+    assert "<strong>WATER LEVEL ALERT</strong>" in html
+    assert "สะพานนวรัฐ" in html  # Thai station name preserved
+    assert "<strong>Current Level:</strong>" in html
+    # Plain fallback carries no leftover markdown markers.
+    assert "**" not in captured["body"]

tests/test_postgres_history.py

@@ -0,0 +1,49 @@
+import datetime
+
+from sqlalchemy import create_engine, text
+
+from src.postgres_history import PostgresHistory
+
+
+def test_history_returns_station_series_in_chronological_order(tmp_path):
+    engine = create_engine(f"sqlite:///{tmp_path / 'history.db'}")
+    with engine.begin() as connection:
+        connection.execute(text("CREATE TABLE stations (id INTEGER PRIMARY KEY, station_code TEXT)"))
+        connection.execute(
+            text(
+                "CREATE TABLE water_measurements ("
+                "timestamp DATETIME, station_id INTEGER, water_level REAL, "
+                "discharge REAL, discharge_percent REAL)"
+            )
+        )
+        connection.execute(text("INSERT INTO stations VALUES (1, 'P.1'), (2, 'P.20')"))
+        connection.execute(
+            text(
+                "INSERT INTO water_measurements VALUES "
+                "('2026-08-09 13:00:00', 1, 3.2, 110.0, 40.0),"
+                "('2026-08-09 14:00:00', 1, 3.4, 120.0, 42.0),"
+                "('2026-08-09 14:00:00', 2, 2.1, 30.0, 15.0)"
+            )
+        )
+
+    history = PostgresHistory(engine=engine).station_history(
+        "P.1",
+        start=datetime.datetime(2026, 8, 9, 12),
+        end=datetime.datetime(2026, 8, 9, 15),
+        limit=100,
+    )
+
+    assert [row["timestamp"].hour for row in history] == [13, 14]
+    assert [row["discharge"] for row in history] == [110.0, 120.0]
+    assert all(row["station_code"] == "P.1" for row in history)
+
+
+def test_history_rejects_excessive_limit():
+    history = PostgresHistory.__new__(PostgresHistory)
+
+    try:
+        history.station_history("P.1", datetime.datetime.now(), datetime.datetime.now(), 100001)
+    except ValueError as error:
+        assert "limit" in str(error)
+    else:
+        raise AssertionError("Expected excessive history limit to be rejected")

tests/test_rid_reservoir.py

@@ -0,0 +1,218 @@
+"""Tests for the RID large-dam daily collector (parsing + persistence)."""
+
+import datetime
+
+import pytest
+
+from src.rid_reservoir import (
+    MAE_NGAT_DAM_ID,
+    RidReservoirCollector,
+    RidReservoirStore,
+    backfill,
+    parse_dam_records,
+)
+
+
+def _dams_payload(date="2026-08-13"):
+    return {
+        "date_th": date,
+        "regions": [
+            {
+                "region_name": "เหนือ",
+                "dams": [
+                    {
+                        "DAM_ID": "200103",
+                        "DAM_Name": "เขื่อนแม่งัดสมบูรณ์ชล",
+                        "DAM_Lat": 19.16138,
+                        "DAM_Lon": 99.04011,
+                        "DMD_Date": date,
+                        "DAM_QMax": "323.00",
+                        "DAM_QStore": "265.00",
+                        "DMD_QUse": "222.01",
+                        "PERCENT_DMD_QUse": "84.00",
+                        "DMD_Inflow": "3.44",
+                        "DMD_Outflow": "3.57",
+                        "DMD_Q": " - ",  # placeholder -> None
+                    },
+                    # No DAM_ID -> skipped
+                    {"DAM_Name": "broken", "DMD_Date": date},
+                    # Bad date -> skipped
+                    {"DAM_ID": "200199", "DMD_Date": "n/a"},
+                ],
+            },
+            {
+                "region_name": "กลาง",
+                "dams": [
+                    {
+                        "DAM_ID": "200301",
+                        "DAM_Name": "เขื่อนป่าสักชลสิทธิ์",
+                        "DMD_Date": date,
+                        "DMD_QUse": "500.10",
+                        "DMD_Q": "255.57",
+                    }
+                ],
+            },
+        ],
+    }
+
+
+class FakeClient:
+    def __init__(self, payload=None, fail_dates=()):
+        self.payload = payload or _dams_payload()
+        self.fail_dates = set(fail_dates)
+        self.calls = []
+
+    def fetch_day(self, date=None):
+        self.calls.append(date)
+        if date in self.fail_dates:
+            raise ConnectionError("boom")
+        if date is not None:
+            return parse_dam_records(_dams_payload(date.isoformat()))
+        return parse_dam_records(self.payload)
+
+
+class TestParsing:
+    def test_parse_dam_records(self):
+        records = parse_dam_records(_dams_payload())
+        assert len(records) == 2  # broken rows skipped
+        ngat = next(r for r in records if r["dam_id"] == MAE_NGAT_DAM_ID)
+        assert ngat["region"] == "เหนือ"
+        assert ngat["date"] == datetime.date(2026, 8, 13)
+        assert ngat["storage_mcm"] == 222.01
+        assert ngat["storage_pct"] == 84.0
+        assert ngat["inflow_mcm"] == 3.44
+        assert ngat["outflow_mcm"] == 3.57
+        assert ngat["level_msl"] is None  # ' - ' placeholder
+        assert ngat["capacity_normal_mcm"] == 265.0
+
+    def test_parse_empty_payload(self):
+        assert parse_dam_records({}) == []
+
+
+class TestStore:
+    @pytest.fixture
+    def store(self, tmp_path):
+        store = RidReservoirStore(f"sqlite:///{tmp_path}/rsv_test.db", "sqlite")
+        assert store.connect()
+        return store
+
+    def test_rejects_non_sql(self):
+        with pytest.raises(ValueError):
+            RidReservoirStore("http://localhost:8428", "victoriametrics")
+
+    def test_roundtrip_and_upsert(self, store):
+        records = parse_dam_records(_dams_payload())
+        assert store.save(records) == 2
+        # Second save of the same day updates in place, no duplicates
+        records[0]["storage_mcm"] = 230.00
+        assert store.save(records) == 2
+        from sqlalchemy import text
+
+        with store.engine.begin() as conn:
+            rows = conn.execute(
+                text(
+                    "SELECT dam_id, storage_mcm FROM rid_reservoir_daily "
+                    "ORDER BY dam_id"
+                )
+            ).fetchall()
+            dams = conn.execute(text("SELECT COUNT(*) FROM rid_dams")).scalar()
+        assert len(rows) == 2
+        assert dams == 2
+        assert float(rows[0][1]) == 230.00
+
+    def test_save_empty(self, store):
+        assert store.save([]) == 0
+
+    def test_present_dates(self, store):
+        lo, hi = datetime.date(2026, 8, 1), datetime.date(2026, 8, 31)
+        assert store.present_dates(lo, hi) == set()
+        store.save(parse_dam_records(_dams_payload()))
+        assert store.present_dates(lo, hi) == {datetime.date(2026, 8, 13)}
+        # Outside the window -> excluded
+        assert store.present_dates(lo, datetime.date(2026, 8, 12)) == set()
+
+
+class TestCollectorAndBackfill:
+    def test_run_cycle_today_and_yesterday(self, tmp_path):
+        collector = RidReservoirCollector(
+            {"type": "sqlite", "connection_string": f"sqlite:///{tmp_path}/c.db"},
+            client=FakeClient(),
+        )
+        saved = collector.run_cycle()
+        assert saved == 4  # 2 dams x (today + yesterday)
+        assert collector.client.calls[0] is None  # today via empty date
+
+    def test_run_cycle_survives_fetch_failure(self, tmp_path):
+        yesterday = datetime.date.today() - datetime.timedelta(days=1)
+        collector = RidReservoirCollector(
+            {"type": "sqlite", "connection_string": f"sqlite:///{tmp_path}/f.db"},
+            client=FakeClient(fail_dates={yesterday}),
+        )
+        assert collector.run_cycle() == 2  # today only
+
+    def test_backfill_range(self, tmp_path):
+        store = RidReservoirStore(f"sqlite:///{tmp_path}/b.db", "sqlite")
+        assert store.connect()
+        client = FakeClient()
+        saved = backfill(
+            store,
+            datetime.date(2024, 9, 24),
+            datetime.date(2024, 9, 26),
+            client=client,
+            throttle_seconds=0,
+        )
+        assert saved == 6  # 3 days x 2 dams
+        assert client.calls == [
+            datetime.date(2024, 9, 24),
+            datetime.date(2024, 9, 25),
+            datetime.date(2024, 9, 26),
+        ]
+
+    def test_backfill_skips_present_days_so_reruns_repair_holes(self, tmp_path):
+        store = RidReservoirStore(f"sqlite:///{tmp_path}/skip.db", "sqlite")
+        assert store.connect()
+        # Day 25 already stored (e.g. by the live collector)
+        store.save(parse_dam_records(_dams_payload("2024-09-25")))
+        client = FakeClient()
+        saved = backfill(
+            store,
+            datetime.date(2024, 9, 24),
+            datetime.date(2024, 9, 26),
+            client=client,
+            throttle_seconds=0,
+        )
+        assert client.calls == [
+            datetime.date(2024, 9, 24),
+            datetime.date(2024, 9, 26),
+        ]
+        assert saved == 4
+
+    def test_backfill_aborts_when_db_saves_nothing(self, tmp_path, monkeypatch):
+        store = RidReservoirStore(f"sqlite:///{tmp_path}/deaddb.db", "sqlite")
+        assert store.connect()
+        monkeypatch.setattr(store, "save", lambda records: 0)  # broken DB
+        client = FakeClient()
+        backfill(
+            store,
+            datetime.date(2024, 1, 1),
+            datetime.date(2024, 3, 1),
+            client=client,
+            throttle_seconds=0,
+        )
+        assert len(client.calls) == 5  # aborted, not one request per day
+
+    def test_backfill_aborts_after_consecutive_failures(self, tmp_path):
+        store = RidReservoirStore(f"sqlite:///{tmp_path}/a.db", "sqlite")
+        assert store.connect()
+        start = datetime.date(2024, 1, 1)
+        fail_dates = {start + datetime.timedelta(days=i) for i in range(2, 30)}
+        client = FakeClient(fail_dates=fail_dates)
+        saved = backfill(
+            store,
+            start,
+            datetime.date(2024, 3, 1),
+            client=client,
+            throttle_seconds=0,
+        )
+        assert saved == 4  # first 2 days succeeded, then 5 failures -> abort
+        assert len(client.calls) == 7

tests/test_scraper_parsing.py

@@ -0,0 +1,125 @@
+"""Assert-based tests for the RID API response parsing.
+
+The parsing in ``fetch_water_data_for_date`` is the riskiest, previously
+untested code: it maps the API's 1..24 "hourlytime" onto real timestamps
+(hour 24 rolls to next-day midnight) and treats ``"***"``/``None`` discharge as
+missing. These tests mock the HTTP call so no network is touched and stub the
+validator so we assert on the parser's output directly.
+"""
+
+import datetime
+from unittest.mock import MagicMock
+
+import pytest
+
+import src.water_scraper_v3 as scraper_mod
+from src.water_scraper_v3 import EnhancedWaterMonitorScraper as Scraper
+
+TARGET = datetime.datetime(2026, 7, 22)
+
+
+@pytest.fixture
+def make_scraper(monkeypatch):
+    """Return a factory that builds a bare scraper returning the given API rows."""
+
+    def _factory(rows):
+        scraper = Scraper.__new__(Scraper)  # bypass __init__ (no DB/network)
+        scraper.api_url = "https://example.invalid/api"
+        scraper.rate_limiter = MagicMock()
+        scraper.request_tracker = MagicMock()
+        scraper.station_config_path = "/nonexistent/stations.json"
+        scraper.station_mapping = scraper._load_station_mapping()  # bundled defaults
+
+        response = MagicMock()
+        response.json.return_value = {"rows": rows}
+        response.raise_for_status.return_value = None
+        scraper.session = MagicMock()
+        scraper.session.post.return_value = response
+
+        # Isolate parsing from validation.
+        monkeypatch.setattr(
+            scraper_mod.DataValidator,
+            "validate_measurements",
+            staticmethod(lambda m: m),
+        )
+        return scraper
+
+    return _factory
+
+
+def test_parses_water_level_and_discharge(make_scraper):
+    rows = [
+        {
+            "hourlytime": "9.00",
+            "wlvalues1": "3.50",
+            "qvalues1": "120.5",
+            "QPercent1": "45.2",
+        }
+    ]
+    data = make_scraper(rows).fetch_water_data_for_date(TARGET)
+
+    p20 = [d for d in data if d["station_code"] == "P.20"]
+    assert len(p20) == 1
+    m = p20[0]
+    assert m["water_level"] == 3.5
+    assert m["discharge"] == 120.5
+    assert m["discharge_percent"] == 45.2
+    assert m["timestamp"] == datetime.datetime(2026, 7, 22, 9, 0)
+    assert m["station_name_en"] == "Ban Chiang Dao"
+
+
+def test_discharge_asterisks_becomes_none(make_scraper):
+    rows = [{"hourlytime": "10.00", "wlvalues8": "4.20", "qvalues8": "***"}]
+    data = make_scraper(rows).fetch_water_data_for_date(TARGET)
+
+    p1 = [d for d in data if d["station_code"] == "P.1"][0]
+    assert p1["water_level"] == 4.2
+    assert p1["discharge"] is None
+    assert p1["discharge_percent"] is None
+
+
+def test_hour_24_rolls_to_next_day_midnight(make_scraper):
+    rows = [{"hourlytime": "24.00", "wlvalues1": "3.00"}]
+    data = make_scraper(rows).fetch_water_data_for_date(TARGET)
+
+    assert data[0]["timestamp"] == datetime.datetime(2026, 7, 23, 0, 0)
+
+
+def test_hours_1_to_23_stay_same_day(make_scraper):
+    rows = [
+        {"hourlytime": "1.00", "wlvalues1": "3.00"},
+        {"hourlytime": "23.00", "wlvalues1": "3.10"},
+    ]
+    data = make_scraper(rows).fetch_water_data_for_date(TARGET)
+
+    times = sorted(d["timestamp"] for d in data)
+    assert times == [
+        datetime.datetime(2026, 7, 22, 1, 0),
+        datetime.datetime(2026, 7, 22, 23, 0),
+    ]
+
+
+def test_none_water_level_is_skipped(make_scraper):
+    rows = [{"hourlytime": "9.00", "wlvalues1": None, "wlvalues2": "2.5"}]
+    data = make_scraper(rows).fetch_water_data_for_date(TARGET)
+
+    codes = {d["station_code"] for d in data}
+    assert "P.20" not in codes  # station 1 skipped (None water level)
+    assert "P.75" in codes  # station 2 present
+
+
+def test_out_of_range_and_empty_hours_skipped(make_scraper):
+    rows = [
+        {"hourlytime": "25.00", "wlvalues1": "3.0"},
+        {"hourlytime": "0.00", "wlvalues1": "3.0"},
+        {"hourlytime": "", "wlvalues1": "3.0"},
+    ]
+    data = make_scraper(rows).fetch_water_data_for_date(TARGET)
+
+    assert data == []
+
+
+def test_missing_rows_key_returns_empty(make_scraper):
+    scraper = make_scraper([])
+    scraper.session.post.return_value.json.return_value = {"unexpected": True}
+    assert scraper.fetch_water_data_for_date(TARGET) == []

tests/test_station_persistence.py

@@ -0,0 +1,77 @@
+"""Assert-based tests for station-mapping persistence.
+
+Station CRUD must survive restarts: the scraper loads its mapping from a
+runtime-writable JSON file (falling back to bundled defaults) and writes it back
+atomically. These tests exercise that load/save behaviour without constructing a
+full scraper (which would open network/DB connections).
+"""
+
+from src.water_scraper_v3 import EnhancedWaterMonitorScraper as Scraper
+
+
+def _bare_scraper(config_path):
+    """A scraper instance with only the station-config attribute set.
+
+    Bypasses __init__ so no database/HTTP connection is attempted.
+    """
+    scraper = Scraper.__new__(Scraper)
+    scraper.station_config_path = config_path
+    return scraper
+
+
+def test_loads_bundled_defaults_when_runtime_file_absent(tmp_path):
+    scraper = _bare_scraper(str(tmp_path / "does_not_exist.json"))
+    mapping = scraper._load_station_mapping()
+
+    assert len(mapping) == 16
+    assert mapping["8"]["code"] == "P.1"
+    assert mapping["8"]["english_name"] == "Nawarat Bridge"
+
+
+def test_save_then_reload_roundtrips_including_thai(tmp_path):
+    path = str(tmp_path / "stations.json")
+    scraper = _bare_scraper(path)
+    scraper.station_mapping = {
+        "1": {
+            "code": "P.99",
+            "thai_name": "สถานีทดสอบ",
+            "english_name": "Test Station",
+            "latitude": 1.0,
+            "longitude": 2.0,
+            "geohash": None,
+        }
+    }
+
+    assert scraper.save_stations() is True
+
+    reloaded = _bare_scraper(path)._load_station_mapping()
+    assert reloaded == scraper.station_mapping
+    assert reloaded["1"]["thai_name"] == "สถานีทดสอบ"
+
+
+def test_runtime_file_takes_precedence_over_defaults(tmp_path):
+    path = str(tmp_path / "stations.json")
+    writer = _bare_scraper(path)
+    writer.station_mapping = {"1": {"code": "ONLY"}}
+    assert writer.save_stations() is True
+
+    mapping = _bare_scraper(path)._load_station_mapping()
+    assert list(mapping.keys()) == ["1"]
+    assert mapping["1"]["code"] == "ONLY"
+
+
+def test_save_returns_false_without_a_path():
+    scraper = _bare_scraper("")
+    scraper.station_mapping = {}
+    assert scraper.save_stations() is False
+
+
+def test_save_is_atomic_no_tmp_left_behind(tmp_path):
+    path = tmp_path / "stations.json"
+    scraper = _bare_scraper(str(path))
+    scraper.station_mapping = {"1": {"code": "P.1"}}
+
+    assert scraper.save_stations() is True
+    assert path.exists()
+    # The temp file used during the atomic write must not remain.
+    assert not (tmp_path / "stations.json.tmp").exists()

tests/test_thaiwater.py

@@ -0,0 +1,75 @@
+from unittest.mock import MagicMock
+
+from src.thaiwater import ThaiWaterClient
+
+
+SAMPLE_RESPONSE = {
+    "data": {
+        "50": {
+            "type": "FeatureCollection",
+            "features": [
+                {
+                    "geometry": {"type": "Point", "coordinates": [98.635262, 19.638411]},
+                    "properties": {
+                        "id": "123",
+                        "waterlevelDatetime": "2026-08-09T15:00:00+07:00",
+                        "waterlevelMsl": 742.34,
+                        "storagePercent": 38.57,
+                        "diffWlBank": 1.87,
+                        "riverName": "Ping River",
+                        "station": {
+                            "stationCode": "G07003-P.65",
+                            "station": "Ban Muang Pok",
+                        },
+                        "agency": {"agencyShort": "RID"},
+                        "basin": {"basin": "Ping"},
+                    },
+                },
+                {
+                    "geometry": {"type": "Point", "coordinates": [100.1, 18.1]},
+                    "properties": {
+                        "id": "999",
+                        "station": {"stationCode": "N.1", "station": "Nan station"},
+                        "basin": {"basin": "Nan"},
+                    },
+                },
+            ],
+        }
+    }
+}
+
+
+def test_fetch_ping_sensors_normalizes_and_filters_basin():
+    session = MagicMock()
+    response = session.get.return_value
+    response.json.return_value = SAMPLE_RESPONSE
+    response.raise_for_status.return_value = None
+
+    sensors = ThaiWaterClient(session=session, api_key="public-key").fetch_ping_sensors()
+
+    assert sensors == [
+        {
+            "id": "thaiwater:123",
+            "station_code": "P.65",
+            "station_name": "Ban Muang Pok",
+            "latitude": 19.638411,
+            "longitude": 98.635262,
+            "timestamp": "2026-08-09T15:00:00+07:00",
+            "water_level_msl": 742.34,
+            "bank_percent": 38.57,
+            "distance_to_bank": 1.87,
+            "river_name": "Ping River",
+            "agency": "RID",
+            "source": "ThaiWater",
+        }
+    ]
+    session.get.assert_called_once()
+    assert session.get.call_args.kwargs["headers"]["x-api-key"] == "public-key"
+
+
+def test_fetch_ping_sensors_skips_features_without_coordinates():
+    response_data = {"data": {"50": {"features": [{"geometry": None, "properties": {"basin": {"basin": "Ping"}}}]}}}
+    session = MagicMock()
+    session.get.return_value.json.return_value = response_data
+
+    assert ThaiWaterClient(session=session, api_key="key").fetch_ping_sensors() == []