Compare commits
14
Commits
330fdf7804
..
master
| Author | SHA1 | Date | |
|---|---|---|---|
 grabowski
|
a40faf3102 | ||
|
grabowski
|
9bf904b23c | ||
|
grabowski
|
a610abcdb8 | ||
|
grabowski
|
701bb87d34 | ||
|
grabowski
|
180d95c6bd | ||
|
grabowski
|
cf10f2e4c8 | ||
|
grabowski
|
ffd3030352 | ||
|
grabowski
|
71cc1b0ca7 | ||
|
grabowski
|
301a629b42 | ||
|
grabowski
|
0e30ddbcb6 | ||
|
grabowski
|
7f29b1831e | ||
|
grabowski
|
24998b033c | ||
|
grabowski
|
6fe2e287e5 | ||
|
grabowski
|
7e642555b4 |
No files matched your search
+66
-119
@@ -1,141 +1,88 @@
|
||||
# Northern Thailand Ping River Monitor Configuration
|
||||
# Copy this file to .env and customize for your environment
|
||||
# Ping River Monitor configuration: copy to .env and fill in.
|
||||
#
|
||||
# Read by ping-collector (the service; rust/collector/src/config.rs) and by the
|
||||
# Python training tools (src/config.py). Under systemd the variables come from
|
||||
# EnvironmentFile= (systemd's parser); for manual runs both programs load
|
||||
# ./.env with python-dotenv's rules. Unset keys take the defaults shown.
|
||||
|
||||
# Database Configuration
|
||||
DB_TYPE=postgresql
|
||||
# Options: sqlite, mysql, postgresql, influxdb, victoriametrics
|
||||
|
||||
# SQLite Configuration (default)
|
||||
WATER_DB_PATH=water_levels.db
|
||||
|
||||
# VictoriaMetrics Configuration
|
||||
VM_HOST=localhost
|
||||
VM_PORT=8428
|
||||
VM_URL=
|
||||
|
||||
# InfluxDB Configuration
|
||||
INFLUX_HOST=localhost
|
||||
INFLUX_PORT=8086
|
||||
INFLUX_DATABASE=ping_river_monitoring
|
||||
INFLUX_USERNAME=
|
||||
INFLUX_PASSWORD=
|
||||
|
||||
# PostgreSQL Configuration (Remote Server)
|
||||
# Option 1: Full connection string (URL encode special characters in password)
|
||||
POSTGRES_CONNECTION_STRING=postgresql://username:url_encoded_password@your-postgres-host:5432/water_monitoring
|
||||
|
||||
# Option 2: Individual components (password will be automatically URL encoded)
|
||||
POSTGRES_HOST=your-postgres-host
|
||||
# --- Database (PostgreSQL only) ---------------------------------------------
|
||||
# Either a full connection string (URL-encode special characters in the
|
||||
# password: : %3A @ %40 # %23 ? %3F & %26 / %2F % %25) ...
|
||||
POSTGRES_CONNECTION_STRING=
|
||||
# ... or the parts (the password is encoded for you)
|
||||
POSTGRES_HOST=localhost
|
||||
POSTGRES_PORT=5432
|
||||
POSTGRES_DB=water_monitoring
|
||||
POSTGRES_USER=username
|
||||
POSTGRES_PASSWORD=your:password@with!special#chars
|
||||
POSTGRES_USER=postgres
|
||||
POSTGRES_PASSWORD=
|
||||
|
||||
# Examples for connection string:
|
||||
# - Local: postgresql://postgres:password@localhost:5432/water_monitoring
|
||||
# - Remote: postgresql://user:pass@192.168.1.100:5432/water_monitoring
|
||||
# - With special chars: postgresql://user:my%3Apass%40word@host:5432/db
|
||||
# - With SSL: postgresql://user:pass@host:port/db?sslmode=require
|
||||
# - Connection pooling: postgresql://user:pass@host:port/db?pool_size=20&max_overflow=0
|
||||
# --- What the service runs ---------------------------------------------------
|
||||
# Website and API: the address to listen on (empty = no website)
|
||||
WEB_LISTEN=0.0.0.0:8000
|
||||
# Forecast + ntfy/Matrix notifications after each collection (false = collect only)
|
||||
FORECAST_IN_COLLECTOR=true
|
||||
# Trained models (models/flood_<station>.json.gz from src/ml/export.py)
|
||||
# MODELS_DIR=models
|
||||
|
||||
# Special character URL encoding:
|
||||
# : → %3A @ → %40 # → %23 ? → %3F & → %26 / → %2F % → %25
|
||||
|
||||
# MySQL Configuration
|
||||
MYSQL_CONNECTION_STRING=mysql://user:password@localhost:3306/ping_river_monitoring
|
||||
|
||||
# API Configuration
|
||||
API_HOST=0.0.0.0
|
||||
API_PORT=8000
|
||||
API_WORKERS=1
|
||||
# Public ThaiWater API key used to add Ping-basin water-level sensors.
|
||||
THAIWATER_API_KEY=
|
||||
|
||||
# Data Collection Settings
|
||||
SCRAPING_INTERVAL_HOURS=1
|
||||
# RID poll minute past the hour; retry interval while RID is behind
|
||||
# --- Collection ----------------------------------------------------------------
|
||||
# RID is polled this many minutes past each hour and retried every
|
||||
# RID_RETRY_MINUTES while the current hour is not published yet
|
||||
RID_POLL_OFFSET_MINUTES=5
|
||||
RID_RETRY_MINUTES=10
|
||||
# Days of RID holes the leader re-fetches at startup (0 = off)
|
||||
# RID reports with holes re-fetched at startup (0 = off); older: `ping-collector fill-gaps all`
|
||||
GAP_FILL_DAYS_ON_START=14
|
||||
# false when the Rust ping-collector service collects (rust/collector);
|
||||
# share the status file between both services for /scraping/status
|
||||
COLLECTION_ENABLED=true
|
||||
# SCRAPING_STATUS_FILE=/opt/thailand-water-monitor/collector-status.json
|
||||
# true: ping-collector also forecasts (models/flood_*.json.gz, rust/forecast)
|
||||
# and sends the ntfy notifications; this app serves its stored forecast and
|
||||
# computes itself only when that is older than FORECAST_MAX_AGE_SECONDS.
|
||||
# Set by scripts/install_collector.sh --cutover-forecast.
|
||||
FORECAST_IN_COLLECTOR=false
|
||||
# FORECAST_MAX_AGE_SECONDS=7200
|
||||
# MODELS_DIR=models
|
||||
REQUEST_TIMEOUT=30
|
||||
MAX_RETRIES=3
|
||||
RETRY_DELAY_SECONDS=60
|
||||
|
||||
# Data Retention
|
||||
DATA_RETENTION_DAYS=365
|
||||
|
||||
# Logging Configuration
|
||||
# ENABLE_HII_COLLECTION=true
|
||||
# ENABLE_RESERVOIR_COLLECTION=true
|
||||
# ENABLE_RAIN_COLLECTION=true
|
||||
# HII_BASIN_CODE=6
|
||||
# REQUEST_TIMEOUT=30
|
||||
# Runtime station list written by the admin API; falls back to src/data/stations.json
|
||||
# STATION_CONFIG_PATH=stations.json
|
||||
LOG_LEVEL=INFO
|
||||
LOG_FILE=water_monitor.log
|
||||
|
||||
# Security (for production)
|
||||
SECRET_KEY=your-secret-key-here
|
||||
API_KEY=your-api-key-here
|
||||
|
||||
# Monitoring
|
||||
ENABLE_METRICS=true
|
||||
ENABLE_HEALTH_CHECKS=true
|
||||
|
||||
# Geographic Settings
|
||||
TIMEZONE=Asia/Bangkok
|
||||
DEFAULT_LATITUDE=18.7875
|
||||
DEFAULT_LONGITUDE=99.0045
|
||||
|
||||
# External Services
|
||||
NOTIFICATION_EMAIL=
|
||||
SMTP_SERVER=
|
||||
SMTP_PORT=587
|
||||
SMTP_USERNAME=
|
||||
SMTP_PASSWORD=
|
||||
|
||||
# Public push notifications via self-hosted ntfy (https://ntfy.sh, single binary).
|
||||
# Leave NTFY_SERVER empty to disable. Topics published: <prefix>-<station>-warning,
|
||||
# <prefix>-<station>-danger, <prefix>-warning, <prefix>-danger, <prefix>-p1-outlook,
|
||||
# <prefix>-status. See docs/NOTIFICATIONS.md.
|
||||
# --- Notifications (docs/NOTIFICATIONS.md) ------------------------------------
|
||||
# Public push alerts via self-hosted ntfy; empty NTFY_SERVER = off. Topics:
|
||||
# <prefix>-<station>-warning|danger, <prefix>-warning|danger,
|
||||
# <prefix>-p1-outlook, <prefix>-status.
|
||||
NTFY_SERVER=
|
||||
# Where the monitor POSTs (defaults to NTFY_SERVER). Use the local ntfy
|
||||
# address (loopback or Tailscale IP) so publishing does not depend on
|
||||
# DNS / the reverse proxy being up.
|
||||
# Where the service POSTs (default NTFY_SERVER): the local ntfy address, so
|
||||
# publishing does not depend on DNS or the reverse proxy
|
||||
NTFY_PUBLISH_URL=
|
||||
NTFY_TOPIC_PREFIX=ping
|
||||
NTFY_TOKEN=
|
||||
PUBLIC_URL=https://water.buildfor.life/
|
||||
# ping-collector notifier: report RID lagging this many hours while ThaiWater
|
||||
# telemetry is current (0 = never)
|
||||
# Report RID lagging this many hours while ThaiWater telemetry is current (0 = never)
|
||||
RID_STALE_ALERT_HOURS=18
|
||||
|
||||
# Matrix: ping-collector sends the ntfy transitions to this room too
|
||||
# (docs/MATRIX_QUICK_START.md); off while token or room is empty
|
||||
# RID publishes nothing overnight: from 21:00 until this hour a quiet river's
|
||||
# feed is not reported stale (0 = off; docs/NOTIFICATIONS.md)
|
||||
FEED_NIGHT_UNTIL_HOUR=9
|
||||
# Matrix room for a team: the same transitions (docs/MATRIX_QUICK_START.md);
|
||||
# off while token or room is empty
|
||||
MATRIX_HOMESERVER=https://matrix.org
|
||||
MATRIX_ACCESS_TOKEN=
|
||||
MATRIX_ROOM_ID=
|
||||
|
||||
# Website and API served by ping-collector (rust/collector/src/web): the
|
||||
# address to listen on; empty = the Python app serves (port 8000). Set by
|
||||
# scripts/install_collector.sh --cutover-api.
|
||||
WEB_LISTEN=
|
||||
# --- Website and API -------------------------------------------------------------
|
||||
# X-API-Key for station editing, /config and POST /scrape/trigger (empty = those are off)
|
||||
ADMIN_API_KEY=
|
||||
# Comma-separated origins allowed cross-site with credentials (empty: any origin, no credentials)
|
||||
CORS_ALLOW_ORIGINS=
|
||||
# ThaiWater API key for /sensors/thaiwater (extra Ping-basin sensors on the map)
|
||||
THAIWATER_API_KEY=
|
||||
# Self-hosted Umami: server-side tracking of API calls
|
||||
# UMAMI_API_URL=https://stats.buildfor.life/api/send
|
||||
# UMAMI_WEBSITE_ID=00b2be73-8f5f-4400-9029-3be852eb08f7
|
||||
# UMAMI_TRACK_API=true
|
||||
# Response cache TTLs (seconds)
|
||||
# LATEST_CACHE_TTL_SECONDS=45
|
||||
# HII_CACHE_TTL_SECONDS=120
|
||||
# HEALTH_CACHE_TTL_SECONDS=30
|
||||
# A stored forecast older than this is served marked X-Data-Stale: true
|
||||
# FORECAST_MAX_AGE_SECONDS=7200
|
||||
# STATIC_DIR=src/static
|
||||
# ADMIN_API_KEY= # X-API-Key for station editing, /config, /scrape/trigger
|
||||
# Only for `ping-collector serve` running separately from `run`: the status file
|
||||
# SCRAPING_STATUS_FILE=collector-status.json
|
||||
|
||||
# Grafana Integration
|
||||
GRAFANA_URL=http://localhost:3000
|
||||
|
||||
# Alert Configuration
|
||||
ALERT_MAX_AGE_HOURS=2
|
||||
ALERT_CHECK_INTERVAL_MINUTES=15
|
||||
|
||||
# Development Settings
|
||||
DEBUG=false
|
||||
DEVELOPMENT_MODE=false
|
||||
# --- Python tools ----------------------------------------------------------------
|
||||
# Training/evaluation read the database above; this overrides it for them only
|
||||
# FLOOD_ML_DB_URL=
|
||||
+25
-15
@@ -1,13 +1,13 @@
|
||||
name: CI
|
||||
|
||||
# What this checks, on every push and PR to master:
|
||||
# The Python side (src/ml training/evaluation stack, archive backfills), on
|
||||
# every push and PR to master:
|
||||
# 1. formatting contract (black + isort, config in pyproject.toml)
|
||||
# 2. flake8 hard-error gate (syntax, undefined names)
|
||||
# 3. the pytest suite (synthetic data, no DB/network; ~1 min)
|
||||
# Docker build / staging / production / perf jobs from the original template
|
||||
# were removed: there is no registry, no staging host, and production is a
|
||||
# systemd unit deployed by `git pull` on the server (docs/FLOOD_FORECASTING.md
|
||||
# section 6, scripts/install.sh). Re-add a job when the thing it deploys exists.
|
||||
# 3. the pytest suite: synthetic data, no network; the database tests run
|
||||
# against a throwaway PostgreSQL service (TEST_DATABASE_URL, as in rust.yml)
|
||||
# The service itself (ping-collector) is checked by rust.yml. Production is a
|
||||
# systemd unit deployed by `git pull` + scripts/install.sh on the server.
|
||||
|
||||
on:
|
||||
push:
|
||||
@@ -15,7 +15,7 @@ on:
|
||||
pull_request:
|
||||
branches: [master]
|
||||
schedule:
|
||||
# daily, catches dependency drift / upstream API changes in the tests
|
||||
# daily, catches dependency drift
|
||||
- cron: "0 2 * * *"
|
||||
workflow_dispatch:
|
||||
|
||||
@@ -41,17 +41,30 @@ jobs:
|
||||
pip install --root-user-action=ignore black==26.5.1 isort==5.12.0 flake8==6.1.0
|
||||
|
||||
- name: black
|
||||
run: black --check --diff src/ *.py
|
||||
run: black --check --diff src/ scripts/ tests/ rust/forecast/testdata rust/collector/testdata
|
||||
|
||||
- name: isort
|
||||
run: isort --check-only --diff src/ *.py
|
||||
run: isort --check-only --diff src/ scripts/ tests/
|
||||
|
||||
- name: flake8 (errors only)
|
||||
run: flake8 src/ --count --select=E9,F63,F7,F82 --show-source --statistics
|
||||
run: flake8 src/ scripts/ tests/ --count --select=E9,F63,F7,F82 --show-source --statistics
|
||||
|
||||
test:
|
||||
name: Test suite
|
||||
runs-on: ubuntu-latest
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16
|
||||
env:
|
||||
POSTGRES_PASSWORD: ping
|
||||
POSTGRES_DB: ping
|
||||
options: >-
|
||||
--health-cmd pg_isready
|
||||
--health-interval 5s
|
||||
--health-timeout 5s
|
||||
--health-retries 20
|
||||
env:
|
||||
TEST_DATABASE_URL: postgres://postgres:ping@postgres:5432/ping
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
@@ -66,10 +79,7 @@ jobs:
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip --root-user-action=ignore
|
||||
pip install --root-user-action=ignore -r requirements.txt
|
||||
pip install --root-user-action=ignore pytest==9.1.1 pytest-asyncio==0.21.1
|
||||
pip install --root-user-action=ignore -r requirements.txt pytest==9.1.1
|
||||
|
||||
- name: pytest
|
||||
env:
|
||||
DB_TYPE: sqlite
|
||||
run: pytest -q -p no:cacheprovider
|
||||
run: pytest -q -p no:cacheprovider -rs
|
||||
+10
-54
@@ -1,36 +1,23 @@
|
||||
name: Docs
|
||||
|
||||
# Checks that the documentation the project actually ships stays consistent:
|
||||
# - every relative link / image path in docs/*.md and README.md resolves
|
||||
# inside the repo (external URLs are NOT fetched: localhost examples,
|
||||
# rate-limited hosts and the Tailscale-era links made that gate permanently
|
||||
# red, and a 200 on a curl --head proves nothing about a doc anyway)
|
||||
# - the FastAPI app imports and its OpenAPI schema is exportable (that is
|
||||
# the reference at https://water.buildfor.life/docs)
|
||||
# The previous Sphinx/apidoc jobs produced artifacts nobody read and were
|
||||
# removed. Reference docs live in docs/*.md; the public overview is at
|
||||
# https://buildfor.life/docs/tooling/ping-river-monitor/.
|
||||
# Every relative link / image path in the shipped Markdown (README, CLAUDE,
|
||||
# CONTRIBUTING, rust/README, docs/) resolves inside the repo. External URLs
|
||||
# are NOT fetched: localhost examples and rate-limited hosts made that gate
|
||||
# permanently red, and a 200 proves nothing about a doc anyway. The API
|
||||
# reference (/docs, /openapi.json) is generated by ping-collector from its
|
||||
# route table and tested in rust/collector/src/web/tests.rs.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [master, develop]
|
||||
paths:
|
||||
- "docs/**"
|
||||
- "README.md"
|
||||
- "CONTRIBUTING.md"
|
||||
- "src/web_api.py"
|
||||
- "src/schemas.py"
|
||||
- "**.md"
|
||||
- ".gitea/workflows/docs.yml"
|
||||
pull_request:
|
||||
paths:
|
||||
- "docs/**"
|
||||
- "README.md"
|
||||
- "CONTRIBUTING.md"
|
||||
- "**.md"
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
PYTHON_VERSION: "3.11"
|
||||
|
||||
jobs:
|
||||
docs:
|
||||
name: Validate documentation
|
||||
@@ -43,7 +30,8 @@ jobs:
|
||||
python3 - <<'PY'
|
||||
import re, sys, pathlib
|
||||
root = pathlib.Path(".")
|
||||
files = [root / "README.md", root / "CONTRIBUTING.md", *root.glob("docs/**/*.md")]
|
||||
files = [root / "README.md", root / "CLAUDE.md", root / "CONTRIBUTING.md",
|
||||
root / "rust" / "README.md", *root.glob("docs/**/*.md")]
|
||||
link = re.compile(r"!?\[[^\]]*\]\(([^)\s]+)(?:\s+\"[^\"]*\")?\)")
|
||||
bad = []
|
||||
for md in files:
|
||||
@@ -65,35 +53,3 @@ jobs:
|
||||
sys.exit(1)
|
||||
print(f"checked {len(files)} files, all relative links resolve")
|
||||
PY
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
cache: pip
|
||||
cache-dependency-path: requirements.txt
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip --root-user-action=ignore
|
||||
pip install --root-user-action=ignore -r requirements.txt
|
||||
|
||||
- name: OpenAPI schema exports
|
||||
env:
|
||||
DB_TYPE: sqlite
|
||||
run: |
|
||||
python - <<'PY'
|
||||
import json
|
||||
from src.web_api import app
|
||||
spec = app.openapi()
|
||||
paths = sorted(spec["paths"])
|
||||
required = {"/forecast", "/measurements/latest", "/measurements/history/{station_code}", "/stations", "/api/stats", "/health"}
|
||||
missing = required - set(paths)
|
||||
assert not missing, f"documented endpoints missing from the app: {missing}"
|
||||
json.dump(spec, open("openapi.json", "w"), indent=1)
|
||||
print(f"{len(paths)} paths; schema written to openapi.json")
|
||||
PY
|
||||
|
||||
- uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: openapi-${{ github.run_number }}
|
||||
path: openapi.json
|
||||
@@ -1,314 +0,0 @@
|
||||
name: Release - Northern Thailand Ping River Monitor
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "v*.*.*"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version:
|
||||
description: "Release version (e.g., v3.1.3)"
|
||||
required: true
|
||||
type: string
|
||||
|
||||
env:
|
||||
PYTHON_VERSION: "3.11"
|
||||
REGISTRY: git.b4l.co.th
|
||||
IMAGE_NAME: b4l/northern-thailand-ping-river-monitor
|
||||
# GitHub token for better rate limits and authentication
|
||||
GH_TOKEN: ${{ secrets.GH_TOKEN }}
|
||||
|
||||
jobs:
|
||||
# Create release
|
||||
create-release:
|
||||
name: Create Release
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
version: ${{ steps.version.outputs.version }}
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
token: ${{ secrets.GITEA_TOKEN }}
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Get version
|
||||
id: version
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "version=${{ github.event.inputs.version }}" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "version=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Generate changelog
|
||||
id: changelog
|
||||
run: |
|
||||
# Generate changelog from git commits
|
||||
echo "## Changes" > CHANGELOG.md
|
||||
git log --pretty=format:"- %s" $(git describe --tags --abbrev=0 HEAD^)..HEAD >> CHANGELOG.md || echo "- Initial release" >> CHANGELOG.md
|
||||
echo "" >> CHANGELOG.md
|
||||
echo "## Docker Images" >> CHANGELOG.md
|
||||
echo "- \`${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.version.outputs.version }}\`" >> CHANGELOG.md
|
||||
echo "- \`${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest\`" >> CHANGELOG.md
|
||||
|
||||
- name: Create Release
|
||||
uses: actions/create-release@v1
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||
with:
|
||||
tag_name: ${{ steps.version.outputs.version }}
|
||||
release_name: Northern Thailand Ping River Monitor ${{ steps.version.outputs.version }}
|
||||
body_path: CHANGELOG.md
|
||||
draft: false
|
||||
prerelease: false
|
||||
|
||||
# Build and test for release
|
||||
test-release:
|
||||
name: Test Release Build
|
||||
runs-on: ubuntu-latest
|
||||
needs: create-release
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: ["3.9", "3.10", "3.11", "3.12"]
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
token: ${{ secrets.GITEA_TOKEN }}
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip --root-user-action=ignore
|
||||
pip install --root-user-action=ignore -r requirements.txt
|
||||
pip install --root-user-action=ignore -r requirements-dev.txt
|
||||
|
||||
- name: Run full test suite
|
||||
run: |
|
||||
python tests/test_integration.py
|
||||
python tests/test_station_management.py
|
||||
python run.py --test
|
||||
|
||||
- name: Build Python package
|
||||
run: |
|
||||
pip install --root-user-action=ignore build
|
||||
python -m build
|
||||
|
||||
- name: Upload Python package
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: python-package-${{ matrix.python-version }}
|
||||
path: dist/
|
||||
|
||||
# Build release Docker images
|
||||
build-release:
|
||||
name: Build Release Images
|
||||
runs-on: ubuntu-latest
|
||||
needs: [create-release, test-release]
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
token: ${{ secrets.GITEA_TOKEN }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ vars.WORKER_USERNAME}}
|
||||
password: ${{ secrets.CI_BOT_TOKEN }}
|
||||
|
||||
- name: Build and push release images
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: true
|
||||
tags: |
|
||||
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.create-release.outputs.version }}
|
||||
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
|
||||
labels: |
|
||||
org.opencontainers.image.title=Northern Thailand Ping River Monitor
|
||||
org.opencontainers.image.description=Real-time water level monitoring for Ping River Basin
|
||||
org.opencontainers.image.version=${{ needs.create-release.outputs.version }}
|
||||
org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
|
||||
org.opencontainers.image.revision=${{ github.sha }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||
|
||||
# Security scan for release
|
||||
security-scan:
|
||||
name: Security Scan
|
||||
runs-on: ubuntu-latest
|
||||
needs: build-release
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
token: ${{ secrets.GITEA_TOKEN}}
|
||||
|
||||
# Test release deployment locally
|
||||
deploy-release:
|
||||
name: Test Release Deployment
|
||||
runs-on: ubuntu-latest
|
||||
needs: [create-release, build-release, security-scan]
|
||||
environment:
|
||||
name: testing
|
||||
url: http://localhost:8080
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
token: ${{ secrets.GITEA_TOKEN }}
|
||||
|
||||
- name: Log in to Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ vars.WORKER_USERNAME}}
|
||||
password: ${{ secrets.CI_BOT_TOKEN }}
|
||||
|
||||
- name: Deploy to production (Local Test)
|
||||
run: |
|
||||
set -euo pipefail
|
||||
echo "🚀 Testing ${{ needs.create-release.outputs.version }} deployment locally..."
|
||||
|
||||
# Create a dedicated network so we can resolve by container name
|
||||
docker network create ci_net || true
|
||||
|
||||
# Pull the built image
|
||||
docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.create-release.outputs.version }}
|
||||
|
||||
# Stop & remove any existing container
|
||||
docker rm -f ping-river-monitor-test 2>/dev/null || true
|
||||
|
||||
# Start the container on the user-defined network
|
||||
docker run -d \
|
||||
--name ping-river-monitor-test \
|
||||
--network ci_net \
|
||||
-p 8080:8000 \
|
||||
-e LOG_LEVEL=INFO \
|
||||
-e DB_TYPE=sqlite \
|
||||
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.create-release.outputs.version }}
|
||||
|
||||
echo "✅ Container started for testing"
|
||||
|
||||
- name: Health check after deployment
|
||||
run: |
|
||||
set -euo pipefail
|
||||
echo "⏳ Waiting for application to start..."
|
||||
|
||||
# Pull a curl-only image for probing (keeps your app image slim)
|
||||
docker pull curlimages/curl:8.10.1
|
||||
|
||||
# Helper: curl via a sibling container on the SAME Docker network
|
||||
probe() {
|
||||
local url="$1"
|
||||
docker run --rm --network ci_net curlimages/curl:8.10.1 \
|
||||
-sS --max-time 5 --connect-timeout 3 -w "HTTP_CODE:%{http_code}" "$url" || true
|
||||
}
|
||||
|
||||
# Wait for /health (up to ~3m 45s)
|
||||
for i in {1..15}; do
|
||||
echo "🔍 Attempt $i/15: checking http://ping-river-monitor-test:8000/health"
|
||||
resp="$(probe http://ping-river-monitor-test:8000/health)"
|
||||
code="$(echo "$resp" | sed -n 's/.*HTTP_CODE:\([0-9]\+\).*/\1/p')"
|
||||
body="$(echo "$resp" | sed 's/HTTP_CODE:[0-9]*$//')"
|
||||
|
||||
echo "HTTP: ${code:-<none>} | Body: ${body:-<empty>}"
|
||||
|
||||
if [ "${code:-}" = "200" ] && [ -n "${body:-}" ]; then
|
||||
echo "✅ Health endpoint responding successfully"
|
||||
break
|
||||
fi
|
||||
|
||||
echo "❌ Not ready yet. Showing recent logs…"
|
||||
docker logs --tail 20 ping-river-monitor-test || true
|
||||
sleep 15
|
||||
|
||||
if [ "$i" -eq 15 ]; then
|
||||
echo "❌ Health never reached 200. Failing."
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo "🧪 Testing API endpoints…"
|
||||
endpoints=("health" "docs" "stations" "metrics")
|
||||
for ep in "${endpoints[@]}"; do
|
||||
url="http://ping-river-monitor-test:8000/$ep"
|
||||
resp="$(probe "$url")"
|
||||
code="$(echo "$resp" | sed -n 's/.*HTTP_CODE:\([0-9]\+\).*/\1/p')"
|
||||
|
||||
if [ "${code:-}" = "200" ]; then
|
||||
echo "✅ /$ep: OK"
|
||||
else
|
||||
echo "❌ /$ep: FAILED (HTTP ${code:-<none>})"
|
||||
echo "Response: $(echo "$resp" | sed 's/HTTP_CODE:[0-9]*$//')"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo "✅ All health checks passed!"
|
||||
|
||||
- name: Container logs and cleanup
|
||||
if: always()
|
||||
run: |
|
||||
echo "📋 Container logs:"
|
||||
docker logs ping-river-monitor-test || true
|
||||
|
||||
echo "🧹 Cleaning up test container..."
|
||||
docker stop ping-river-monitor-test || true
|
||||
docker rm ping-river-monitor-test || true
|
||||
|
||||
echo "📊 Deployment Test Summary:"
|
||||
echo "Version: ${{ needs.create-release.outputs.version }}"
|
||||
echo "Image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.create-release.outputs.version }}"
|
||||
echo "Status: Container tested successfully"
|
||||
echo "Ready for production deployment"
|
||||
|
||||
# Notify stakeholders
|
||||
notify:
|
||||
name: Notify Release
|
||||
runs-on: ubuntu-latest
|
||||
needs: [create-release, deploy-release]
|
||||
if: always()
|
||||
|
||||
steps:
|
||||
- name: Notify success
|
||||
if: needs.deploy-release.result == 'success'
|
||||
run: |
|
||||
echo "🎉 Release ${{ needs.create-release.outputs.version }} tested successfully!"
|
||||
echo "🧪 Local Test: Passed all health checks"
|
||||
echo "� GDocker Image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.create-release.outputs.version }}"
|
||||
echo "✅ Ready for production deployment"
|
||||
|
||||
# Add notification to Slack, Discord, email, etc.
|
||||
# curl -X POST -H 'Content-type: application/json' \
|
||||
# --data '{"text":"🎉 Northern Thailand Ping River Monitor ${{ needs.create-release.outputs.version }} tested and ready for deployment!"}' \
|
||||
# ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
|
||||
- name: Notify failure
|
||||
if: needs.deploy-release.result == 'failure'
|
||||
run: |
|
||||
echo "❌ Release ${{ needs.create-release.outputs.version }} testing failed!"
|
||||
echo "Please check the logs and fix issues before production deployment."
|
||||
|
||||
# Add failure notification
|
||||
# curl -X POST -H 'Content-type: application/json' \
|
||||
# --data '{"text":"❌ Northern Thailand Ping River Monitor ${{ needs.create-release.outputs.version }} testing failed!"}' \
|
||||
# ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
@@ -5,16 +5,20 @@ Guidance for AI coding agents working in this repository.
|
||||
## What this is
|
||||
|
||||
Flood monitoring and forecasting for the Ping River, Chiang Mai. Public dashboard and
|
||||
API at https://water.buildfor.life/ (never publish the server's private/Tailscale IP).
|
||||
Production: one systemd unit on a small VPS, `/opt/thailand-water-monitor`, user
|
||||
`water-monitor`, interpreter `.venv/bin/python` (uv-managed), updated by `git pull`.
|
||||
API at https://water.buildfor.life/ (never publish the server's private/mesh IP).
|
||||
Production: one systemd unit, `ping-collector.service` (the Rust binary in `rust/`),
|
||||
on a small VPS in `/opt/thailand-water-monitor` (a git checkout), user `water-monitor`;
|
||||
plus the monthly `water-monitor-retrain.timer` (Python, `.venv`, uv-managed). Updated by
|
||||
`git pull`, `cargo build --release`, `sudo bash scripts/install.sh`.
|
||||
|
||||
## Rules
|
||||
|
||||
- Python 3.11 only. `uv sync --python 3.11`; run everything as `uv run ...`.
|
||||
- `make format` (black 88 / isort black profile, config in pyproject.toml) before
|
||||
committing; CI fails on formatting. `make test` must stay green — tests are
|
||||
synthetic-data only, never add one that needs the DB or network.
|
||||
- `make format` (black 88 / isort black profile, config in pyproject.toml; cargo fmt)
|
||||
before committing; CI fails on formatting. `make test` must stay green. Tests use
|
||||
synthetic data, never the network; database tests (both languages) run against a
|
||||
throwaway PostgreSQL schema from `TEST_DATABASE_URL` and skip without it.
|
||||
- PostgreSQL only. No other database backends.
|
||||
- Timestamps everywhere are Asia/Bangkok wall-clock with no offset. The dashboard
|
||||
parses them with `parseTs()` and renders with `timeZone: TZ`; keep it that way.
|
||||
- Model changes go through the rolling-origin harness (`scripts/evaluate_variants.py`)
|
||||
@@ -26,38 +30,45 @@ Production: one systemd unit on a small VPS, `/opt/thailand-water-monitor`, user
|
||||
- No `git add -A`: zero-byte shell-accident files (`#`, `$(wc`, ...) have been committed
|
||||
before. Stage files by name.
|
||||
- Do not add Co-Authored-By trailers.
|
||||
- The dashboard is a single file, `src/static/dashboard.html`, EN + TH via the `t()`
|
||||
table: every user-visible string needs both languages.
|
||||
- The dashboard is a single file, `src/static/dashboard.html` (design and rules in
|
||||
docs/DASHBOARD_REDESIGN.md), EN + TH via the `t()` table: every user-visible string
|
||||
needs both languages, Thai is the default, no emoji. Every path it fetches must be a
|
||||
route in `rust/collector/src/web/mod.rs` (tests/test_dashboard.py checks). A replay or
|
||||
a simulation must never be shown as live data. Preview it against production with
|
||||
`uv run python scripts/dev_proxy.py`; the server reads it at start (restart).
|
||||
- Rust (`rust/`, see rust/README.md): `cargo fmt --all`, `cargo clippy --all-targets
|
||||
-- -D warnings`, `cargo test` (database tests need `TEST_DATABASE_URL`). The Rust
|
||||
collector writes the same tables as the Python collectors: a change to a table,
|
||||
upsert or parser on one side must be made on the other until the Python side is
|
||||
retired (check with a live run of both into two databases).
|
||||
- The forecast is served by Rust (`rust/forecast`) from models trained in Python.
|
||||
A change to `src/ml/features.py`, `predict.py`, the HII fill or the dam frame
|
||||
must be ported in the same change: regenerate
|
||||
`rust/forecast/testdata/gen_forecast_fixtures.py` and keep
|
||||
`cargo test -p ping-forecast` green. Changing thresholds, stages, leads or
|
||||
notify constants: `uv run python -m src.ml.export --config` (a test fails
|
||||
otherwise) and rebuild the collector. Notify texts:
|
||||
`rust/collector/testdata/gen_notify_cases.py`.
|
||||
- API routes live in `rust/collector/src/web` (phase 3). Until the FastAPI app is
|
||||
removed, a route change goes into both, and `scripts/api_parity.py` (both
|
||||
servers on one database) must report no differences. A route's parameters are
|
||||
declared once (`web/params.rs`); that list validates and documents (/docs).
|
||||
-- -D warnings`, `cargo test`; minimum version 1.88
|
||||
(`CARGO_TARGET_DIR=target/msrv cargo +1.88 check --all-targets --locked`). Read
|
||||
`.env` only through `rust/collector/src/envfile.rs` (python-dotenv semantics, never
|
||||
dotenvy: the Python tools read the same file).
|
||||
- The forecast is served by Rust (`rust/forecast`) from models trained in Python;
|
||||
`src/ml/predict.py` is the reference it is tested against. A change to
|
||||
`src/ml/features.py`, `predict.py`, the HII fill or the dam frame must be ported in
|
||||
the same change: regenerate `rust/forecast/testdata/gen_forecast_fixtures.py` and keep
|
||||
`cargo test -p ping-forecast` green. Changing thresholds, stages, leads or the
|
||||
notifier settings (`src/ml/export.py`): `uv run python -m src.ml.export --config`
|
||||
(a test fails otherwise) and rebuild the collector.
|
||||
- Notifier texts are pinned by `rust/collector/testdata/notify_python_cases.json.gz`;
|
||||
a deliberate wording change updates that fixture in the same commit.
|
||||
- API routes live in `rust/collector/src/web`. A route's parameters are declared once
|
||||
(`web/params.rs`); that list validates and documents (/docs). Keep response formats
|
||||
stable: external clients use them.
|
||||
- The Python backfill stores (`src/hii_collector.py`, `src/rid_reservoir.py`,
|
||||
`src/ml/rain.py`) write the same tables as `rust/collector/src/db.rs`: a table or
|
||||
upsert change goes into both.
|
||||
|
||||
## Where things are
|
||||
|
||||
- `src/web_api.py` FastAPI app; `src/water_scraper_v3.py` RID collector;
|
||||
`src/hii_collector.py` ThaiWater/HII; `src/ml/` features/train/evaluate/predict,
|
||||
`rain.py` (Open-Meteo), `dam.py`, `hii_rain.py`.
|
||||
- `scripts/retrain.sh` + `water-monitor-retrain.timer`: monthly retrain with staged
|
||||
promote. `scripts/dev_proxy.py`: serve the working-copy dashboard against the live API.
|
||||
- `rust/collector` (`ping-collector`): Rust data collector; with it deployed the
|
||||
Python app runs `COLLECTION_ENABLED=false`. With `FORECAST_IN_COLLECTOR=true` it
|
||||
also forecasts (`rust/forecast`) and sends ntfy + Matrix; with `WEB_LISTEN` it
|
||||
serves the website and API (`rust/collector/src/web`, the port of
|
||||
`src/web_api.py`; the dashboard stays `src/static/dashboard.html`). Rewrite
|
||||
plan: `docs/RUST_REWRITE.md`.
|
||||
- `rust/collector` (`ping-collector`): collection (`rid.rs`, `hii.rs`, `dams.rs`,
|
||||
`rain.rs`), forecast run (`forecasting.rs`), ntfy/Matrix (`notify.rs`), website/API
|
||||
(`web/`), tables (`db.rs`). `rust/forecast`: inference.
|
||||
- `src/ml/`: features/train/evaluate/predict/export, `rain.py` (Open-Meteo), `dam.py`,
|
||||
`hii_rain.py`. `src/hii_backfill.py`, `src/hii_collector.py`, `src/rid_reservoir.py`:
|
||||
archive backfills (`scripts/backfill_*.py`).
|
||||
- `scripts/install.sh`: install/update on the server. `scripts/retrain.sh` +
|
||||
`water-monitor-retrain.timer`: monthly retrain with staged promote.
|
||||
`scripts/forecast_parity.py`: Rust vs Python forecast on a live database.
|
||||
`scripts/dev_proxy.py`: serve the working-copy dashboard against the live API.
|
||||
- `docs/FLOOD_FORECASTING.md` is the authoritative model write-up; `docs/DATA_SOURCES.md`
|
||||
the source catalog.
|
||||
the source catalog; `docs/NOTIFICATIONS.md` the alerts; `docs/RUST_REWRITE.md` the
|
||||
history of the Python-to-Rust move.
|
||||
+43
-268
@@ -1,277 +1,52 @@
|
||||
# Contributing to Northern Thailand Ping River Monitor
|
||||
# Contributing
|
||||
|
||||
Thank you for your interest in contributing to the Northern Thailand Ping River Monitor! This document provides guidelines and information for contributors.
|
||||
Issues and merge requests on
|
||||
[git.b4l.co.th](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor).
|
||||
The repository has two halves: the service in Rust (`rust/`, see
|
||||
[rust/README.md](rust/README.md)) and the model training/evaluation stack in Python
|
||||
(`src/ml`, see [docs/FLOOD_FORECASTING.md](docs/FLOOD_FORECASTING.md)).
|
||||
|
||||
## 🚀 Getting Started
|
||||
## Setup
|
||||
|
||||
### Prerequisites
|
||||
- Python 3.9 or higher
|
||||
- Git
|
||||
- Basic knowledge of water monitoring systems
|
||||
- Familiarity with FastAPI and time-series databases (optional)
|
||||
- Python 3.11 and [uv](https://docs.astral.sh/uv/) (3.12+ has no pandas 2.0.3 wheels)
|
||||
- Rust >= 1.88 from [rustup](https://rustup.rs), plus `pkg-config` and `libssl-dev` on Linux
|
||||
- Docker (or any PostgreSQL 16) for the database tests
|
||||
|
||||
### Development Setup
|
||||
```bash
|
||||
git clone https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor.git
|
||||
cd Northern-Thailand-Ping-River-Monitor
|
||||
make install # uv sync + pre-commit hooks
|
||||
docker run -d --name ping-pg -e POSTGRES_PASSWORD=ping -e POSTGRES_DB=ping -p 55432:5432 postgres:16
|
||||
export TEST_DATABASE_URL=postgres://postgres:ping@localhost:55432/ping
|
||||
make check # format check, flake8, clippy, pytest, cargo test, Rust 1.88
|
||||
```
|
||||
|
||||
1. **Clone the repository:**
|
||||
```bash
|
||||
git clone https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor.git
|
||||
cd Northern-Thailand-Ping-River-Monitor
|
||||
```
|
||||
Without `TEST_DATABASE_URL` the database tests are skipped, the rest still run.
|
||||
|
||||
2. **Create a virtual environment:**
|
||||
```bash
|
||||
python -m venv venv
|
||||
source venv/bin/activate # On Windows: venv\Scripts\activate
|
||||
```
|
||||
## Before a merge request
|
||||
|
||||
3. **Install dependencies:**
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
- `make format`, then `make check` green. CI runs the same (`.gitea/workflows/`).
|
||||
- One logical change per commit, imperative subject (`fix(collector): ...`,
|
||||
`feat(web): ...`, `docs: ...`).
|
||||
- Dashboard changes (`src/static/dashboard.html`): every visible string in English
|
||||
and Thai (`t()` table), no emoji; screenshots (light and dark, phone and desktop)
|
||||
in the merge request. Preview with `uv run python scripts/dev_proxy.py`.
|
||||
- API changes: keep existing fields and formats; external clients read them. New
|
||||
parameters go into the route's declaration in `rust/collector/src/web/params.rs`,
|
||||
which also documents them at `/docs`.
|
||||
- Model changes: evaluate with `scripts/evaluate_variants.py` (rolling origin, judged
|
||||
on first-alert lead and false alarms) and record the result in
|
||||
`docs/FLOOD_FORECASTING.md` section 5, whether it helped or not. A feature change
|
||||
must be ported to `rust/forecast` in the same change (see CLAUDE.md for the
|
||||
fixture and config regeneration steps).
|
||||
- Never commit credentials, `.env`, trained models (`models/*.joblib`,
|
||||
`models/*.json.gz`) or the server's private addresses.
|
||||
|
||||
4. **Run tests:**
|
||||
```bash
|
||||
python test_integration.py
|
||||
```
|
||||
## Useful commands
|
||||
|
||||
5. **Start the development server:**
|
||||
```bash
|
||||
python run.py --web-api
|
||||
```
|
||||
|
||||
## 🔧 Development Guidelines
|
||||
|
||||
### Code Style
|
||||
- Follow PEP 8 Python style guidelines
|
||||
- Use type hints for all function parameters and return values
|
||||
- Write descriptive docstrings for all classes and functions
|
||||
- Keep functions focused and under 50 lines when possible
|
||||
|
||||
### Testing
|
||||
- Write tests for new features
|
||||
- Ensure all existing tests pass before submitting
|
||||
- Test both success and error cases
|
||||
- Include integration tests for API endpoints
|
||||
|
||||
### Documentation
|
||||
- Update README.md for new features
|
||||
- Add docstrings to all new functions and classes
|
||||
- Update API documentation for new endpoints
|
||||
- Include examples in documentation
|
||||
|
||||
## 📝 Contribution Process
|
||||
|
||||
### 1. Issue Creation
|
||||
- Check existing issues before creating new ones
|
||||
- Use clear, descriptive titles
|
||||
- Provide detailed descriptions with steps to reproduce (for bugs)
|
||||
- Include system information and logs when relevant
|
||||
|
||||
### 2. Pull Request Process
|
||||
1. **Fork the repository**
|
||||
2. **Create a feature branch:**
|
||||
```bash
|
||||
git checkout -b feature/your-feature-name
|
||||
```
|
||||
3. **Make your changes**
|
||||
4. **Test thoroughly:**
|
||||
```bash
|
||||
python test_integration.py
|
||||
python test_station_management.py
|
||||
```
|
||||
5. **Commit with clear messages:**
|
||||
```bash
|
||||
git commit -m "Add: New station validation feature"
|
||||
```
|
||||
6. **Push to your fork:**
|
||||
```bash
|
||||
git push origin feature/your-feature-name
|
||||
```
|
||||
7. **Create a pull request**
|
||||
|
||||
### Pull Request Guidelines
|
||||
- Use clear, descriptive titles
|
||||
- Provide detailed descriptions of changes
|
||||
- Reference related issues
|
||||
- Include screenshots for UI changes
|
||||
- Ensure all tests pass
|
||||
- Update documentation as needed
|
||||
|
||||
## 🎯 Areas for Contribution
|
||||
|
||||
### High Priority
|
||||
- **Additional Database Support** - MongoDB, TimescaleDB
|
||||
- **Authentication System** - User management and API keys
|
||||
- **Real-time Alerts** - Email/SMS notifications for critical conditions
|
||||
- **Mobile App** - React Native or Flutter interface
|
||||
- **Advanced Analytics** - Trend analysis and forecasting
|
||||
|
||||
### Medium Priority
|
||||
- **Bulk Station Operations** - Import/export multiple stations
|
||||
- **Map Interface** - Visual station management
|
||||
- **Data Export** - CSV, Excel format support
|
||||
- **Performance Optimization** - Query optimization and caching
|
||||
- **Internationalization** - Multi-language support
|
||||
|
||||
### Low Priority
|
||||
- **Additional Visualizations** - Custom chart types
|
||||
- **Report Generation** - Automated PDF reports
|
||||
- **Integration APIs** - Third-party service connections
|
||||
- **Advanced Logging** - Structured logging with ELK stack
|
||||
|
||||
## 🐛 Bug Reports
|
||||
|
||||
When reporting bugs, please include:
|
||||
|
||||
1. **Environment Information:**
|
||||
- Operating system and version
|
||||
- Python version
|
||||
- Database type and version
|
||||
- Browser (for web interface issues)
|
||||
|
||||
2. **Steps to Reproduce:**
|
||||
- Clear, numbered steps
|
||||
- Expected vs actual behavior
|
||||
- Screenshots or error messages
|
||||
|
||||
3. **Additional Context:**
|
||||
- Log files (sanitized of sensitive data)
|
||||
- Configuration details
|
||||
- Recent changes to the system
|
||||
|
||||
## 💡 Feature Requests
|
||||
|
||||
For feature requests, please provide:
|
||||
|
||||
1. **Use Case:** Why is this feature needed?
|
||||
2. **Description:** What should the feature do?
|
||||
3. **Acceptance Criteria:** How do we know it's complete?
|
||||
4. **Priority:** How important is this feature?
|
||||
5. **Alternatives:** Are there workarounds available?
|
||||
|
||||
## 🔒 Security
|
||||
|
||||
### Reporting Security Issues
|
||||
- **DO NOT** create public issues for security vulnerabilities
|
||||
- Email security issues to: [security contact]
|
||||
- Include detailed information about the vulnerability
|
||||
- Allow time for fixes before public disclosure
|
||||
|
||||
### Security Guidelines
|
||||
- Never commit sensitive data (passwords, API keys)
|
||||
- Use environment variables for configuration
|
||||
- Validate all user inputs
|
||||
- Follow OWASP security guidelines
|
||||
- Keep dependencies updated
|
||||
|
||||
## 📊 Database Contributions
|
||||
|
||||
### Adding New Database Support
|
||||
1. Create adapter in `src/database_adapters.py`
|
||||
2. Implement all required methods
|
||||
3. Add configuration options in `src/config.py`
|
||||
4. Update documentation
|
||||
5. Add integration tests
|
||||
|
||||
### Database Schema Changes
|
||||
- Provide migration scripts
|
||||
- Test with existing data
|
||||
- Document breaking changes
|
||||
- Update all database adapters
|
||||
|
||||
## 🌐 API Contributions
|
||||
|
||||
### Adding New Endpoints
|
||||
1. Define Pydantic models in `src/models.py`
|
||||
2. Implement endpoint in `src/web_api.py`
|
||||
3. Add input validation
|
||||
4. Include error handling
|
||||
5. Write comprehensive tests
|
||||
6. Update API documentation
|
||||
|
||||
### API Guidelines
|
||||
- Use RESTful conventions
|
||||
- Include proper HTTP status codes
|
||||
- Provide clear error messages
|
||||
- Support pagination for large datasets
|
||||
- Include rate limiting considerations
|
||||
|
||||
## 📈 Performance Guidelines
|
||||
|
||||
### Optimization Areas
|
||||
- Database query efficiency
|
||||
- API response times
|
||||
- Memory usage
|
||||
- Concurrent request handling
|
||||
- Data processing speed
|
||||
|
||||
### Monitoring
|
||||
- Use built-in metrics collection
|
||||
- Monitor response times
|
||||
- Track error rates
|
||||
- Watch resource usage
|
||||
- Set up alerts for issues
|
||||
|
||||
## 🧪 Testing Guidelines
|
||||
|
||||
### Test Types
|
||||
- **Unit Tests:** Individual function testing
|
||||
- **Integration Tests:** Component interaction testing
|
||||
- **API Tests:** Endpoint functionality testing
|
||||
- **Performance Tests:** Load and stress testing
|
||||
|
||||
### Test Coverage
|
||||
- Aim for >80% code coverage
|
||||
- Test both success and failure paths
|
||||
- Include edge cases
|
||||
- Test with different database backends
|
||||
|
||||
## 📚 Documentation Standards
|
||||
|
||||
### Code Documentation
|
||||
- Use clear, descriptive variable names
|
||||
- Write comprehensive docstrings
|
||||
- Include type hints
|
||||
- Add inline comments for complex logic
|
||||
|
||||
### User Documentation
|
||||
- Keep README.md updated
|
||||
- Provide clear installation instructions
|
||||
- Include usage examples
|
||||
- Document configuration options
|
||||
|
||||
## 🤝 Community Guidelines
|
||||
|
||||
### Communication
|
||||
- Be respectful and professional
|
||||
- Help newcomers get started
|
||||
- Share knowledge and experiences
|
||||
- Provide constructive feedback
|
||||
|
||||
### Code Reviews
|
||||
- Review code thoroughly
|
||||
- Provide specific, actionable feedback
|
||||
- Be open to suggestions
|
||||
- Focus on code quality and maintainability
|
||||
|
||||
## 📞 Getting Help
|
||||
|
||||
### Resources
|
||||
- **Documentation:** README.md and docs/
|
||||
- **API Reference:** http://localhost:8000/docs
|
||||
- **Issues:** GitHub Issues section
|
||||
- **Discussions:** GitHub Discussions
|
||||
|
||||
### Contact
|
||||
- **General Questions:** Create a GitHub issue
|
||||
- **Security Issues:** [security contact]
|
||||
- **Maintainer:** [maintainer contact]
|
||||
|
||||
## 🎉 Recognition
|
||||
|
||||
Contributors will be recognized in:
|
||||
- README.md contributors section
|
||||
- Release notes for significant contributions
|
||||
- Special recognition for major features
|
||||
|
||||
Thank you for contributing to the Northern Thailand Ping River Monitor! Your contributions help improve water resource monitoring and management in the region.
|
||||
```bash
|
||||
make build && make serve # the website/API from your database on 127.0.0.1:8001
|
||||
uv run python scripts/dev_proxy.py # the working-copy dashboard against the live API
|
||||
uv run python scripts/evaluate_variants.py --stations P.1
|
||||
uv run python -m src.ml.export --config # after changing thresholds or notifier settings
|
||||
```
|
||||
-58
@@ -1,58 +0,0 @@
|
||||
# Multi-stage Docker build for Northern Thailand Ping River Monitor
|
||||
|
||||
# Build stage
|
||||
FROM python:3.11-slim as builder
|
||||
|
||||
# Set working directory
|
||||
WORKDIR /app
|
||||
|
||||
# Install build dependencies
|
||||
RUN apt-get update && apt-get install -y \
|
||||
build-essential \
|
||||
gcc \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Copy requirements and install Python dependencies
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir --user -r requirements.txt
|
||||
|
||||
# Production stage
|
||||
FROM python:3.11-slim
|
||||
|
||||
# Set working directory
|
||||
WORKDIR /app
|
||||
|
||||
# Install runtime dependencies and create user
|
||||
RUN apt-get update && apt-get install -y \
|
||||
wget \
|
||||
curl \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& groupadd -r appuser && useradd -r -g appuser appuser \
|
||||
&& mkdir -p /home/appuser/.local
|
||||
|
||||
# Copy Python packages from builder stage
|
||||
COPY --from=builder /root/.local /home/appuser/.local
|
||||
|
||||
# Copy application code
|
||||
COPY . .
|
||||
|
||||
# Create logs directory and set permissions
|
||||
RUN mkdir -p logs && chown -R appuser:appuser /app /home/appuser/.local
|
||||
|
||||
# Set environment variables
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
ENV TZ=Asia/Bangkok
|
||||
ENV PATH=/home/appuser/.local/bin:$PATH
|
||||
|
||||
# Switch to non-root user
|
||||
USER appuser
|
||||
|
||||
# Health check for API mode
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
|
||||
CMD python -c "import requests; requests.get('http://localhost:8000/health', timeout=5)" 2>/dev/null || exit 1
|
||||
|
||||
# Expose port for API
|
||||
EXPOSE 8000
|
||||
|
||||
# Default command
|
||||
CMD ["python", "run.py", "--web-api"]
|
||||
@@ -1,165 +0,0 @@
|
||||
# Migration to uv
|
||||
|
||||
This document describes the migration from traditional Python package management (pip + requirements.txt) to [uv](https://docs.astral.sh/uv/), a fast Python package installer and resolver.
|
||||
|
||||
## What Changed
|
||||
|
||||
### Files Added
|
||||
- `pyproject.toml` - Modern Python project configuration combining dependencies and metadata
|
||||
- `.python-version` - Specifies Python version for uv
|
||||
- `scripts/setup_uv.sh` - Unix setup script for uv environment
|
||||
- `scripts/setup_uv.bat` - Windows setup script for uv environment
|
||||
- This migration guide
|
||||
|
||||
### Files Modified
|
||||
- `Makefile` - Updated all commands to use `uv run` instead of direct Python execution
|
||||
|
||||
### Files That Can Be Removed (Optional)
|
||||
- `requirements.txt` - Dependencies now in pyproject.toml
|
||||
- `requirements-dev.txt` - Dev dependencies now in pyproject.toml
|
||||
- `setup.py` - Configuration now in pyproject.toml
|
||||
|
||||
## Installation
|
||||
|
||||
### Install uv
|
||||
|
||||
**Unix/macOS:**
|
||||
```bash
|
||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
```
|
||||
|
||||
**Windows (PowerShell):**
|
||||
```powershell
|
||||
powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"
|
||||
```
|
||||
|
||||
### Setup Project
|
||||
|
||||
**Unix/macOS:**
|
||||
```bash
|
||||
# Run the setup script
|
||||
chmod +x scripts/setup_uv.sh
|
||||
./scripts/setup_uv.sh
|
||||
|
||||
# Or manually:
|
||||
uv sync
|
||||
uv run pre-commit install
|
||||
```
|
||||
|
||||
**Windows:**
|
||||
```batch
|
||||
REM Run the setup script
|
||||
scripts\setup_uv.bat
|
||||
|
||||
REM Or manually:
|
||||
uv sync
|
||||
uv run pre-commit install
|
||||
```
|
||||
|
||||
## New Workflow
|
||||
|
||||
### Common Commands
|
||||
|
||||
| Old Command | New Command | Description |
|
||||
|-------------|-------------|-------------|
|
||||
| `pip install -r requirements.txt` | `uv sync --no-dev` | Install production dependencies |
|
||||
| `pip install -r requirements-dev.txt` | `uv sync` | Install all dependencies (including dev) |
|
||||
| `python run.py` | `uv run python run.py` | Run the application |
|
||||
| `pytest` | `uv run pytest` | Run tests |
|
||||
| `black src/` | `uv run black src/` | Format code |
|
||||
|
||||
### Using the Makefile
|
||||
|
||||
The Makefile has been updated to use uv, so all existing commands work the same:
|
||||
|
||||
```bash
|
||||
make install-dev # Install dev dependencies with uv
|
||||
make test # Run tests with uv
|
||||
make run-api # Start API server with uv
|
||||
make lint # Lint code with uv
|
||||
make format # Format code with uv
|
||||
```
|
||||
|
||||
### Adding Dependencies
|
||||
|
||||
**Production dependency:**
|
||||
```bash
|
||||
uv add requests
|
||||
```
|
||||
|
||||
**Development dependency:**
|
||||
```bash
|
||||
uv add --dev pytest
|
||||
```
|
||||
|
||||
**Specific version:**
|
||||
```bash
|
||||
uv add "fastapi==0.104.1"
|
||||
```
|
||||
|
||||
### Managing Python Versions
|
||||
|
||||
uv can automatically manage Python versions:
|
||||
|
||||
```bash
|
||||
# Install and use Python 3.11
|
||||
uv python install 3.11
|
||||
uv sync
|
||||
|
||||
# Use specific Python version
|
||||
uv sync --python 3.11
|
||||
```
|
||||
|
||||
## Benefits of uv
|
||||
|
||||
1. **Speed** - 10-100x faster than pip
|
||||
2. **Reliability** - Better dependency resolution
|
||||
3. **Simplicity** - Single tool for packages and Python versions
|
||||
4. **Reproducibility** - Lock file ensures consistent environments
|
||||
5. **Modern** - Built-in support for pyproject.toml
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Command not found
|
||||
Make sure uv is in your PATH after installation. Restart your terminal or run:
|
||||
```bash
|
||||
source ~/.bashrc # or ~/.zshrc
|
||||
```
|
||||
|
||||
### Lock file conflicts
|
||||
If you encounter lock file issues:
|
||||
```bash
|
||||
rm uv.lock
|
||||
uv sync
|
||||
```
|
||||
|
||||
### Python version issues
|
||||
Ensure the Python version in `.python-version` is available:
|
||||
```bash
|
||||
uv python list
|
||||
uv python install 3.11 # if needed
|
||||
```
|
||||
|
||||
## Rollback (if needed)
|
||||
|
||||
If you need to rollback to the old system:
|
||||
|
||||
1. Use the original requirements files:
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
pip install -r requirements-dev.txt
|
||||
```
|
||||
|
||||
2. Revert the Makefile changes to use `python` instead of `uv run python`
|
||||
|
||||
3. Remove uv-specific files:
|
||||
```bash
|
||||
rm pyproject.toml .python-version uv.lock
|
||||
rm -rf .venv # if created by uv
|
||||
```
|
||||
|
||||
## Additional Resources
|
||||
|
||||
- [uv Documentation](https://docs.astral.sh/uv/)
|
||||
- [Migration Guide](https://docs.astral.sh/uv/guides/projects/)
|
||||
- [pyproject.toml Reference](https://packaging.python.org/en/latest/specifications/pyproject-toml/)
|
||||
@@ -1,181 +1,70 @@
|
||||
# Northern Thailand Ping River Monitor - Makefile
|
||||
# Ping River Monitor. The service is Rust (rust/, ping-collector); Python is
|
||||
# the model training/evaluation stack and the archive backfills (src/).
|
||||
# Database tests (both sides) need TEST_DATABASE_URL, e.g.
|
||||
# docker run -d --name ping-pg -e POSTGRES_PASSWORD=ping -e POSTGRES_DB=ping -p 55432:5432 postgres:16
|
||||
# export TEST_DATABASE_URL=postgres://postgres:ping@localhost:55432/ping
|
||||
|
||||
.PHONY: help install install-dev test test-coverage lint format clean run run-api docker-build docker-run docs
|
||||
PY_SRC := src/ scripts/ tests/ rust/forecast/testdata rust/collector/testdata
|
||||
CARGO := cargo
|
||||
MANIFEST := --manifest-path rust/Cargo.toml
|
||||
|
||||
.PHONY: help install format lint test test-py test-rust build check serve train evaluate export clean
|
||||
|
||||
# Default target
|
||||
help:
|
||||
@echo "Northern Thailand Ping River Monitor - Available Commands:"
|
||||
@echo ""
|
||||
@echo "Setup:"
|
||||
@echo " install Install production dependencies"
|
||||
@echo " install-dev Install development dependencies"
|
||||
@echo ""
|
||||
@echo "Development:"
|
||||
@echo " test Run all tests"
|
||||
@echo " test-cov Run tests with coverage report"
|
||||
@echo " lint Run code linting"
|
||||
@echo " format Format code with black and isort"
|
||||
@echo " clean Clean up temporary files"
|
||||
@echo ""
|
||||
@echo "Running:"
|
||||
@echo " run Run the monitor in continuous mode"
|
||||
@echo " run-api Run the web API server"
|
||||
@echo " run-test Run a single test cycle"
|
||||
@echo " run-status Show system status"
|
||||
@echo ""
|
||||
@echo "Alerting:"
|
||||
@echo " alert-check Check water levels and send alerts"
|
||||
@echo " alert-test Send test Matrix message"
|
||||
@echo ""
|
||||
@echo "Distribution:"
|
||||
@echo " build-exe Build standalone executable"
|
||||
@echo " package Build and create distribution package"
|
||||
@echo ""
|
||||
@echo "Docker:"
|
||||
@echo " docker-build Build Docker image"
|
||||
@echo " docker-run Run with Docker Compose"
|
||||
@echo " docker-stop Stop Docker services"
|
||||
@echo ""
|
||||
@echo "Database:"
|
||||
@echo " setup-postgres Setup PostgreSQL database"
|
||||
@echo " test-postgres Test PostgreSQL connection"
|
||||
@echo " encode-password URL encode password for connection string"
|
||||
@echo " migrate-sqlite Migrate SQLite data to PostgreSQL"
|
||||
@echo " migrate-fast Fast migration with 10K batch size"
|
||||
@echo " analyze-sqlite Analyze SQLite database structure (dry run)"
|
||||
@echo ""
|
||||
@echo "Documentation:"
|
||||
@echo " docs Generate documentation"
|
||||
@echo "make install uv sync (Python 3.11) + pre-commit hooks"
|
||||
@echo "make format black + isort (Python), cargo fmt (Rust)"
|
||||
@echo "make lint the CI checks without tests (format, flake8, clippy)"
|
||||
@echo "make test pytest + cargo test"
|
||||
@echo "make build release build of ping-collector (rust/target/release)"
|
||||
@echo "make check lint + test + the Rust 1.88 minimum-version check"
|
||||
@echo "make serve website/API from the local database on 127.0.0.1:8001"
|
||||
@echo "make train retrain every station's model into models/"
|
||||
@echo "make evaluate rolling-origin harness on P.1 (see scripts/evaluate_variants.py)"
|
||||
@echo "make export export trained models + flood_config.json for the Rust forecaster"
|
||||
|
||||
# Installation
|
||||
install:
|
||||
uv sync --no-dev
|
||||
|
||||
install-dev:
|
||||
uv sync
|
||||
uv sync --python 3.11
|
||||
uv run pre-commit install
|
||||
|
||||
# Testing
|
||||
test:
|
||||
format:
|
||||
uv run black $(PY_SRC)
|
||||
uv run isort src/ scripts/ tests/
|
||||
$(CARGO) fmt --all $(MANIFEST)
|
||||
|
||||
lint:
|
||||
uv run black --check $(PY_SRC)
|
||||
uv run isort --check-only src/ scripts/ tests/
|
||||
uv run flake8 src/ scripts/ tests/ --count --select=E9,F63,F7,F82 --show-source --statistics
|
||||
$(CARGO) fmt --all --check $(MANIFEST)
|
||||
$(CARGO) clippy --all-targets --locked $(MANIFEST) -- -D warnings
|
||||
|
||||
test: test-py test-rust
|
||||
|
||||
test-py:
|
||||
uv run pytest -q
|
||||
|
||||
test-cov:
|
||||
uv run pytest --cov=src --cov-report=html --cov-report=term
|
||||
test-rust:
|
||||
$(CARGO) test --locked $(MANIFEST)
|
||||
|
||||
# Code quality
|
||||
lint:
|
||||
uv run flake8 src/ --max-line-length=100
|
||||
uv run mypy src/
|
||||
build:
|
||||
$(CARGO) build --release --locked $(MANIFEST)
|
||||
|
||||
format:
|
||||
uv run black src/ *.py
|
||||
uv run isort src/ *.py
|
||||
check: lint test
|
||||
CARGO_TARGET_DIR=rust/target/msrv $(CARGO) +1.88 check --all-targets --locked $(MANIFEST)
|
||||
|
||||
serve:
|
||||
rust/target/release/ping-collector serve --listen 127.0.0.1:8001 --no-analytics
|
||||
|
||||
train:
|
||||
uv run python scripts/train_flood_model.py --stations all
|
||||
|
||||
evaluate:
|
||||
uv run python scripts/evaluate_variants.py --stations P.1
|
||||
|
||||
export:
|
||||
uv run python -m src.ml.export
|
||||
uv run python -m src.ml.export --config
|
||||
|
||||
# Cleanup
|
||||
clean:
|
||||
find . -type f -name "*.pyc" -delete
|
||||
find . -type d -name "__pycache__" -delete
|
||||
find . -type f -name "*.log" -delete
|
||||
rm -rf .pytest_cache/
|
||||
rm -rf .mypy_cache/
|
||||
rm -rf htmlcov/
|
||||
rm -rf dist/
|
||||
rm -rf build/
|
||||
rm -rf *.egg-info/
|
||||
|
||||
# Running
|
||||
run:
|
||||
uv run python run.py
|
||||
|
||||
run-api:
|
||||
uv run python run.py --web-api
|
||||
|
||||
run-test:
|
||||
uv run python run.py --test
|
||||
|
||||
run-status:
|
||||
uv run python run.py --status
|
||||
|
||||
# Alerting
|
||||
alert-check:
|
||||
uv run python run.py --alert-check
|
||||
|
||||
alert-test:
|
||||
uv run python run.py --alert-test
|
||||
|
||||
# Docker
|
||||
docker-build:
|
||||
docker build -t ping-river-monitor .
|
||||
|
||||
docker-run:
|
||||
docker-compose -f docker-compose.victoriametrics.yml up -d
|
||||
|
||||
docker-stop:
|
||||
docker-compose -f docker-compose.victoriametrics.yml down
|
||||
|
||||
docker-logs:
|
||||
docker-compose -f docker-compose.victoriametrics.yml logs -f
|
||||
|
||||
# Documentation
|
||||
docs:
|
||||
cd docs && make html
|
||||
|
||||
# Monitoring
|
||||
health-check:
|
||||
curl -f http://localhost:8000/health || exit 1
|
||||
|
||||
metrics:
|
||||
curl -s http://localhost:8000/metrics | jq .
|
||||
|
||||
# Development helpers
|
||||
dev-setup: install-dev
|
||||
cp .env.example .env
|
||||
@echo "Development environment set up!"
|
||||
@echo "Edit .env file with your configuration"
|
||||
|
||||
# Production deployment
|
||||
deploy-check:
|
||||
uv run python run.py --test
|
||||
@echo "Deployment check passed!"
|
||||
|
||||
# Database management
|
||||
setup-postgres:
|
||||
uv run python scripts/setup_postgres.py
|
||||
|
||||
test-postgres:
|
||||
uv run python -c "from scripts.setup_postgres import test_postgres_connection; from src.config import Config; config = Config.get_database_config(); test_postgres_connection(config['connection_string'])"
|
||||
|
||||
migrate-sqlite:
|
||||
uv run python scripts/migrate_sqlite_to_postgres.py
|
||||
|
||||
migrate-fast:
|
||||
uv run python scripts/migrate_sqlite_to_postgres.py --fast
|
||||
|
||||
analyze-sqlite:
|
||||
uv run python scripts/migrate_sqlite_to_postgres.py --dry-run
|
||||
|
||||
# Git helpers
|
||||
git-setup:
|
||||
git remote add origin https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor.git
|
||||
@echo "Git remote configured!"
|
||||
|
||||
# Quick start
|
||||
quick-start: install dev-setup run-test
|
||||
@echo "Quick start completed!"
|
||||
@echo "Run 'make run-api' to start the web interface"
|
||||
|
||||
# Gitea Actions
|
||||
validate-workflows:
|
||||
@echo "Validating Gitea Actions workflows..."
|
||||
@for file in .gitea/workflows/*.yml; do \
|
||||
echo "Checking $$file..."; \
|
||||
uv run python -c "import yaml; yaml.safe_load(open('$$file', encoding='utf-8'))" || exit 1; \
|
||||
done
|
||||
@echo "✅ All workflows are valid"
|
||||
|
||||
workflow-test:
|
||||
@echo "Testing workflow components locally..."
|
||||
make test
|
||||
make lint
|
||||
make format
|
||||
@echo "✅ Workflow test completed"
|
||||
find . -type d -name "__pycache__" -prune -exec rm -rf {} +
|
||||
rm -rf .pytest_cache/ htmlcov/ dist/ build/ *.egg-info/
|
||||
@@ -1,287 +0,0 @@
|
||||
# PostgreSQL Setup for Northern Thailand Ping River Monitor
|
||||
|
||||
This guide helps you configure PostgreSQL as the database backend for the water monitoring system.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- PostgreSQL server running on a remote machine (already available)
|
||||
- Network connectivity to the PostgreSQL server
|
||||
- Database credentials (username, password, host, port)
|
||||
|
||||
## Quick Setup
|
||||
|
||||
### 1. Configure Environment
|
||||
|
||||
Copy the example environment file and configure it:
|
||||
|
||||
```bash
|
||||
cp .env.example .env
|
||||
```
|
||||
|
||||
Edit `.env` and update the PostgreSQL configuration:
|
||||
|
||||
```bash
|
||||
# Database Configuration
|
||||
DB_TYPE=postgresql
|
||||
|
||||
# PostgreSQL Configuration (Remote Server)
|
||||
POSTGRES_CONNECTION_STRING=postgresql://username:password@your-postgres-host:5432/water_monitoring
|
||||
```
|
||||
|
||||
### 2. Run Setup Script
|
||||
|
||||
Use the interactive setup script:
|
||||
|
||||
```bash
|
||||
# Using uv
|
||||
uv run python scripts/setup_postgres.py
|
||||
|
||||
# Or using make
|
||||
make setup-postgres
|
||||
```
|
||||
|
||||
The script will:
|
||||
- Test your database connection
|
||||
- Create the database if it doesn't exist
|
||||
- Initialize the required tables and indexes
|
||||
- Set up sample monitoring stations
|
||||
|
||||
### 3. Test Connection
|
||||
|
||||
Test your PostgreSQL connection:
|
||||
|
||||
```bash
|
||||
make test-postgres
|
||||
```
|
||||
|
||||
### 4. Run the Application
|
||||
|
||||
Start collecting data:
|
||||
|
||||
```bash
|
||||
# Run a test cycle
|
||||
make run-test
|
||||
|
||||
# Start the web API
|
||||
make run-api
|
||||
```
|
||||
|
||||
## Manual Configuration
|
||||
|
||||
If you prefer manual setup, here's what you need:
|
||||
|
||||
### Connection String Format
|
||||
|
||||
```
|
||||
postgresql://username:password@host:port/database
|
||||
```
|
||||
|
||||
**Examples:**
|
||||
- Basic: `postgresql://postgres:mypassword@192.168.1.100:5432/water_monitoring`
|
||||
- With SSL: `postgresql://user:pass@host:5432/db?sslmode=require`
|
||||
- With connection pooling: `postgresql://user:pass@host:5432/db?pool_size=20&max_overflow=0`
|
||||
|
||||
### Environment Variables
|
||||
|
||||
| Variable | Description | Example |
|
||||
|----------|-------------|---------|
|
||||
| `DB_TYPE` | Database type | `postgresql` |
|
||||
| `POSTGRES_CONNECTION_STRING` | Full connection string | See above |
|
||||
|
||||
### Database Schema
|
||||
|
||||
The application uses these main tables:
|
||||
|
||||
1. **stations** - Monitoring station information
|
||||
2. **water_measurements** - Time series water level data
|
||||
3. **alert_thresholds** - Warning/danger level definitions
|
||||
4. **data_quality_log** - Data collection issue tracking
|
||||
|
||||
See `sql/init_postgres.sql` for the complete schema.
|
||||
|
||||
## Connection Options
|
||||
|
||||
### SSL Connection
|
||||
|
||||
For secure connections, add SSL parameters:
|
||||
|
||||
```bash
|
||||
POSTGRES_CONNECTION_STRING=postgresql://user:pass@host:5432/db?sslmode=require
|
||||
```
|
||||
|
||||
SSL modes:
|
||||
- `disable` - No SSL
|
||||
- `require` - Require SSL
|
||||
- `prefer` - Use SSL if available
|
||||
- `verify-ca` - Verify certificate authority
|
||||
- `verify-full` - Full certificate verification
|
||||
|
||||
### Connection Pooling
|
||||
|
||||
For high-performance applications, configure connection pooling:
|
||||
|
||||
```bash
|
||||
POSTGRES_CONNECTION_STRING=postgresql://user:pass@host:5432/db?pool_size=20&max_overflow=0
|
||||
```
|
||||
|
||||
Parameters:
|
||||
- `pool_size` - Number of connections to maintain
|
||||
- `max_overflow` - Additional connections allowed
|
||||
- `pool_timeout` - Seconds to wait for connection
|
||||
- `pool_recycle` - Seconds before connection refresh
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Common Issues
|
||||
|
||||
**1. Connection Refused**
|
||||
```
|
||||
psycopg2.OperationalError: could not connect to server
|
||||
```
|
||||
- Check if PostgreSQL server is running
|
||||
- Verify host/port in connection string
|
||||
- Check firewall settings
|
||||
|
||||
**2. Authentication Failed**
|
||||
```
|
||||
psycopg2.OperationalError: FATAL: password authentication failed
|
||||
```
|
||||
- Verify username/password in connection string
|
||||
- Check PostgreSQL pg_hba.conf configuration
|
||||
- Ensure user has database access permissions
|
||||
|
||||
**3. Database Does Not Exist**
|
||||
```
|
||||
psycopg2.OperationalError: FATAL: database "water_monitoring" does not exist
|
||||
```
|
||||
- Run the setup script to create the database
|
||||
- Or manually create: `CREATE DATABASE water_monitoring;`
|
||||
|
||||
**4. Permission Denied**
|
||||
```
|
||||
psycopg2.ProgrammingError: permission denied for table
|
||||
```
|
||||
- Ensure user has appropriate permissions
|
||||
- Grant access: `GRANT ALL PRIVILEGES ON DATABASE water_monitoring TO username;`
|
||||
|
||||
### Network Configuration
|
||||
|
||||
For remote PostgreSQL servers, ensure:
|
||||
|
||||
1. **PostgreSQL allows remote connections** (`postgresql.conf`):
|
||||
```
|
||||
listen_addresses = '*'
|
||||
port = 5432
|
||||
```
|
||||
|
||||
2. **Client authentication is configured** (`pg_hba.conf`):
|
||||
```
|
||||
# Allow connections from your application server
|
||||
host water_monitoring username your.app.ip/32 md5
|
||||
```
|
||||
|
||||
3. **Firewall allows PostgreSQL port**:
|
||||
```bash
|
||||
# On PostgreSQL server
|
||||
sudo ufw allow 5432/tcp
|
||||
```
|
||||
|
||||
### Performance Tuning
|
||||
|
||||
For optimal performance with time series data:
|
||||
|
||||
1. **Increase work_mem** for sorting operations
|
||||
2. **Tune shared_buffers** for caching
|
||||
3. **Configure maintenance_work_mem** for indexing
|
||||
4. **Set up regular VACUUM and ANALYZE** for statistics
|
||||
|
||||
Example PostgreSQL configuration additions:
|
||||
```
|
||||
# postgresql.conf
|
||||
shared_buffers = 256MB
|
||||
work_mem = 16MB
|
||||
maintenance_work_mem = 256MB
|
||||
effective_cache_size = 1GB
|
||||
```
|
||||
|
||||
## Monitoring
|
||||
|
||||
### Check Application Status
|
||||
|
||||
```bash
|
||||
# View current configuration
|
||||
uv run python -c "from src.config import Config; Config.print_settings()"
|
||||
|
||||
# Test database connection
|
||||
make test-postgres
|
||||
|
||||
# Check latest data
|
||||
psql "postgresql://user:pass@host:5432/water_monitoring" -c "SELECT COUNT(*) FROM water_measurements;"
|
||||
```
|
||||
|
||||
### PostgreSQL Monitoring
|
||||
|
||||
Connect directly to check database status:
|
||||
|
||||
```bash
|
||||
# Connect to database
|
||||
psql "postgresql://username:password@host:5432/water_monitoring"
|
||||
|
||||
# Check table sizes
|
||||
\dt+
|
||||
|
||||
# View latest measurements
|
||||
SELECT * FROM latest_measurements LIMIT 10;
|
||||
|
||||
# Check data quality
|
||||
SELECT issue_type, COUNT(*) FROM data_quality_log
|
||||
WHERE created_at > NOW() - INTERVAL '24 hours'
|
||||
GROUP BY issue_type;
|
||||
```
|
||||
|
||||
## Backup and Maintenance
|
||||
|
||||
### Backup Database
|
||||
|
||||
```bash
|
||||
# Full backup
|
||||
pg_dump "postgresql://user:pass@host:5432/water_monitoring" > backup.sql
|
||||
|
||||
# Data only
|
||||
pg_dump --data-only "postgresql://user:pass@host:5432/water_monitoring" > data_backup.sql
|
||||
```
|
||||
|
||||
### Restore Database
|
||||
|
||||
```bash
|
||||
# Restore full backup
|
||||
psql "postgresql://user:pass@host:5432/water_monitoring" < backup.sql
|
||||
|
||||
# Restore data only
|
||||
psql "postgresql://user:pass@host:5432/water_monitoring" < data_backup.sql
|
||||
```
|
||||
|
||||
### Regular Maintenance
|
||||
|
||||
Set up regular maintenance tasks:
|
||||
|
||||
```sql
|
||||
-- Update table statistics (run weekly)
|
||||
ANALYZE;
|
||||
|
||||
-- Reclaim disk space (run monthly)
|
||||
VACUUM;
|
||||
|
||||
-- Reindex tables (run quarterly)
|
||||
REINDEX DATABASE water_monitoring;
|
||||
```
|
||||
|
||||
## Next Steps
|
||||
|
||||
1. Set up monitoring and alerting
|
||||
2. Configure data retention policies
|
||||
3. Set up automated backups
|
||||
4. Implement connection pooling if needed
|
||||
5. Configure SSL for production use
|
||||
|
||||
For more advanced configuration, see the [PostgreSQL documentation](https://www.postgresql.org/docs/).
|
||||
@@ -2,18 +2,21 @@
|
||||
|
||||
Live water levels, discharge, rainfall and machine-learning flood forecasts for the
|
||||
Ping River basin around Chiang Mai. Collects hourly gauge data from public sources,
|
||||
keeps the full history in PostgreSQL, and serves a bilingual dashboard, an open REST
|
||||
API, and 6/12/24-hour flood-risk forecasts per gauge.
|
||||
keeps the full history in PostgreSQL, serves a bilingual dashboard and an open REST
|
||||
API, forecasts the flood risk per gauge 6/12/24 hours ahead, and pushes alerts when
|
||||
a gauge crosses its warning or danger level.
|
||||
|
||||
**Live: [water.buildfor.life](https://water.buildfor.life/)** · API reference at
|
||||
[/docs](https://water.buildfor.life/docs) · built by [buildfor.life](https://buildfor.life)
|
||||
after the [October 2024 flood](https://buildfor.life/blog/chiang-mai-flood-2024/) —
|
||||
background in [Teaching a Model to See the Ping River Rise 13 Hours Early](https://buildfor.life/blog/ping-river-monitor/).
|
||||
|
||||
|
||||
|
||||
[](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/actions)
|
||||
[](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/actions)
|
||||
[](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/actions)
|
||||
[](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/actions)
|
||||
[](https://python.org)
|
||||
[](LICENSE)
|
||||
|
||||
## What it does
|
||||
@@ -21,11 +24,12 @@ background in [Teaching a Model to See the Ping River Rise 13 Hours Early](https
|
||||
- **Collects** hourly water level and discharge from 16 Royal Irrigation Department
|
||||
(RID) telemetry gauges, Chiang Dao to the southern basin, since 2018-08; hourly
|
||||
rainfall and water level from 400+ ThaiWater/HII stations; Open-Meteo catchment
|
||||
rainfall (archive + 48 h forecast); daily Mae Ngat reservoir state. Every source and
|
||||
its quirks: [docs/DATA_SOURCES.md](docs/DATA_SOURCES.md).
|
||||
rainfall (archive + 48 h forecast); daily state of the large RID dams, Mae Ngat
|
||||
above all. Every source and its quirks: [docs/DATA_SOURCES.md](docs/DATA_SOURCES.md).
|
||||
- **Fills gaps.** The raw RID grid had readings for ~56 % of hours; a full-history
|
||||
re-fetch plus HII cross-fill brought it to ~93 %. `GET /api/stats` reports the
|
||||
current figure.
|
||||
re-fetch plus HII cross-fill brought it to ~93 %. RID's report often arrives
|
||||
hours late in the evening, so late readings are always stored and holes are
|
||||
re-fetched. `GET /api/stats` reports the current coverage.
|
||||
- **Forecasts.** Per gauge and horizon, a gradient-boosted model predicts the rise
|
||||
within 6/12/24 h and the probability of crossing the station's warning and danger
|
||||
levels. Trained on the monitor's own history plus catchment rain; evaluated
|
||||
@@ -33,48 +37,46 @@ background in [Teaching a Model to See the Ping River Rise 13 Hours Early](https
|
||||
data through August 2024, the first alert came **13 hours before** P.1 crossed
|
||||
3.70 m. Everything about the model, including what did not work:
|
||||
[docs/FLOOD_FORECASTING.md](docs/FLOOD_FORECASTING.md).
|
||||
- **Shows it.** A Leaflet map with the river drawn as OSM geometry and styled by live
|
||||
discharge, rain gauges, the Chiang Mai inundation zones, per-station history, the
|
||||
forecast card, a replay of the 2024 flood, English/Thai, light/dark.
|
||||
- **Shows it.** One screen: Chiang Mai's flood verdict, the gauges with their status
|
||||
and trend, a self-hosted map of the basin with rain, the city's flood zones and
|
||||
the high-water marks of 2024, and a panel per station with its history since
|
||||
2018, the forecast and how accurate it has been. ThaiWater's basin stations, a
|
||||
replay of the 2024 flood, Thai/English, dark/light.
|
||||
- **Notifies.** Public push alerts over a self-hosted [ntfy](https://ntfy.sh): one
|
||||
message when a gauge crosses its warning or danger level, one all-clear on the
|
||||
way down, an opt-in early-warning topic from the model, nothing in between.
|
||||
Subscribe from the free app, no account. Matrix room alerts for a team are
|
||||
also supported.
|
||||
Subscribe from the free app, no account. A Matrix room can get the same
|
||||
messages for a team. [docs/NOTIFICATIONS.md](docs/NOTIFICATIONS.md).
|
||||
|
||||
## Quick start
|
||||
## How it runs
|
||||
|
||||
Python **3.11** (3.13 breaks the pinned `psycopg2-binary`), PostgreSQL for anything
|
||||
beyond a quick look, [uv](https://docs.astral.sh/uv/).
|
||||
|
||||
```bash
|
||||
git clone https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor.git
|
||||
cd Northern-Thailand-Ping-River-Monitor
|
||||
uv sync --python 3.11
|
||||
cp .env.example .env # DB_TYPE, POSTGRES_CONNECTION_STRING, optional MATRIX_*
|
||||
uv run python run.py --web-api # dashboard + API on http://localhost:8000
|
||||
```text
|
||||
RID gauges · ThaiWater/HII · RID dams · Open-Meteo
|
||||
│ hourly
|
||||
▼
|
||||
┌──────────────────────────────────────┐ ┌───────────────────────────┐
|
||||
│ ping-collector (Rust, one process) │ │ monthly retrain (Python) │
|
||||
│ collect → forecast → notify │◄───────│ scikit-learn, exports │
|
||||
│ website + API on :8000 │ models │ models/flood_*.json.gz │
|
||||
└──────┬──────────────┬────────────────┘ └─────────────┬─────────────┘
|
||||
│ │ │
|
||||
▼ ▼ │
|
||||
ntfy topics, PostgreSQL ◄──────────── history ────────────┘
|
||||
Matrix room (gauges, rain, dams, forecasts, alert state)
|
||||
```
|
||||
|
||||
`DB_TYPE=sqlite` works for the dashboard and API; the forecasting path expects the
|
||||
PostgreSQL history.
|
||||
|
||||
```bash
|
||||
uv run python run.py --status # collector status
|
||||
uv run python run.py --test # one collection cycle
|
||||
uv run python run.py --fill-gaps 7 # re-fetch the last 7 days from RID
|
||||
uv run python run.py --collect-hii # one ThaiWater/HII collection cycle
|
||||
uv run python run.py --alert-check # evaluate thresholds, notify Matrix
|
||||
uv run python scripts/train_flood_model.py --stations all # retrain (~12 min)
|
||||
make test # pytest, synthetic data, no network
|
||||
make format # black + isort (the CI contract)
|
||||
```
|
||||
|
||||
RID collection in `--web-api` mode: polled at :05 past each hour
|
||||
(`RID_POLL_OFFSET_MINUTES`); while RID has not published the current hour yet,
|
||||
retried every `RID_RETRY_MINUTES` (10). Every poll stores all readings the report
|
||||
has, and yesterday's report is re-fetched while it still has holes (RID's evening
|
||||
hours often arrive 1-2 h late). On startup the leader re-fetches holes from the
|
||||
last `GAP_FILL_DAYS_ON_START` (14) days; for older history run `--fill-gaps all`.
|
||||
- **The service** is one Rust binary, `ping-collector` ([rust/README.md](rust/README.md)),
|
||||
run by systemd. It polls RID at :05 past each hour and retries every 10 minutes
|
||||
while the report is behind, fetches HII, the dams and the rain once an hour,
|
||||
forecasts after every cycle that brought new readings, sends the notifications,
|
||||
and serves the dashboard and API, in under 100 MB of memory with every model
|
||||
loaded.
|
||||
- **The models** are trained in Python (`src/ml`, scikit-learn
|
||||
HistGradientBoosting) by a monthly timer and exported for the Rust forecaster,
|
||||
which reproduces Python's features bit for bit and reloads new models without
|
||||
a restart.
|
||||
- Until October 2026 all of this was a Python/FastAPI app; how and why it moved:
|
||||
[docs/RUST_REWRITE.md](docs/RUST_REWRITE.md).
|
||||
|
||||
## API
|
||||
|
||||
@@ -88,76 +90,114 @@ Asia/Bangkok wall-clock without an offset suffix.
|
||||
| `GET /measurements/history/{code}?hours=N` | Hourly history; or `?start=YYYY-MM-DD&end=YYYY-MM-DD`; `limit` ≤ 100000 |
|
||||
| `GET /forecast` | Current flood-risk forecast, every station × horizon, with thresholds and P.1 inundation-stage probabilities |
|
||||
| `GET /api/forecast/history/{code}?hours=N&horizon=24` | Forecasts as issued, for auditing lead time after the fact |
|
||||
| `GET /api/forecast/skill?station_code=P.1` | Issued forecasts vs what happened, per deployed model version |
|
||||
| `GET /api/hii/rainfall/latest`, `/api/hii/waterlevel/latest` | Latest ThaiWater/HII gauge readings |
|
||||
| `GET /api/hii/rainfall/catchment?days=N` | HII gauge catchment-mean rain next to the Open-Meteo series the model uses |
|
||||
| `GET /api/forecast/skill?station_code=P.1` | Issued forecasts vs what happened, per deployed model version |
|
||||
| `GET /api/notifications` | ntfy server and topic names for the subscribe panel |
|
||||
| `GET /api/stats` | Row counts per source, date range, coverage |
|
||||
| `GET /health` | DB / upstream / memory checks |
|
||||
| `GET /health` | Database and data freshness (RID, ThaiWater as backup), the RID API, memory |
|
||||
| `GET /scraping/status` | Collection runs, the last forecast run, service version |
|
||||
|
||||
Interactive reference with schemas: [water.buildfor.life/docs](https://water.buildfor.life/docs).
|
||||
Responses are cached briefly server-side; poll no faster than once a minute — the data
|
||||
changes hourly.
|
||||
Interactive reference with schemas: [water.buildfor.life/docs](https://water.buildfor.life/docs)
|
||||
(also `/redoc`, `/openapi.json`). Responses are cached briefly server-side; poll no
|
||||
faster than once a minute — the data changes hourly. Station editing
|
||||
(`POST/PUT/DELETE /stations`), `/config` and `POST /scrape/trigger` need the
|
||||
server's `ADMIN_API_KEY` in an `X-API-Key` header.
|
||||
|
||||
## Run it locally
|
||||
|
||||
Needs Rust >= 1.88 ([rustup](https://rustup.rs); on Linux also `pkg-config` and
|
||||
`libssl-dev`) and PostgreSQL. Python 3.11 with [uv](https://docs.astral.sh/uv/)
|
||||
only for training models.
|
||||
|
||||
```bash
|
||||
git clone https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor.git
|
||||
cd Northern-Thailand-Ping-River-Monitor
|
||||
docker run -d --name ping-pg -e POSTGRES_PASSWORD=ping -e POSTGRES_DB=ping -p 55432:5432 postgres:16
|
||||
cp .env.example .env
|
||||
# POSTGRES_CONNECTION_STRING=postgresql://postgres:ping@localhost:55432/ping
|
||||
# WEB_LISTEN=127.0.0.1:8000
|
||||
cargo build --release --manifest-path rust/Cargo.toml
|
||||
rust/target/release/ping-collector check-db # create the tables
|
||||
rust/target/release/ping-collector fill-gaps 30 # the last 30 days from RID
|
||||
rust/target/release/ping-collector run # collect, forecast, serve http://127.0.0.1:8000
|
||||
```
|
||||
|
||||
Notifications stay off until `NTFY_SERVER` or `MATRIX_*` is set. Without trained
|
||||
models the forecast is the persistence heuristic, marked `source: "heuristic"`.
|
||||
Training needs years of history (the server's database, or `FLOOD_ML_DB_URL`) and
|
||||
the Open-Meteo archive:
|
||||
|
||||
```bash
|
||||
uv sync --python 3.11
|
||||
uv run python scripts/train_flood_model.py --stations all # writes models/flood_*
|
||||
uv run python scripts/evaluate_variants.py --stations P.1 # the rolling-origin harness
|
||||
```
|
||||
|
||||
`make help` lists the rest (`make check` runs every CI gate locally).
|
||||
|
||||
## Deployment
|
||||
|
||||
Production is a systemd unit on a small VPS behind Cloudflare, updated by `git pull`.
|
||||
`scripts/install.sh` (run as root from a checkout) creates the `water-monitor` user,
|
||||
deploys to `/opt/thailand-water-monitor`, runs `uv sync` into `.venv`, installs
|
||||
`water-monitor.service` and the monthly `water-monitor-retrain.timer`.
|
||||
Production is one systemd unit on a small VPS behind Cloudflare; the app directory
|
||||
`/opt/thailand-water-monitor` is a git checkout. To update:
|
||||
|
||||
```bash
|
||||
cd /opt/thailand-water-monitor && git pull
|
||||
cargo build --release --locked --manifest-path rust/Cargo.toml
|
||||
sudo bash scripts/install.sh
|
||||
sudo systemctl start water-monitor.service
|
||||
systemctl list-timers water-monitor-retrain.timer
|
||||
```
|
||||
|
||||
The retrain timer runs `scripts/retrain.sh`, which trains into `models/.staging`,
|
||||
refuses to promote anything that is not a rain-enabled (`hgb-v3+`) set covering the
|
||||
expected stations, and renames the bundles into place. Details and the operations
|
||||
runbook: [docs/FLOOD_FORECASTING.md](docs/FLOOD_FORECASTING.md) sections 6–8.
|
||||
`scripts/install.sh` (idempotent, also for a fresh host) installs the binary and
|
||||
`ping-collector.service`, syncs the Python environment for the monthly
|
||||
`water-monitor-retrain.timer`, runs a dry run of every source and a forecast check
|
||||
against the Python reference, restarts the service and waits for the website. The
|
||||
retrain trains into `models/.staging`, refuses to promote anything that is not a
|
||||
rain-enabled (`hgb-v3+`) set covering the expected stations, and renames the
|
||||
bundles into place; the service picks them up on its next forecast. Details and
|
||||
the operations runbook: [docs/FLOOD_FORECASTING.md](docs/FLOOD_FORECASTING.md)
|
||||
sections 6–8. Configuration: [.env.example](.env.example).
|
||||
|
||||
## Repository layout
|
||||
|
||||
```
|
||||
src/ collector, API (web_api.py), dashboard (static/dashboard.html)
|
||||
src/ml/ features, training, evaluation harness, prediction, rain/dam/HII loaders
|
||||
scripts/ train_flood_model.py, retrain.sh, evaluate_variants.py, install.sh, dev_proxy.py
|
||||
tests/ pytest suite (synthetic data; no DB or network)
|
||||
docs/ FLOOD_FORECASTING.md, DATA_SOURCES.md, deployment and station guides
|
||||
models/ trained bundles + metrics.json (gitignored) and evaluation results (tracked)
|
||||
.gitea/workflows/ ci (format/lint/tests), security (pip-audit/bandit), docs (link + OpenAPI checks)
|
||||
rust/collector/ ping-collector: collection, forecast run, ntfy/Matrix, website + API (src/web)
|
||||
rust/forecast/ forecast inference from the exported models; parity tests against Python
|
||||
src/static/ the dashboard (single file, EN/TH) and its map data
|
||||
src/data/ the bundled station list
|
||||
src/ml/ features, training, evaluation harness, export; predict.py is the reference
|
||||
src/*.py HII and RID-dam clients/stores for the archive backfills
|
||||
scripts/ install.sh, retrain.sh, train/evaluate/backfill tools, systemd units
|
||||
tests/ pytest (synthetic data; database tests need TEST_DATABASE_URL)
|
||||
docs/ model write-up, data sources, notifications, Matrix, rewrite history
|
||||
models/ trained bundles (gitignored) and evaluation results
|
||||
```
|
||||
|
||||
## Documentation
|
||||
|
||||
- [docs/FLOOD_FORECASTING.md](docs/FLOOD_FORECASTING.md) — the model: data, features, evaluation, measured performance, negatives, deployment, retraining
|
||||
- [docs/DATA_SOURCES.md](docs/DATA_SOURCES.md) — every ingested and candidate source, endpoints, quirks
|
||||
- [docs/STATION_MANAGEMENT_GUIDE.md](docs/STATION_MANAGEMENT_GUIDE.md) — adding/editing gauges
|
||||
- [docs/DATABASE_DEPLOYMENT_GUIDE.md](docs/DATABASE_DEPLOYMENT_GUIDE.md), [POSTGRESQL_SETUP.md](POSTGRESQL_SETUP.md) — database setup
|
||||
- [docs/NOTIFICATIONS.md](docs/NOTIFICATIONS.md) — public push alerts: topics, semantics, ntfy deployment
|
||||
- [docs/MATRIX_QUICK_START.md](docs/MATRIX_QUICK_START.md) — Matrix room alerts for a team
|
||||
- [docs/GAP_FILLING_GUIDE.md](docs/GAP_FILLING_GUIDE.md) — data integrity tooling
|
||||
- [docs/MATRIX_QUICK_START.md](docs/MATRIX_QUICK_START.md) — the same alerts in a Matrix room
|
||||
- [rust/README.md](rust/README.md) — the service: commands, configuration, develop, deploy
|
||||
- [docs/RUST_REWRITE.md](docs/RUST_REWRITE.md) — why and how it moved from Python to Rust
|
||||
- [docs/DASHBOARD_REDESIGN.md](docs/DASHBOARD_REDESIGN.md) — the dashboard: layout, rules, how it replaced the old page
|
||||
- [docs/references/NOTABLE_DOCUMENTS.md](docs/references/NOTABLE_DOCUMENTS.md) — official Thai government resources
|
||||
- Public overview: [buildfor.life/docs/tooling/ping-river-monitor](https://buildfor.life/docs/tooling/ping-river-monitor/)
|
||||
|
||||
Other database backends (VictoriaMetrics, InfluxDB, MySQL, SQLite) and the Grafana
|
||||
dashboards under `grafana/` are supported by the adapters but not what production
|
||||
runs; see [docs/VICTORIAMETRICS_SETUP.md](docs/VICTORIAMETRICS_SETUP.md) if you want them.
|
||||
|
||||
## Contributing
|
||||
|
||||
`make format` before committing (black 88 columns, isort black profile — the CI gate),
|
||||
`make test` must stay green, tests use synthetic data only. See
|
||||
`make format` before committing, `make check` must stay green; see
|
||||
[CONTRIBUTING.md](CONTRIBUTING.md). Issues and merge requests on
|
||||
[git.b4l.co.th](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor).
|
||||
|
||||
## Data sources and thanks
|
||||
|
||||
Royal Irrigation Department (RID) gauge telemetry; Hydro-Informatics Institute (HII) /
|
||||
ThaiWater open API; Open-Meteo; OpenStreetMap contributors for the river geometry;
|
||||
Chiang Mai Municipality for the inundation map the P.1 stages are keyed to. All
|
||||
instruments are theirs; we aggregate, store, fill gaps and forecast.
|
||||
Royal Irrigation Department (RID) gauge telemetry and reservoir reports;
|
||||
Hydro-Informatics Institute (HII) / ThaiWater open API; Open-Meteo; OpenStreetMap
|
||||
contributors for the river geometry; the CMU Water Center (Chiang Mai University)
|
||||
for the flood hazard map of the 2024 flood that the P.1 stages and the city's flood
|
||||
zones come from. All instruments are theirs; we aggregate, store, fill
|
||||
gaps and forecast.
|
||||
|
||||
## License
|
||||
|
||||
|
||||
@@ -1,278 +0,0 @@
|
||||
# SQLite to PostgreSQL Migration Guide
|
||||
|
||||
This guide helps you migrate your existing SQLite water monitoring data to PostgreSQL.
|
||||
|
||||
## Quick Migration
|
||||
|
||||
### 1. Analyze Your SQLite Database (Optional)
|
||||
|
||||
First, check what's in your SQLite database:
|
||||
|
||||
```bash
|
||||
# Analyze without migrating
|
||||
make analyze-sqlite
|
||||
|
||||
# Or specify a specific SQLite file
|
||||
uv run python scripts/migrate_sqlite_to_postgres.py --dry-run /path/to/your/database.db
|
||||
```
|
||||
|
||||
### 2. Run the Migration
|
||||
|
||||
```bash
|
||||
# Auto-detect SQLite file and migrate
|
||||
make migrate-sqlite
|
||||
|
||||
# Or specify a specific SQLite file
|
||||
uv run python scripts/migrate_sqlite_to_postgres.py /path/to/your/database.db
|
||||
```
|
||||
|
||||
The migration tool will:
|
||||
- ✅ Connect to both databases
|
||||
- ✅ Analyze your SQLite schema automatically
|
||||
- ✅ Migrate station information
|
||||
- ✅ Migrate all measurement data in batches
|
||||
- ✅ Handle different SQLite table structures
|
||||
- ✅ Verify the migration results
|
||||
- ✅ Generate a detailed log file
|
||||
|
||||
## What Gets Migrated
|
||||
|
||||
### Station Data
|
||||
- Station IDs and codes
|
||||
- Thai and English names
|
||||
- Coordinates (latitude/longitude)
|
||||
- Geohash data (if available)
|
||||
- Creation/update timestamps
|
||||
|
||||
### Measurement Data
|
||||
- Water level readings
|
||||
- Discharge measurements
|
||||
- Discharge percentages
|
||||
- Timestamps
|
||||
- Station associations
|
||||
- Data quality status
|
||||
|
||||
## Supported SQLite Schemas
|
||||
|
||||
The migration tool automatically detects and handles various SQLite table structures:
|
||||
|
||||
### Modern Schema
|
||||
```sql
|
||||
-- Stations
|
||||
stations: id, station_code, station_name_th, station_name_en, latitude, longitude, geohash
|
||||
|
||||
-- Measurements
|
||||
water_measurements: timestamp, station_id, water_level, discharge, discharge_percent, status
|
||||
```
|
||||
|
||||
### Legacy Schema
|
||||
```sql
|
||||
-- Stations
|
||||
water_stations: station_id, station_code, station_name, lat, lon
|
||||
|
||||
-- Measurements
|
||||
measurements: timestamp, station_id, water_level, discharge, discharge_percent
|
||||
```
|
||||
|
||||
### Simple Schema
|
||||
```sql
|
||||
-- Any table with basic water level data
|
||||
-- The tool will adapt and map columns automatically
|
||||
```
|
||||
|
||||
## Migration Process
|
||||
|
||||
### Step 1: Database Connection
|
||||
- Connects to your SQLite database
|
||||
- Verifies PostgreSQL connection
|
||||
- Validates configuration
|
||||
|
||||
### Step 2: Schema Analysis
|
||||
- Scans SQLite tables and columns
|
||||
- Reports data counts
|
||||
- Identifies table structures
|
||||
|
||||
### Step 3: Station Migration
|
||||
- Extracts station metadata
|
||||
- Maps to PostgreSQL format
|
||||
- Handles missing data gracefully
|
||||
|
||||
### Step 4: Measurement Migration
|
||||
- Processes data in batches (1000 records at a time)
|
||||
- Converts timestamps correctly
|
||||
- Preserves all measurement values
|
||||
- Shows progress during migration
|
||||
|
||||
### Step 5: Verification
|
||||
- Compares record counts
|
||||
- Validates data integrity
|
||||
- Reports migration statistics
|
||||
|
||||
## Command Options
|
||||
|
||||
```bash
|
||||
# Basic migration (auto-detects SQLite file)
|
||||
uv run python scripts/migrate_sqlite_to_postgres.py
|
||||
|
||||
# Specify SQLite database path
|
||||
uv run python scripts/migrate_sqlite_to_postgres.py /path/to/database.db
|
||||
|
||||
# Dry run (analyze only, no migration)
|
||||
uv run python scripts/migrate_sqlite_to_postgres.py --dry-run
|
||||
|
||||
# Custom batch size for large databases
|
||||
uv run python scripts/migrate_sqlite_to_postgres.py --batch-size 5000
|
||||
```
|
||||
|
||||
## Auto-Detection
|
||||
|
||||
The tool automatically searches for SQLite files in common locations:
|
||||
- `water_levels.db`
|
||||
- `water_monitoring.db`
|
||||
- `database.db`
|
||||
- `../water_levels.db`
|
||||
|
||||
## Migration Output
|
||||
|
||||
The tool provides detailed logging:
|
||||
|
||||
```
|
||||
========================================
|
||||
SQLite to PostgreSQL Migration Tool
|
||||
========================================
|
||||
SQLite database: water_levels.db
|
||||
PostgreSQL: postgresql
|
||||
|
||||
Step 1: Connecting to databases...
|
||||
Connected to SQLite database: water_levels.db
|
||||
Connected to PostgreSQL database
|
||||
|
||||
Step 2: Analyzing SQLite database structure...
|
||||
Table 'stations': 8 columns, 25 rows
|
||||
Table 'water_measurements': 7 columns, 15420 rows
|
||||
|
||||
Step 3: Migrating station data...
|
||||
Migrated 25 stations
|
||||
|
||||
Step 4: Migrating measurement data...
|
||||
Found 15420 measurements to migrate
|
||||
Migrated 1000/15420 measurements
|
||||
Migrated 2000/15420 measurements
|
||||
...
|
||||
Successfully migrated 15420 measurements
|
||||
|
||||
Step 5: Verifying migration...
|
||||
SQLite stations: 25
|
||||
SQLite measurements: 15420
|
||||
PostgreSQL measurements retrieved: 15420
|
||||
Migrated stations: 25
|
||||
Migrated measurements: 15420
|
||||
|
||||
========================================
|
||||
MIGRATION COMPLETED
|
||||
========================================
|
||||
Duration: 0:02:15
|
||||
Stations migrated: 25
|
||||
Measurements migrated: 15420
|
||||
No errors encountered
|
||||
```
|
||||
|
||||
## Error Handling
|
||||
|
||||
The migration tool is robust and handles:
|
||||
- **Missing tables** - Tries alternative table names
|
||||
- **Different column names** - Maps common variations
|
||||
- **Missing data** - Uses sensible defaults
|
||||
- **Invalid timestamps** - Attempts multiple date formats
|
||||
- **Connection issues** - Provides clear error messages
|
||||
- **Large datasets** - Processes in batches to avoid memory issues
|
||||
|
||||
## Log Files
|
||||
|
||||
Migration creates a detailed log file:
|
||||
- `migration.log` - Complete migration log
|
||||
- Shows all operations, errors, and statistics
|
||||
- Useful for troubleshooting
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Common Issues
|
||||
|
||||
**1. SQLite file not found**
|
||||
```
|
||||
SQLite database file not found. Please specify the path:
|
||||
python migrate_sqlite_to_postgres.py /path/to/database.db
|
||||
```
|
||||
**Solution**: Specify the correct path to your SQLite file
|
||||
|
||||
**2. PostgreSQL not configured**
|
||||
```
|
||||
Error: PostgreSQL not configured. Set DB_TYPE=postgresql in your .env file
|
||||
```
|
||||
**Solution**: Ensure your .env file has `DB_TYPE=postgresql`
|
||||
|
||||
**3. Connection failed**
|
||||
```
|
||||
Database connection error: connection refused
|
||||
```
|
||||
**Solution**: Check your PostgreSQL connection settings
|
||||
|
||||
**4. No tables found**
|
||||
```
|
||||
Could not analyze SQLite database structure
|
||||
```
|
||||
**Solution**: Verify your SQLite file contains water monitoring data
|
||||
|
||||
### Performance Tips
|
||||
|
||||
- **Large databases**: Use `--batch-size 5000` for faster processing
|
||||
- **Slow networks**: Reduce batch size to `--batch-size 100`
|
||||
- **Memory issues**: Process smaller batches
|
||||
|
||||
## After Migration
|
||||
|
||||
Once migration is complete:
|
||||
|
||||
1. **Verify data**:
|
||||
```bash
|
||||
make run-test
|
||||
make run-api
|
||||
```
|
||||
|
||||
2. **Check the web interface**: Latest readings should show your migrated data
|
||||
|
||||
3. **Backup your SQLite**: Keep the original file as backup
|
||||
|
||||
4. **Update configurations**: Remove SQLite references from configs
|
||||
|
||||
## Rollback
|
||||
|
||||
If you need to rollback:
|
||||
|
||||
1. **Clear PostgreSQL data**:
|
||||
```sql
|
||||
DELETE FROM water_measurements;
|
||||
DELETE FROM stations;
|
||||
```
|
||||
|
||||
2. **Switch back to SQLite**:
|
||||
```bash
|
||||
# In .env file
|
||||
DB_TYPE=sqlite
|
||||
WATER_DB_PATH=water_levels.db
|
||||
```
|
||||
|
||||
3. **Test the rollback**:
|
||||
```bash
|
||||
make run-test
|
||||
```
|
||||
|
||||
The migration tool is designed to be safe and can be run multiple times - it handles duplicates appropriately.
|
||||
|
||||
## Next Steps
|
||||
|
||||
After successful migration:
|
||||
- Set up automated backups for PostgreSQL
|
||||
- Configure monitoring and alerting
|
||||
- Consider data retention policies
|
||||
- Update documentation references
|
||||
@@ -1,106 +0,0 @@
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
victoriametrics:
|
||||
image: victoriametrics/victoria-metrics:latest
|
||||
container_name: ping_river_victoriametrics
|
||||
ports:
|
||||
- "8428:8428"
|
||||
volumes:
|
||||
- vm_data:/victoria-metrics-data
|
||||
- ./vm-config:/etc/victoriametrics
|
||||
command:
|
||||
- '--storageDataPath=/victoria-metrics-data'
|
||||
- '--retentionPeriod=2y'
|
||||
- '--httpListenAddr=:8428'
|
||||
- '--maxConcurrentInserts=32'
|
||||
- '--search.maxQueryDuration=60s'
|
||||
- '--search.maxConcurrentRequests=16'
|
||||
- '--dedup.minScrapeInterval=30s'
|
||||
- '--memory.allowedPercent=80'
|
||||
- '--loggerLevel=INFO'
|
||||
- '--loggerFormat=json'
|
||||
- '--search.maxSeries=1000000'
|
||||
- '--search.maxPointsPerTimeseries=100000'
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- TZ=Asia/Bangkok
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost:8428/health"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 40s
|
||||
labels:
|
||||
- "com.victoriametrics.service=ping-river-monitoring"
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:latest
|
||||
container_name: ping_river_grafana
|
||||
ports:
|
||||
- "3000:3000"
|
||||
volumes:
|
||||
- grafana_data:/var/lib/grafana
|
||||
- ./grafana/provisioning:/etc/grafana/provisioning
|
||||
- ./grafana/dashboards:/var/lib/grafana/dashboards
|
||||
environment:
|
||||
- GF_SECURITY_ADMIN_PASSWORD=admin_password
|
||||
- GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-worldmap-panel
|
||||
- GF_USERS_ALLOW_SIGN_UP=false
|
||||
- GF_SECURITY_ALLOW_EMBEDDING=true
|
||||
- GF_AUTH_ANONYMOUS_ENABLED=false
|
||||
- TZ=Asia/Bangkok
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
victoriametrics:
|
||||
condition: service_healthy
|
||||
labels:
|
||||
- "com.victoriametrics.service=ping-river-monitoring"
|
||||
|
||||
ping-river-monitor:
|
||||
build: .
|
||||
container_name: ping_river_monitor
|
||||
environment:
|
||||
- DB_TYPE=victoriametrics
|
||||
- VM_HOST=victoriametrics
|
||||
- VM_PORT=8428
|
||||
- TZ=Asia/Bangkok
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
victoriametrics:
|
||||
condition: service_healthy
|
||||
volumes:
|
||||
- ./logs:/app/logs
|
||||
labels:
|
||||
- "com.victoriametrics.service=ping-river-monitoring"
|
||||
|
||||
ping-river-api:
|
||||
build: .
|
||||
container_name: ping_river_api
|
||||
ports:
|
||||
- "8000:8000"
|
||||
environment:
|
||||
- DB_TYPE=victoriametrics
|
||||
- VM_HOST=victoriametrics
|
||||
- VM_PORT=8428
|
||||
- TZ=Asia/Bangkok
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
victoriametrics:
|
||||
condition: service_healthy
|
||||
volumes:
|
||||
- ./logs:/app/logs
|
||||
command: ["python", "-m", "src.main", "--web-api"]
|
||||
labels:
|
||||
- "com.victoriametrics.service=ping-river-monitoring"
|
||||
|
||||
volumes:
|
||||
vm_data:
|
||||
driver: local
|
||||
grafana_data:
|
||||
driver: local
|
||||
|
||||
networks:
|
||||
default:
|
||||
name: ping-river-monitoring
|
||||
driver: bridge
|
||||
@@ -0,0 +1,157 @@
|
||||
# Dashboard redesign: the console
|
||||
|
||||
Status: **phase 4 done** (2026-10-20). The console is the dashboard: it replaced the
|
||||
old page at `/` (`src/static/dashboard.html`; the server reads it at start, so a
|
||||
change needs a restart). Phase 5 (polish) is open.
|
||||
|
||||
- **Locally:** `uv run python scripts/dev_proxy.py 8765`, then <http://localhost:8765/>.
|
||||
The API is proxied to production.
|
||||
- **Rules** (`tests/test_dashboard.py`):
|
||||
- every path it fetches is a Rust route;
|
||||
- every string exists in EN and TH, and Thai is the default;
|
||||
- no emoji;
|
||||
- timestamps are Bangkok wall-clock time;
|
||||
- a replay or a simulation is never shown as live data.
|
||||
|
||||
## Why: the old page
|
||||
|
||||
Observed on 2026-10-19 at 1440 px and on a 390 px phone.
|
||||
|
||||
- **No clear first answer.** "Is Chiang Mai flooding?" is a thin green banner above
|
||||
four KPI cards of equal weight. Two of them answer nothing for a resident: "Most
|
||||
stressed gauge 16%" and "Chiang Mai river flow 53.9" (no unit in the number). At the
|
||||
bottom there are five more (2,700,706 datapoints, 3,002 days, …).
|
||||
- **The colours answer the wrong question.** Markers and river lines are coloured by
|
||||
discharge class (m³/s), not by distance to the warning level, and the station list
|
||||
shows discharge.
|
||||
- **The map is noisy.** OSM terrain with Thai labels, numbered markers and animated
|
||||
dashes. The legend covers about a third of the map, and more than half of the first
|
||||
phone screen.
|
||||
- **The header is busy.** It has six buttons with emoji (↻ 🌙 🔔 ▶); on a phone they
|
||||
wrap into three rows.
|
||||
- **Internal detail on the main page.** The model-skill table lists seven model hashes;
|
||||
the history card is an empty 360 px box until a station is clicked.
|
||||
- **Inconsistent styling.** A gradient background, several card and pill styles, three
|
||||
greens.
|
||||
|
||||
## Decision
|
||||
|
||||
Two directions were sketched on 2026-10-19 against live data (throwaway HTML, not kept):
|
||||
|
||||
- **A:** a public information page.
|
||||
- **B:** an operations console.
|
||||
|
||||
The user chose **B**.
|
||||
|
||||
## Layout
|
||||
|
||||
One screen, no page scroll on a desktop:
|
||||
|
||||
| Area | Content |
|
||||
|---|---|
|
||||
| Top bar | brand · **Chiang Mai verdict** (P.1 level + 24 h forecast, always visible, click selects P.1) · gauges reporting · highest 24 h flood chance · data age · EN/ไทย · light/dark · Get alerts |
|
||||
| Left | two lists: **RID gauges** (status bar, code, name, margin to the warning level, 72 h sparkline, level and 6 h change; sort by river order, closest to the warning level, rising) and **Basin stations** (ThaiWater/HII: % of bank height, river and agency; sort fullest first or by name); filter; legend |
|
||||
| Centre | map on a self-hosted vector basemap (labels in the page language): Ping mainstem and tributaries, gauges coloured by status with code + level labels, layers Basin stations (diamonds), Rain 24 h, Flood zones and Flood marks 2024, the team's markers from zoom 12, one legend |
|
||||
| Right | selected RID gauge: level, 6 h change, discharge, warning/danger level with margins, capacity; level scale with now and 24 h peak; history (3 d / 7 d / 30 d / 90 d / 1 y / all or dates: level, discharge, the 24 h peaks as issued, warning/danger bands, hover values; up to 7 d it ends in the current forecast); forecast table (peak, chance of warning/danger at 6/12/24 h, model, issue time, disclaimer); for P.1 the five city flood stages, the zones toggle and "How accurate is the forecast?"; data links. A basin station shows level (m MSL), % of bank height with ThaiWater's class, distance to and height of the bank top, gauge reading, agency (latest reading only: no history is kept for them) |
|
||||
| Status bar | data credits, "2.7M readings since 2018" (opens the data statistics), API, source code |
|
||||
|
||||
**Narrower screens:**
|
||||
|
||||
| Width | Layout |
|
||||
|---|---|
|
||||
| < 1280 px | The panel moves below list and map, in columns |
|
||||
| < 860 px | One column: verdict, map, list, panel |
|
||||
|
||||
**Behaviour:**
|
||||
- Selection is in the URL (`#P.84`, `#hii-1198299`); opening a basin station switches to its list.
|
||||
- Arrow keys move through the list.
|
||||
- Data refreshes every 5 minutes.
|
||||
|
||||
## Where every current element goes
|
||||
|
||||
| Old page (`dashboard.html` id) | Console | Phase |
|
||||
|---|---|---|
|
||||
| `flood-verdict`, `verdict-*` | Verdict in the top bar | 1 |
|
||||
| `station-count`, `last-updated`, `data-age`, `live-pill` | Top-bar KPIs (reporting, data age) | 1 |
|
||||
| `river-flow`, `peak-station` (P.1 flow, most stressed gauge) | Discharge and capacity in the panel; "Closest to warning level" sort | 1 |
|
||||
| `station-map`, `map-legend`, `legend-toggle`, `rain-toggle`, `zones-toggle` | Map, layer buttons, one legend | 1 |
|
||||
| side list, `station-search` | Gauge list with filter and sorts | 1 |
|
||||
| `forecast-card`, `p1-outlook`, `p1-peak`, `p1-stages`, `forecast-grid` | Panel: forecast table for every gauge, stages for P.1 | 1 |
|
||||
| `alerts-panel`, `alerts-*` | Alerts drawer | 1 |
|
||||
| `refresh-button` | Removed: refresh every 5 min, age in the top bar | 1 |
|
||||
| `lang-toggle`, `theme-toggle` | EN / ไทย segmented control, icon button | 1 |
|
||||
| `error`, `loading` | Map overlay with retry; "Loading" in the list | 1 |
|
||||
| `thaiwater-sensors`, `sensors-*` | Basin stations list + map layer | 2 |
|
||||
| custom markers | Pins from zoom 12, popup with their flood zone | 2 |
|
||||
| `history-card`, `history-*` | Panel history with ranges | 3 |
|
||||
| `skill-panel`, `skill-*` | Panel section for P.1 | 3 |
|
||||
| `db-stats`, `db-*` | Status-bar popover | 3 |
|
||||
| `replay-2024`, `replay-*`, demo hooks | Replay/simulation mode with banner and mode pill | 4 |
|
||||
|
||||
## Design rules
|
||||
|
||||
- **Type:** Inter (Latin) and Noto Sans Thai, self-hosted in `src/static/fonts/` (75 KB, OFL). Numbers use tabular figures.
|
||||
- **Colours:** CSS variables for light and dark, with dark the default; a saved choice wins (`ping-river-theme`, the old dashboard's key, so the choice carried over).
|
||||
- **Accent:** `#1d6fd6` light, `#4c9bf0` dark.
|
||||
- **Status:** colours are used only for status, each with an `-ink` variant for text (WCAG AA on white).
|
||||
- **One status rule** (`statusOf`) for the map, list and panel. It is display-only; the notifier keeps its own transitions.
|
||||
|
||||
| Status | Rule |
|
||||
|---|---|
|
||||
| Normal | none of the rules below |
|
||||
| Watch | within 0.5 m of the warning level, or 24 h warning chance ≥ 25 % |
|
||||
| Warning | level ≥ `threshold_warning` |
|
||||
| Danger | level ≥ `threshold_danger` |
|
||||
| No data | no reading for 24 h |
|
||||
|
||||
- **City verdict:** the classes of `statusOf` for P.1 — flooding (the Ping over its banks) from 4.20 m, high water from 3.70 m; "flooding likely" at danger ≥ 50 %, "high water likely" at warning ≥ 70 %; watch at warning ≥ 20 % or level ≥ 3.20 m. (Until 2026-10-25 it said flooding from 3.70 m, after the older municipal map.)
|
||||
- **Data age:** "feed stale" after 3 h, except during RID's overnight pause (21:00 to 09:00 with an evening reading). The pause shows as "overnight pause", the same rule as the notifier's `FEED_NIGHT_UNTIL_HOUR`.
|
||||
- **Discharge:** comes with RID's report. When the newest hour is a level-only reading, the panel shows the last RID discharge with its time ("at 14:00").
|
||||
- **Rain layer:** off by default unless any station has heavy rain (> 35 mm / 24 h); an explicit choice is remembered.
|
||||
- **Flood zones:** `src/static/flood-zones-p1.geojson`, CMU Water Center's flood hazard map of the October 2024 flood (built by `scripts/build_flood_zones.py` from CMU's KML): five disjoint zones, zone k floods first when P.1 reaches stage k (4.30, 4.50, 4.70, 5.00, 5.30 m), in CMU's colours from deep red to yellow, credited in the map attribution. Shown automatically while P.1 is over its banks (danger level) or stage 1 is at least 50 % likely within 24 h, unless hidden by the user. A point's zone (team markers) honours the holes: dry islands inside a zone are outside it.
|
||||
- **Basin stations:** ThaiWater's classes for the level as a share of the bank height (critically low ≤ 10 %, low ≤ 30 %, normal ≤ 70 %, high ≤ 100 %, over the bank). On the map and in the list over the bank is danger, high is watch, the rest normal; no data after 24 h. Gauges RID already lists are shown once, as RID gauges. The layer is off until the list is opened or the button pressed.
|
||||
- **History:** above 2,500 readings (30 d and longer for most gauges) the chart shows the daily mean with the daily low–high band, so a flood peak is not averaged away; gaps longer than 3.5 h (2.5 days when daily) break the line rather than bridging RID's nightly pause. Short ranges reload with every refresh, long ones at most hourly. "All" asks for `limit=100000`: `/measurements/history` returns the oldest rows first and stops at its 50,000 default, which made the old dashboard's "All time" for P.1 end in October 2024.
|
||||
- **Issued forecasts:** the history's violet line is the model's 24 h peak for every hour (`/api/forecast/history`, `source = model` only: a heuristic row is the level itself and drew a dip into the line). The nightly gaps were RID's overnight batch; the collector now forecasts those hours when they arrive (docs/FLOOD_FORECASTING.md, Serving).
|
||||
- **Forecast accuracy:** loaded when the section is opened (a cold `/api/forecast/skill` query takes up to 20 s), cached for 15 minutes; bias and persistence error are in each row's tooltip.
|
||||
- **Replay of October 2024:** `src/static/flood-2024-all.json` (hourly readings of the 16 gauges, 15 September – 12 October 2024, and the 24 h warning chance and peak of a model trained on data before 1 September 2024). The page's clock, verdict, list, markers, flood zones and panel follow the replay; the panel's chart shows the season on fixed axes up to the replay hour. It plays hour by hour while the model alerts or P.1 is above 3.50 m, faster through quiet days, and can be paused and dragged. Rain, the basin stations and the 5-minute refresh are off during the replay, since they are today's data. A violet bar (REPLAY, date, P.1 level, the model's message) and the top bar's "Replay time" replace the live data age.
|
||||
- **Simulation:** `?demo_level=4.4` pins P.1 at 4.40 m; `?demo_rise=1` raises it by 5 cm every 0.7 s from the real level to the 2024 record (5.30 m). An amber SIMULATION bar names the real level; the top bar and the P.1 panel say "simulated". Only P.1 is simulated: the forecasts and other gauges stay live.
|
||||
- **Flood marks 2024:** `src/static/flood-marks-2024.json`, CMU Water Center's ~4,900 marks of the October 2024 high-water line (depth above the road at P.1 5.30 m; `scripts/build_flood_marks.py`), on a canvas from zoom 12 in CMU's five depth classes; turning the layer on below zoom 12 flies to the city. A mark's popup gives the depth and its flood zone; zone and marker popups name the nearest mark within 250 m (the marks load with the zones). Off by default; also available in the replay, since it is 2024 data. Popups pan clear of the layer bar, which wraps to two rows on phones.
|
||||
- **Basemap:** `src/static/basemap/ping-basin.pmtiles`, a Protomaps extract (bbox 97.85–99.95 E, 16.85–20.2 N, zoom 0–12, 15 MB, OpenStreetMap data), drawn by protomaps-leaflet 5.1.0 in its `light`/`dark` style with `name:th`/`name:en` labels. Refresh instructions in `src/static/basemap/README.md`. The map cannot be panned far outside the basin. OSM raster tiles are the fallback if the renderer does not load.
|
||||
|
||||
## Phases
|
||||
|
||||
Each phase is one commit, checked with screenshots: 1440, 1100 and 390 px; light and dark; EN and TH.
|
||||
|
||||
1. **Done — Console shell with live data.**
|
||||
- Layout, design tokens, fonts, EN/TH.
|
||||
- Verdict and KPIs, gauge list with sparklines and sorts.
|
||||
- Map with status markers, rain and flood-zone layers.
|
||||
- Station panel (readings, scale, 72 h chart, forecast table, P.1 stages).
|
||||
- Alerts drawer (ntfy topics); URL selection; 5-minute refresh.
|
||||
- `tests/test_console.py` (now `tests/test_dashboard.py`); `dev_proxy.py` serves `src/static` with proper content types.
|
||||
2. **Done — Map and basin.**
|
||||
- **Basemap:** the self-hosted Protomaps extract replaces the dimmed OSM raster tiles, whose baked-in labels were the noisiest part of the screen. Labels follow the page language; light and dark styles.
|
||||
- **Server:** ServeDir answers HTTP range requests (checked: 206); `dev_proxy.py` now does too.
|
||||
- **ThaiWater/HII stations:** the Basin stations list and map layer, from `/api/hii/waterlevel/latest`, falling back to `/sensors/thaiwater`.
|
||||
- **Custom markers:** `custom-markers.json` as pins, with their flood-zone popup.
|
||||
3. **Done — Station depth.**
|
||||
- **History in the panel** (inline SVG, no chart library): ranges and dates, level and discharge, issued 24 h forecasts overlaid, warning/danger bands, hover/touch values.
|
||||
- **P.1 forecast accuracy** as a collapsible panel section, from `/api/forecast/skill`.
|
||||
- **Data statistics:** a status-bar popover, from `/api/stats`.
|
||||
4. **Done — Modes and the swap.**
|
||||
- **October 2024 replay:** banner, clock, markers and verdict follow the replay; zones auto-show.
|
||||
- **Demo hooks:** `?demo_level=`, `?demo_rise=`. A mode pill (LIVE / REPLAY / SIMULATION) never relabels a replay or simulation as live.
|
||||
- **The swap:** `console.html` became `dashboard.html`, with the old page's title, description, Open Graph and JSON-LD tags; the robots `noindex` is gone. `tests/test_dashboard.py` is the console's tests plus the old ones that still apply (one P.1 level, SEO and indexing files).
|
||||
- **Docs:** README screenshot and the CLAUDE.md rule.
|
||||
5. **Polish.**
|
||||
- Phone layout pass.
|
||||
- Native-speaker review of the Thai strings.
|
||||
- Keyboard and screen reader pass.
|
||||
- Lighthouse ≥ 90 (performance, accessibility).
|
||||
- No console errors.
|
||||
|
||||
## Data notes from building phase 1
|
||||
|
||||
- **Negative levels.** P.4A (−1.28 m) and P.67 (−0.59 m) report negative levels: gauge datums, not errors. The panel's scale starts at the lowest of 0 and the station's 72 h readings.
|
||||
- **P.1 discharge disagrees between endpoints.** At 15:00 on 2026-10-19, `/measurements/latest` had level 1.30 m with no discharge, while `/measurements/history` had 26.0 m³/s with no capacity. RID's 14:00 report said 53.9 m³/s. The rows without capacity (`discharge_percent`) are ThaiWater's copy of the gauge, which fills hours RID has not published yet (its night hours); their discharge comes from a different rating, about half of RID's at P.1. The panel shows the last RID discharge, and the history chart drops the discharge of those rows for gauges RID reports discharge for.
|
||||
- **P.84 spike.** A one-hour spike at 13:00 the same day: 0.73 → 1.05 → 0.78 m.
|
||||
@@ -1,447 +0,0 @@
|
||||
# Database Deployment Guide for Thailand Water Monitor
|
||||
|
||||
This guide covers deployment options for storing water monitoring data in production environments.
|
||||
|
||||
## 🏆 Recommendation Summary
|
||||
|
||||
| Database | Best For | Performance | Complexity | Cost |
|
||||
|----------|----------|-------------|------------|------|
|
||||
| **InfluxDB** | Time-series data, dashboards | ⭐⭐⭐⭐⭐ | ⭐⭐⭐ | ⭐⭐⭐ |
|
||||
| **VictoriaMetrics** | High-performance metrics | ⭐⭐⭐⭐⭐ | ⭐⭐⭐⭐ | ⭐⭐⭐⭐ |
|
||||
| **PostgreSQL** | Complex queries, reliability | ⭐⭐⭐⭐ | ⭐⭐ | ⭐⭐⭐⭐ |
|
||||
| **MySQL** | Familiar, existing infrastructure | ⭐⭐⭐ | ⭐⭐ | ⭐⭐⭐⭐ |
|
||||
|
||||
## 1. InfluxDB Deployment (Recommended for Time-Series)
|
||||
|
||||
### Why InfluxDB?
|
||||
- **Purpose-built** for time-series data
|
||||
- **Excellent compression** (10:1 typical ratio)
|
||||
- **Built-in retention policies** and downsampling
|
||||
- **Great Grafana integration** for dashboards
|
||||
- **High write throughput** (100k+ points/second)
|
||||
|
||||
### Docker Deployment
|
||||
|
||||
```yaml
|
||||
# docker-compose.yml
|
||||
version: '3.8'
|
||||
services:
|
||||
influxdb:
|
||||
image: influxdb:1.8
|
||||
container_name: water_influxdb
|
||||
ports:
|
||||
- "8086:8086"
|
||||
volumes:
|
||||
- influxdb_data:/var/lib/influxdb
|
||||
- ./influxdb.conf:/etc/influxdb/influxdb.conf:ro
|
||||
environment:
|
||||
- INFLUXDB_DB=water_monitoring
|
||||
- INFLUXDB_ADMIN_USER=admin
|
||||
- INFLUXDB_ADMIN_PASSWORD=your_secure_password
|
||||
- INFLUXDB_USER=water_user
|
||||
- INFLUXDB_USER_PASSWORD=water_password
|
||||
restart: unless-stopped
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:latest
|
||||
container_name: water_grafana
|
||||
ports:
|
||||
- "3000:3000"
|
||||
volumes:
|
||||
- grafana_data:/var/lib/grafana
|
||||
environment:
|
||||
- GF_SECURITY_ADMIN_PASSWORD=admin_password
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
influxdb_data:
|
||||
grafana_data:
|
||||
```
|
||||
|
||||
### Environment Variables
|
||||
```bash
|
||||
# .env file
|
||||
DB_TYPE=influxdb
|
||||
INFLUX_HOST=localhost
|
||||
INFLUX_PORT=8086
|
||||
INFLUX_DATABASE=water_monitoring
|
||||
INFLUX_USERNAME=water_user
|
||||
INFLUX_PASSWORD=water_password
|
||||
```
|
||||
|
||||
### InfluxDB Configuration
|
||||
```toml
|
||||
# influxdb.conf
|
||||
[meta]
|
||||
dir = "/var/lib/influxdb/meta"
|
||||
|
||||
[data]
|
||||
dir = "/var/lib/influxdb/data"
|
||||
wal-dir = "/var/lib/influxdb/wal"
|
||||
|
||||
# Optimize for time-series data
|
||||
cache-max-memory-size = "1g"
|
||||
cache-snapshot-memory-size = "25m"
|
||||
cache-snapshot-write-cold-duration = "10m"
|
||||
|
||||
# Retention and compression
|
||||
compact-full-write-cold-duration = "4h"
|
||||
max-series-per-database = 1000000
|
||||
max-values-per-tag = 100000
|
||||
|
||||
[coordinator]
|
||||
write-timeout = "10s"
|
||||
max-concurrent-queries = 0
|
||||
query-timeout = "0s"
|
||||
|
||||
[retention]
|
||||
enabled = true
|
||||
check-interval = "30m"
|
||||
|
||||
[http]
|
||||
enabled = true
|
||||
bind-address = ":8086"
|
||||
auth-enabled = true
|
||||
max-body-size = "25000000"
|
||||
max-concurrent-requests = 0
|
||||
max-enqueued-requests = 0
|
||||
```
|
||||
|
||||
### Production Setup Commands
|
||||
```bash
|
||||
# Start services
|
||||
docker-compose up -d
|
||||
|
||||
# Create retention policies
|
||||
docker exec -it water_influxdb influx -username admin -password your_secure_password -execute "
|
||||
CREATE RETENTION POLICY \"raw_data\" ON \"water_monitoring\" DURATION 90d REPLICATION 1 DEFAULT;
|
||||
CREATE RETENTION POLICY \"downsampled\" ON \"water_monitoring\" DURATION 730d REPLICATION 1;
|
||||
"
|
||||
|
||||
# Create continuous queries for downsampling
|
||||
docker exec -it water_influxdb influx -username admin -password your_secure_password -execute "
|
||||
CREATE CONTINUOUS QUERY \"downsample_hourly\" ON \"water_monitoring\"
|
||||
BEGIN
|
||||
SELECT mean(water_level) AS water_level, mean(discharge) AS discharge, mean(discharge_percent) AS discharge_percent
|
||||
INTO \"downsampled\".\"water_data_hourly\"
|
||||
FROM \"water_data\"
|
||||
GROUP BY time(1h), station_code, station_name_en, station_name_th
|
||||
END
|
||||
"
|
||||
```
|
||||
|
||||
## 2. VictoriaMetrics Deployment (High Performance)
|
||||
|
||||
### Why VictoriaMetrics?
|
||||
- **Extremely fast** and resource-efficient
|
||||
- **Better compression** than InfluxDB
|
||||
- **Prometheus-compatible** API
|
||||
- **Lower memory usage**
|
||||
- **Built-in clustering**
|
||||
|
||||
### Docker Deployment
|
||||
```yaml
|
||||
# docker-compose.yml
|
||||
version: '3.8'
|
||||
services:
|
||||
victoriametrics:
|
||||
image: victoriametrics/victoria-metrics:latest
|
||||
container_name: water_victoriametrics
|
||||
ports:
|
||||
- "8428:8428"
|
||||
volumes:
|
||||
- vm_data:/victoria-metrics-data
|
||||
command:
|
||||
- '--storageDataPath=/victoria-metrics-data'
|
||||
- '--retentionPeriod=2y'
|
||||
- '--httpListenAddr=:8428'
|
||||
- '--maxConcurrentInserts=16'
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
vm_data:
|
||||
```
|
||||
|
||||
### Environment Variables
|
||||
```bash
|
||||
# .env file
|
||||
DB_TYPE=victoriametrics
|
||||
VM_HOST=localhost
|
||||
VM_PORT=8428
|
||||
```
|
||||
|
||||
## 3. PostgreSQL Deployment (Relational + Time-Series)
|
||||
|
||||
### Why PostgreSQL?
|
||||
- **Mature and reliable**
|
||||
- **Excellent for complex queries**
|
||||
- **TimescaleDB extension** for time-series optimization
|
||||
- **Strong consistency guarantees**
|
||||
- **Rich ecosystem**
|
||||
|
||||
### Docker Deployment with TimescaleDB
|
||||
```yaml
|
||||
# docker-compose.yml
|
||||
version: '3.8'
|
||||
services:
|
||||
postgres:
|
||||
image: timescale/timescaledb:latest-pg14
|
||||
container_name: water_postgres
|
||||
ports:
|
||||
- "5432:5432"
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
- ./init.sql:/docker-entrypoint-initdb.d/init.sql
|
||||
environment:
|
||||
- POSTGRES_DB=water_monitoring
|
||||
- POSTGRES_USER=water_user
|
||||
- POSTGRES_PASSWORD=secure_password
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
```
|
||||
|
||||
### Database Initialization
|
||||
```sql
|
||||
-- init.sql
|
||||
CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE;
|
||||
|
||||
-- Create hypertable for time-series optimization
|
||||
CREATE TABLE water_measurements (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
timestamp TIMESTAMPTZ NOT NULL,
|
||||
station_id INT NOT NULL,
|
||||
water_level NUMERIC(10,3),
|
||||
discharge NUMERIC(10,2),
|
||||
discharge_percent NUMERIC(5,2),
|
||||
status VARCHAR(20) DEFAULT 'active',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Convert to hypertable (TimescaleDB)
|
||||
SELECT create_hypertable('water_measurements', 'timestamp', chunk_time_interval => INTERVAL '1 day');
|
||||
|
||||
-- Create indexes
|
||||
CREATE INDEX idx_water_measurements_station_time ON water_measurements (station_id, timestamp DESC);
|
||||
CREATE INDEX idx_water_measurements_timestamp ON water_measurements (timestamp DESC);
|
||||
|
||||
-- Create retention policy (keep raw data for 2 years)
|
||||
SELECT add_retention_policy('water_measurements', INTERVAL '2 years');
|
||||
|
||||
-- Create continuous aggregates for performance
|
||||
CREATE MATERIALIZED VIEW water_measurements_hourly
|
||||
WITH (timescaledb.continuous) AS
|
||||
SELECT
|
||||
time_bucket('1 hour', timestamp) AS bucket,
|
||||
station_id,
|
||||
AVG(water_level) as avg_water_level,
|
||||
MAX(water_level) as max_water_level,
|
||||
MIN(water_level) as min_water_level,
|
||||
AVG(discharge) as avg_discharge,
|
||||
MAX(discharge) as max_discharge,
|
||||
MIN(discharge) as min_discharge,
|
||||
AVG(discharge_percent) as avg_discharge_percent
|
||||
FROM water_measurements
|
||||
GROUP BY bucket, station_id;
|
||||
|
||||
-- Refresh policy for continuous aggregates
|
||||
SELECT add_continuous_aggregate_policy('water_measurements_hourly',
|
||||
start_offset => INTERVAL '1 day',
|
||||
end_offset => INTERVAL '1 hour',
|
||||
schedule_interval => INTERVAL '1 hour');
|
||||
```
|
||||
|
||||
### Environment Variables
|
||||
```bash
|
||||
# .env file
|
||||
DB_TYPE=postgresql
|
||||
POSTGRES_CONNECTION_STRING=postgresql://water_user:secure_password@localhost:5432/water_monitoring
|
||||
```
|
||||
|
||||
## 4. MySQL Deployment (Traditional Relational)
|
||||
|
||||
### Docker Deployment
|
||||
```yaml
|
||||
# docker-compose.yml
|
||||
version: '3.8'
|
||||
services:
|
||||
mysql:
|
||||
image: mysql:8.0
|
||||
container_name: water_mysql
|
||||
ports:
|
||||
- "3306:3306"
|
||||
volumes:
|
||||
- mysql_data:/var/lib/mysql
|
||||
- ./mysql.cnf:/etc/mysql/conf.d/mysql.cnf
|
||||
- ./init.sql:/docker-entrypoint-initdb.d/init.sql
|
||||
environment:
|
||||
- MYSQL_ROOT_PASSWORD=root_password
|
||||
- MYSQL_DATABASE=water_monitoring
|
||||
- MYSQL_USER=water_user
|
||||
- MYSQL_PASSWORD=water_password
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
mysql_data:
|
||||
```
|
||||
|
||||
### MySQL Configuration
|
||||
```ini
|
||||
# mysql.cnf
|
||||
[mysqld]
|
||||
# Optimize for time-series data
|
||||
innodb_buffer_pool_size = 1G
|
||||
innodb_log_file_size = 256M
|
||||
innodb_flush_log_at_trx_commit = 2
|
||||
innodb_flush_method = O_DIRECT
|
||||
|
||||
# Partitioning support
|
||||
partition = ON
|
||||
|
||||
# Query cache
|
||||
query_cache_type = 1
|
||||
query_cache_size = 128M
|
||||
|
||||
# Connection settings
|
||||
max_connections = 200
|
||||
connect_timeout = 10
|
||||
wait_timeout = 600
|
||||
```
|
||||
|
||||
### Environment Variables
|
||||
```bash
|
||||
# .env file
|
||||
DB_TYPE=mysql
|
||||
MYSQL_CONNECTION_STRING=mysql://water_user:water_password@localhost:3306/water_monitoring
|
||||
```
|
||||
|
||||
## 5. Installation and Dependencies
|
||||
|
||||
### Required Python Packages
|
||||
```bash
|
||||
# Base requirements
|
||||
pip install requests schedule
|
||||
|
||||
# Database-specific packages
|
||||
pip install influxdb # For InfluxDB
|
||||
pip install sqlalchemy pymysql # For MySQL
|
||||
pip install sqlalchemy psycopg2-binary # For PostgreSQL
|
||||
# VictoriaMetrics uses HTTP API (no extra packages needed)
|
||||
```
|
||||
|
||||
### Updated requirements.txt
|
||||
```txt
|
||||
requests>=2.28.0
|
||||
schedule>=1.2.0
|
||||
pandas>=1.5.0
|
||||
|
||||
# Database adapters (install as needed)
|
||||
influxdb>=5.3.1
|
||||
sqlalchemy>=1.4.0
|
||||
pymysql>=1.0.2
|
||||
psycopg2-binary>=2.9.0
|
||||
```
|
||||
|
||||
## 6. Production Deployment Examples
|
||||
|
||||
### Using InfluxDB (Recommended)
|
||||
```bash
|
||||
# Set environment variables
|
||||
export DB_TYPE=influxdb
|
||||
export INFLUX_HOST=your-influx-server.com
|
||||
export INFLUX_PORT=8086
|
||||
export INFLUX_DATABASE=water_monitoring
|
||||
export INFLUX_USERNAME=water_user
|
||||
export INFLUX_PASSWORD=your_secure_password
|
||||
|
||||
# Run the scraper
|
||||
python water_scraper_v3.py
|
||||
```
|
||||
|
||||
### Using PostgreSQL with TimescaleDB
|
||||
```bash
|
||||
# Set environment variables
|
||||
export DB_TYPE=postgresql
|
||||
export POSTGRES_CONNECTION_STRING=postgresql://water_user:password@your-postgres-server.com:5432/water_monitoring
|
||||
|
||||
# Run the scraper
|
||||
python water_scraper_v3.py
|
||||
```
|
||||
|
||||
### Using VictoriaMetrics
|
||||
```bash
|
||||
# Set environment variables
|
||||
export DB_TYPE=victoriametrics
|
||||
export VM_HOST=your-vm-server.com
|
||||
export VM_PORT=8428
|
||||
|
||||
# Run the scraper
|
||||
python water_scraper_v3.py
|
||||
```
|
||||
|
||||
## 7. Monitoring and Alerting
|
||||
|
||||
### Grafana Dashboard Setup
|
||||
1. **Add Data Source**: Configure your database as a Grafana data source
|
||||
2. **Import Dashboard**: Use pre-built water monitoring dashboards
|
||||
3. **Set Alerts**: Configure alerts for abnormal water levels or discharge rates
|
||||
|
||||
### Example Grafana Queries
|
||||
|
||||
#### InfluxDB Queries
|
||||
```sql
|
||||
-- Current water levels
|
||||
SELECT last("water_level") FROM "water_data" GROUP BY "station_code"
|
||||
|
||||
-- Discharge trends (last 24h)
|
||||
SELECT mean("discharge") FROM "water_data" WHERE time >= now() - 24h GROUP BY time(1h), "station_code"
|
||||
```
|
||||
|
||||
#### PostgreSQL/TimescaleDB Queries
|
||||
```sql
|
||||
-- Current water levels
|
||||
SELECT DISTINCT ON (station_id)
|
||||
station_id, water_level, discharge, timestamp
|
||||
FROM water_measurements
|
||||
ORDER BY station_id, timestamp DESC;
|
||||
|
||||
-- Hourly averages (last 24h)
|
||||
SELECT
|
||||
time_bucket('1 hour', timestamp) as hour,
|
||||
station_id,
|
||||
AVG(water_level) as avg_level,
|
||||
AVG(discharge) as avg_discharge
|
||||
FROM water_measurements
|
||||
WHERE timestamp >= NOW() - INTERVAL '24 hours'
|
||||
GROUP BY hour, station_id
|
||||
ORDER BY hour DESC;
|
||||
```
|
||||
|
||||
## 8. Performance Optimization Tips
|
||||
|
||||
### For All Databases
|
||||
- **Batch inserts**: Insert multiple measurements at once
|
||||
- **Connection pooling**: Reuse database connections
|
||||
- **Indexing**: Ensure proper indexes on timestamp and station_id
|
||||
- **Retention policies**: Automatically delete old data
|
||||
|
||||
### InfluxDB Specific
|
||||
- Use **tags** for metadata (station codes, names)
|
||||
- Use **fields** for numeric values (water levels, discharge)
|
||||
- Configure **retention policies** and **continuous queries**
|
||||
- Enable **compression** for long-term storage
|
||||
|
||||
### PostgreSQL/TimescaleDB Specific
|
||||
- Use **hypertables** for automatic partitioning
|
||||
- Create **continuous aggregates** for common queries
|
||||
- Configure **compression** for older chunks
|
||||
- Use **parallel queries** for large datasets
|
||||
|
||||
### VictoriaMetrics Specific
|
||||
- Use **labels** efficiently (similar to Prometheus)
|
||||
- Configure **retention periods** appropriately
|
||||
- Use **downsampling** for long-term storage
|
||||
- Enable **deduplication** if needed
|
||||
|
||||
This deployment guide provides production-ready configurations for all supported database backends. Choose the one that best fits your infrastructure and requirements.
|
||||
+70
-14
@@ -23,10 +23,29 @@ The only source persisted to the database and used by the ML pipeline.
|
||||
| Params | `DW[UtokID]=1`, `DW[BasinID]=6` (Ping), `DW[TimeCurrent]=<Buddhist-calendar date>`, `rows=100` |
|
||||
| Variables | Water level (m, gauge datum), discharge (m³/s, `'***'` = malformed), discharge % of channel capacity |
|
||||
| Stations | 16 P-series gauges (P.1 anchor at Nawarat Bridge; see `src/data/stations.json`) |
|
||||
| Client | `src/water_scraper_v3.py` |
|
||||
| Client | `rust/collector/src/rid.rs` (`ping-collector`; `fill-gaps` re-fetches old reports with holes, `refetch FROM [TO]` any reports) |
|
||||
|
||||
Human-facing page: <https://hyd-app-db.rid.go.th/hydro1h.html>
|
||||
|
||||
**Spikes.** The report sometimes carries a wrong value for one hour and corrects
|
||||
it later: a slipped decimal point (P.1 10.6 m between 1.04 and 1.07 on 2026-10-25
|
||||
00:00, later 1.06; P.85 0.101 for 1.1), a flipped sign (P.4A ±1.18, P.67 ±0.6) or
|
||||
a placeholder (−5, −6, −9, 50 m). The collector re-fetches a report only while it
|
||||
has holes, so a later correction never arrived. Since 2026-10-26 every batch is
|
||||
screened before it is stored (`rid::judge`, `rid::screen`): a reading that leaves
|
||||
the previous one (within 3 h) by more than 1 m (2 m on the flashy tributaries
|
||||
P.76, P.77, P.81, P.82, P.84, P.85, P.87) while the next reading comes back near
|
||||
the previous level is not stored, and a stored copy is deleted. A rising or falling
|
||||
limb, or a step the next reading confirms, is kept. The newest reading of a gauge
|
||||
waits for the next one when it jumps more than twice the limit. On the 2018–2026
|
||||
history the rule finds 16 spikes and would have rejected no genuine change (the
|
||||
largest genuine 1-h changes: 1.1 m on the mainstem, 3.1 m at P.76). Two wrong hours
|
||||
in a row are not caught.
|
||||
|
||||
Evening hours often arrive level-only (no discharge) and gain the discharge
|
||||
later; the API's history fills a missing discharge with 20 × level (an old
|
||||
placeholder, kept for compatibility). `refetch` brings in RID's discharge.
|
||||
|
||||
---
|
||||
|
||||
## 2. ThaiWater / HII ecosystem
|
||||
@@ -117,9 +136,9 @@ GET .../rain_24h_graph?station_type=tele_rainfall&station_id=418&start_date=...&
|
||||
|
||||
### 2.2 `twa-api-public.thaiwater.net` — auth-gated (x-api-key / session) 🔒
|
||||
|
||||
The layer our existing `src/thaiwater.py` client uses
|
||||
(`GET /v2/waterlevel` with `x-api-key: $THAIWATER_API_KEY`; wired to
|
||||
`GET /sensors/thaiwater` in the web API, display-only, never persisted).
|
||||
The layer the website's `GET /sensors/thaiwater` uses (`GET /v2/waterlevel`
|
||||
with an `x-api-key` header from `THAIWATER_API_KEY`;
|
||||
`rust/collector/src/web/hii.rs`, display-only, never persisted).
|
||||
Without a key: HTTP 401/500. No public key-registration page was found —
|
||||
obtain a sanctioned key from HII (<https://hii.or.th>).
|
||||
|
||||
@@ -235,6 +254,39 @@ Oct 2024 flood). Mae Kuang Udom Thara is the second upstream reservoir.
|
||||
| `https://hydro-1.net` | RID Upper-Northern Hydrology Center — hourly/daily tables, hydrology yearbooks (rating curves) for P-series stations; scrape/download |
|
||||
| `https://water.rid.go.th/flood/flood/daily.pdf` | RID daily flood bulletin (PDF only) |
|
||||
|
||||
### 5.1 CMU Water Center — CMFlood (Chiang Mai city) (verified 2026-10-25)
|
||||
|
||||
`https://watercenter.cmu.ac.th/chiangmai/cmflood/warning/*`, the Chiang Mai urban
|
||||
flood warning site of Chiang Mai University's water center (with RID, ONWR, DDPM
|
||||
and the province). A Next.js app; its data paths are in the page chunks
|
||||
(`/_next/static/chunks/app/...`). No licence is stated; we credit CMU.
|
||||
|
||||
- **Flood hazard map, 2024 flood** — `/data/KML5Layer/L1.kml` … `L5.kml`: the
|
||||
flooded extent at P.1 = 4.30 / 4.50 / 4.70 / 5.00 / 5.30 m (2.5 / 3.0 / 22 / 57 /
|
||||
156 km², cumulative). CMU: the Ping overflows its banks above P.1 4.20 m, and the
|
||||
older municipal map (2011 flood) no longer fits the city's defences. **In use:**
|
||||
`P1_FLOOD_STAGES` and `src/static/flood-zones-p1.geojson`
|
||||
(`uv run scripts/build_flood_zones.py`).
|
||||
- **Flood marks 2024** — `/data/New_floodMark_2025_fixed_like_pole.geojson` (4.9 MB):
|
||||
4,869 points in Chiang Mai and Lamphun with the October 2024 depth above the road
|
||||
in cm (P.1 = 5.30 m), the marker's place and type. **In use:** the dashboard's
|
||||
"Flood marks 2024" layer, `src/static/flood-marks-2024.json` (position and depth
|
||||
only, 110 KB; `uv run python scripts/build_flood_marks.py`). The depth does not
|
||||
scale one-to-one with P.1: marks in the zone that floods only at 5.30 m stood a
|
||||
median 65 cm deep in 2024, so the site shows the 2024 depth and no estimate for
|
||||
other levels.
|
||||
- **Flood poles** — `/data/PoleCNX2025_v2.geojson`: 219 poles with ground level and
|
||||
maximum flood level (m MSL) and the depth.
|
||||
- **City-reach gauges** (station page): P.67, P.103, P.1, P.104 (RID, `ST.16`
|
||||
every 15 min), FloodBoy FBP.2/FBP.3 (CMU climate data centre) and DDPM CMI01–03
|
||||
(camera gauges, ~5 min, public CCTV snapshots), with gauge zero, bank levels
|
||||
and chainage. CMU reads them from its own Supabase project with a public browser
|
||||
key (tables `disaster_level`, `flagship_hydro`, `water_levels`, `hydro_levels`;
|
||||
timestamps are Bangkok time labelled `+00:00`); history starts 2026-08 (FBP
|
||||
2025-09). Not ingested: too short to train on, and polling CMU's database is
|
||||
using their infrastructure — prefer the original sources or ask CMU.
|
||||
- **P.1 forecast** (`floodforecast`): login only.
|
||||
|
||||
---
|
||||
|
||||
## 6. Integration status & recommended order
|
||||
@@ -242,21 +294,25 @@ Oct 2024 flood). Mae Kuang Udom Thara is the second upstream reservoir.
|
||||
| Source | Status | Action |
|
||||
|---|---|---|
|
||||
| RID hourly gauges | ✅ Ingested (hourly → PostgreSQL) | — |
|
||||
| ThaiWater `/v2/waterlevel` | 🟡 Display-only (`src/thaiwater.py`, needs `THAIWATER_API_KEY`, never persisted) | Optionally persist |
|
||||
| HII `rain_24h` | ✅ Ingested hourly via `src/hii_collector.py` → `hii_rain_stations` + `hii_rainfall` (Ping-filtered; ~300 stations) | — |
|
||||
| HII `waterlevel_load` | ✅ Ingested hourly via `src/hii_collector.py` → `hii_wl_stations` + `hii_waterlevel` (125 Ping stations, m MSL; `rid_code` column maps mirrors like `ridhydro_P.1` → `P.1`, `offset_msl` converts MSL → gauge datum) | — |
|
||||
| ThaiWater `/v2/waterlevel` | 🟡 Display-only (`/sensors/thaiwater`, needs `THAIWATER_API_KEY`, never persisted) | Optionally persist |
|
||||
| HII `rain_24h` | ✅ Ingested hourly by ping-collector (`rust/collector/src/hii.rs`) → `hii_rain_stations` + `hii_rainfall` (Ping-filtered; ~300 stations) | — |
|
||||
| HII `waterlevel_load` | ✅ Ingested hourly by ping-collector (`rust/collector/src/hii.rs`) → `hii_wl_stations` + `hii_waterlevel` (125 Ping stations, m MSL; `rid_code` column maps mirrors like `ridhydro_P.1` → `P.1`, `offset_msl` converts MSL → gauge datum) | — |
|
||||
| HII `waterlevel_graph` | ✅ Backfill via `scripts/backfill_hii_waterlevel.py` → `hii_waterlevel` (hourly MSL + discharge, archive ≥2019; full-year windows per request; upserts never overwrite live-snapshot columns) | Run once on the box: `python scripts/backfill_hii_waterlevel.py` (defaults: 2019-01-01 → today, RID-mirror + key stations; `--stations P.1,P.67`, `--all` for every Ping station) |
|
||||
| Mae Ngat reservoir (app.rid.go.th) | ✅ Ingested (2026-08-13) | Daily storage/inflow/outflow for all large dams → `rid_reservoir_daily`; candidate model features for next retrain |
|
||||
| Satellite QPE (GSMaP/IMERG) | ❌ | Backfill training rainfall (GEE) |
|
||||
| Open-Meteo forecasts | ❌ | Add forecast features (live + 2021 archive for training) |
|
||||
| Open-Meteo rain | ✅ Hourly by ping-collector → `openmeteo_rain` (5 catchment points, past 5 days + 48 h forecast); the 2021+ archive trains the hgb-v3 rain features | — |
|
||||
| HII API key (dams, forecasts) | ❌ | Contact HII for sanctioned access |
|
||||
| CMU CMFlood flood map (2024) | ✅ Static (2026-10-25): P.1 stages and the city's flood zones | Rebuild with `scripts/build_flood_zones.py` if CMU revises the map |
|
||||
| CMU CMFlood flood marks 2024 | ✅ Static (2026-10-25): dashboard layer, nearest mark in zone and marker popups | Rebuild with `scripts/build_flood_marks.py` |
|
||||
| CMU CMFlood poles, city gauges | ❌ | See 5.1 |
|
||||
|
||||
**Collector configuration** (`src/hii_collector.py`): runs automatically every
|
||||
scraping cycle (hourly cadence, even while the RID scraper is in 1-minute retry
|
||||
mode) from both `--web-api` and continuous-monitoring modes; one-shot via
|
||||
`python -m src.main --collect-hii`. Env vars: `ENABLE_HII_COLLECTION`
|
||||
(default `true`), `HII_BASIN_CODE` (default `6` = Ping). Requires a SQL
|
||||
`DB_TYPE` (sqlite/postgresql/mysql); tables are created automatically.
|
||||
**Collector configuration** (ping-collector): HII, the dams and Open-Meteo are
|
||||
fetched once per clock hour, also while RID is being retried; one-shot via
|
||||
`ping-collector once`. Env vars: `ENABLE_HII_COLLECTION`,
|
||||
`ENABLE_RESERVOIR_COLLECTION`, `ENABLE_RAIN_COLLECTION` (default `true`),
|
||||
`HII_BASIN_CODE` (default `6` = Ping). Tables are created automatically
|
||||
(PostgreSQL). The archive backfills stay in Python (`scripts/backfill_*.py`,
|
||||
using `src/hii_collector.py` and `src/rid_reservoir.py`).
|
||||
|
||||
**Caveats**: `api-v3` is an undocumented backend — no SLA, no ToS, may change
|
||||
without notice. Poll hourly at most, cache aggressively, and pursue official
|
||||
|
||||
@@ -1,329 +0,0 @@
|
||||
# Debian/Linux Troubleshooting Guide
|
||||
|
||||
This guide addresses common issues when running the Thailand Water Monitor on Debian and other Linux distributions.
|
||||
|
||||
## Fixed Issues
|
||||
|
||||
### SQLAlchemy Connection Error (RESOLVED)
|
||||
|
||||
**Error Message:**
|
||||
```
|
||||
2025-07-24 19:48:31,920 - ERROR - Failed to connect to SQLITE: 'Connection' object has no attribute 'commit'
|
||||
2025-07-24 19:48:32,740 - ERROR - Error saving to SQLITE: 'Connection' object has no attribute 'commit'
|
||||
```
|
||||
|
||||
**Root Cause:**
|
||||
This error occurred due to incompatibility between the database adapter code and newer versions of SQLAlchemy. The code was calling `conn.commit()` on a connection object that doesn't have a `commit()` method in newer SQLAlchemy versions.
|
||||
|
||||
**Solution Applied:**
|
||||
Changed from `engine.connect()` to `engine.begin()` context manager, which automatically handles transactions:
|
||||
|
||||
```python
|
||||
# OLD (problematic) code:
|
||||
with self.engine.connect() as conn:
|
||||
conn.execute(text(sql))
|
||||
conn.commit() # This fails in newer SQLAlchemy
|
||||
|
||||
# NEW (fixed) code:
|
||||
with self.engine.begin() as conn:
|
||||
conn.execute(text(sql))
|
||||
# Transaction automatically committed when context exits
|
||||
```
|
||||
|
||||
**Status:** ✅ **FIXED** - The issue has been resolved in the current version.
|
||||
|
||||
## Installation on Debian/Ubuntu
|
||||
|
||||
### System Requirements
|
||||
|
||||
```bash
|
||||
# Update package list
|
||||
sudo apt update
|
||||
|
||||
# Install Python and pip
|
||||
sudo apt install python3 python3-pip python3-venv
|
||||
|
||||
# Install system dependencies for database drivers
|
||||
sudo apt install build-essential python3-dev
|
||||
|
||||
# For MySQL support (optional)
|
||||
sudo apt install default-libmysqlclient-dev
|
||||
|
||||
# For PostgreSQL support (optional)
|
||||
sudo apt install libpq-dev
|
||||
```
|
||||
|
||||
### Python Environment Setup
|
||||
|
||||
```bash
|
||||
# Create virtual environment
|
||||
python3 -m venv water_monitor_env
|
||||
|
||||
# Activate virtual environment
|
||||
source water_monitor_env/bin/activate
|
||||
|
||||
# Install requirements
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
### Running the Monitor
|
||||
|
||||
```bash
|
||||
# Test run
|
||||
python water_scraper_v3.py --test
|
||||
|
||||
# Run with specific database
|
||||
export DB_TYPE=sqlite
|
||||
python water_scraper_v3.py
|
||||
|
||||
# Run demo
|
||||
python demo_databases.py
|
||||
```
|
||||
|
||||
## Common Linux Issues
|
||||
|
||||
### 1. Permission Errors
|
||||
|
||||
**Error:**
|
||||
```
|
||||
PermissionError: [Errno 13] Permission denied: 'water_levels.db'
|
||||
```
|
||||
|
||||
**Solution:**
|
||||
```bash
|
||||
# Check current directory permissions
|
||||
ls -la
|
||||
|
||||
# Create data directory with proper permissions
|
||||
mkdir -p data
|
||||
chmod 755 data
|
||||
|
||||
# Set database path to data directory
|
||||
export WATER_DB_PATH=data/water_levels.db
|
||||
```
|
||||
|
||||
### 2. Missing System Dependencies
|
||||
|
||||
**Error:**
|
||||
```
|
||||
ImportError: No module named '_sqlite3'
|
||||
```
|
||||
|
||||
**Solution:**
|
||||
```bash
|
||||
# Install SQLite development headers
|
||||
sudo apt install libsqlite3-dev
|
||||
|
||||
# Reinstall Python if needed
|
||||
sudo apt install python3-sqlite
|
||||
```
|
||||
|
||||
### 3. Network/Firewall Issues
|
||||
|
||||
**Error:**
|
||||
```
|
||||
requests.exceptions.ConnectionError: HTTPSConnectionPool
|
||||
```
|
||||
|
||||
**Solution:**
|
||||
```bash
|
||||
# Test network connectivity
|
||||
curl -I https://hyd-app-db.rid.go.th/hydro1h.html
|
||||
|
||||
# Check firewall rules
|
||||
sudo ufw status
|
||||
|
||||
# Allow outbound HTTPS if needed
|
||||
sudo ufw allow out 443
|
||||
```
|
||||
|
||||
### 4. Systemd Service Setup
|
||||
|
||||
Create service file `/etc/systemd/system/water-monitor.service`:
|
||||
|
||||
```ini
|
||||
[Unit]
|
||||
Description=Thailand Water Level Monitor
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=water-monitor
|
||||
Group=water-monitor
|
||||
WorkingDirectory=/opt/water_level_monitor
|
||||
Environment=PATH=/opt/water_level_monitor/venv/bin
|
||||
Environment=DB_TYPE=sqlite
|
||||
Environment=WATER_DB_PATH=/opt/water_level_monitor/data/water_levels.db
|
||||
ExecStart=/opt/water_level_monitor/venv/bin/python water_scraper_v3.py
|
||||
Restart=always
|
||||
RestartSec=60
|
||||
|
||||
# Security settings
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
ReadWritePaths=/opt/water_level_monitor/data
|
||||
ReadWritePaths=/opt/water_level_monitor/logs
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
```
|
||||
|
||||
Enable and start:
|
||||
```bash
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable water-monitor.service
|
||||
sudo systemctl start water-monitor.service
|
||||
sudo systemctl status water-monitor.service
|
||||
```
|
||||
|
||||
### 5. Log Rotation
|
||||
|
||||
Create `/etc/logrotate.d/water-monitor`:
|
||||
|
||||
```
|
||||
/opt/water_level_monitor/water_monitor.log {
|
||||
daily
|
||||
missingok
|
||||
rotate 30
|
||||
compress
|
||||
delaycompress
|
||||
notifempty
|
||||
create 644 water-monitor water-monitor
|
||||
postrotate
|
||||
systemctl reload water-monitor.service
|
||||
endscript
|
||||
}
|
||||
```
|
||||
|
||||
## Database-Specific Issues
|
||||
|
||||
### SQLite
|
||||
|
||||
**Issue:** Database locked
|
||||
```bash
|
||||
# Check for processes using the database
|
||||
sudo lsof /path/to/water_levels.db
|
||||
|
||||
# Kill processes if needed
|
||||
sudo pkill -f water_scraper_v3.py
|
||||
```
|
||||
|
||||
### VictoriaMetrics with HTTPS
|
||||
|
||||
**Configuration:**
|
||||
```bash
|
||||
export DB_TYPE=victoriametrics
|
||||
export VM_HOST=https://your-vm-server.com
|
||||
export VM_PORT=443
|
||||
```
|
||||
|
||||
**Test connection:**
|
||||
```bash
|
||||
curl -k https://your-vm-server.com/health
|
||||
```
|
||||
|
||||
## Performance Optimization
|
||||
|
||||
### 1. System Tuning
|
||||
|
||||
```bash
|
||||
# Increase file descriptor limits
|
||||
echo "* soft nofile 65536" >> /etc/security/limits.conf
|
||||
echo "* hard nofile 65536" >> /etc/security/limits.conf
|
||||
|
||||
# Optimize network settings
|
||||
echo "net.core.rmem_max = 16777216" >> /etc/sysctl.conf
|
||||
echo "net.core.wmem_max = 16777216" >> /etc/sysctl.conf
|
||||
sysctl -p
|
||||
```
|
||||
|
||||
### 2. Database Optimization
|
||||
|
||||
```bash
|
||||
# For SQLite
|
||||
export SQLITE_CACHE_SIZE=10000
|
||||
export SQLITE_SYNCHRONOUS=NORMAL
|
||||
|
||||
# Monitor database size
|
||||
du -h data/water_levels.db
|
||||
```
|
||||
|
||||
## Monitoring and Maintenance
|
||||
|
||||
### Health Check Script
|
||||
|
||||
Create `health_check.sh`:
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
LOG_FILE="/opt/water_level_monitor/water_monitor.log"
|
||||
SERVICE_NAME="water-monitor"
|
||||
|
||||
# Check if service is running
|
||||
if ! systemctl is-active --quiet $SERVICE_NAME; then
|
||||
echo "ERROR: $SERVICE_NAME is not running"
|
||||
systemctl restart $SERVICE_NAME
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check recent log entries
|
||||
RECENT_ERRORS=$(tail -n 100 $LOG_FILE | grep -c "ERROR")
|
||||
if [ $RECENT_ERRORS -gt 5 ]; then
|
||||
echo "WARNING: $RECENT_ERRORS errors found in recent logs"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "OK: Service is healthy"
|
||||
exit 0
|
||||
```
|
||||
|
||||
### Cron Job for Health Checks
|
||||
|
||||
```bash
|
||||
# Add to crontab
|
||||
*/5 * * * * /opt/water_level_monitor/health_check.sh >> /var/log/water-monitor-health.log 2>&1
|
||||
```
|
||||
|
||||
## Getting Help
|
||||
|
||||
### Debug Information
|
||||
|
||||
```bash
|
||||
# System information
|
||||
uname -a
|
||||
python3 --version
|
||||
pip list | grep -E "(sqlalchemy|requests|influxdb)"
|
||||
|
||||
# Service logs
|
||||
journalctl -u water-monitor.service -f
|
||||
|
||||
# Application logs
|
||||
tail -f water_monitor.log
|
||||
|
||||
# Database information
|
||||
sqlite3 water_levels.db ".schema"
|
||||
sqlite3 water_levels.db "SELECT COUNT(*) FROM water_measurements;"
|
||||
```
|
||||
|
||||
### Common Commands
|
||||
|
||||
```bash
|
||||
# Restart service
|
||||
sudo systemctl restart water-monitor.service
|
||||
|
||||
# View logs
|
||||
sudo journalctl -u water-monitor.service --since "1 hour ago"
|
||||
|
||||
# Test configuration
|
||||
python config.py
|
||||
|
||||
# Test database connection
|
||||
python demo_databases.py
|
||||
|
||||
# Manual data fetch
|
||||
python water_scraper_v3.py --test
|
||||
```
|
||||
|
||||
This troubleshooting guide should help resolve most common issues encountered when running the Thailand Water Monitor on Debian and other Linux distributions.
|
||||
+96
-63
@@ -62,8 +62,8 @@ P.5 before 2022-01-01; the rest are handled by the per-row coverage gates.
|
||||
|
||||
**How `src/ml/data.py` loads it.** `resolve_db_url()` picks a connection string in
|
||||
priority order: an explicit `--db-url` argument, then the `FLOOD_ML_DB_URL`
|
||||
environment variable, then `Config.get_database_config()` when `DB_TYPE` is
|
||||
`postgresql`, else `None`. `load_measurements()` then tries three tiers:
|
||||
environment variable, then the `POSTGRES_*` settings (`src/config.py`, the same
|
||||
`.env` the service reads), else `None`. `load_measurements()` then tries three tiers:
|
||||
|
||||
1. **PostgreSQL** (`_fetch_from_db`) — the primary path. NULL discharge stays
|
||||
NULL, which matters because the models must learn from the real missingness
|
||||
@@ -465,15 +465,19 @@ validated.
|
||||
**Thresholds are per-station as of 2026-08-10.** `THRESHOLDS` now carries
|
||||
calibrated (warning, danger) pairs for all 16 stations, derived from the DB's
|
||||
`discharge_percent` (RID % of channel capacity): warning = median level at
|
||||
75–85% capacity, danger = median level at 95–105%. P.1 instead uses the official
|
||||
Chiang Mai inundation map (`P1_FLOOD_STAGES`): warning 3.70 m (city flooding
|
||||
begins, stage 1) and danger 4.20 m (stage 5). The prior single default of
|
||||
75–85% capacity, danger = median level at 95–105%. P.1 is keyed to its banks:
|
||||
danger 4.20 m, where the Ping overflows them (CMU Water Center's flood map of the
|
||||
2024 flood; the lower bank is 304.65 m MSL = 4.15 m on the gauge), and warning
|
||||
3.70 m, half a metre below. (Until 2026-10-25 the docs called 3.70 m "city flooding
|
||||
begins", after the older municipal map; the values did not change.) The prior single default of
|
||||
(3.0, 4.5) m made P.103 badly over-alert (its bank-full level is ~6.75 m) and
|
||||
P.67 under-alert (overflow at ~2.9 m, 1.6 m below the old danger line).
|
||||
**A retrain is required after any threshold change** — classifier labels depend
|
||||
on them; until then, model rows report the thresholds baked into their bundle.
|
||||
P.1 additionally reports `stages`: exceedance probability for each of the seven
|
||||
official inundation stages (3.70–4.60 m), computed from the regression head and
|
||||
P.1 additionally reports `stages`: exceedance probability for each of the five
|
||||
city flood stages (`P1_FLOOD_STAGES`, 4.30–5.30 m, from CMU Water Center's flood
|
||||
hazard map of the October 2024 flood; the seven stages of the older municipal map,
|
||||
3.70–4.60 m, were replaced on 2026-10-25), computed from the regression head and
|
||||
its calibration sigma, so they need no retrain and no per-stage classifiers.
|
||||
|
||||
### 2026-08-13: Mae Ngat dam features — a documented negative result
|
||||
@@ -614,46 +618,73 @@ fix, the served features match what this section evaluates.
|
||||
|
||||
## 6. Deployment
|
||||
|
||||
### API
|
||||
### Serving
|
||||
|
||||
With `FORECAST_IN_COLLECTOR=true` the forecast is computed by `ping-collector`
|
||||
(`rust/forecast`, a parity-tested port of this serving path) from
|
||||
The forecast is computed by `ping-collector` (`rust/forecast`, a parity-tested
|
||||
port of `src/ml/predict.py`, which stays as the reference implementation) from
|
||||
`models/flood_<station>.json.gz`, which `train_all()` exports next to each
|
||||
joblib bundle; `/forecast` serves its `forecast_current` row and computes
|
||||
in-process only when that is older than `FORECAST_MAX_AGE_SECONDS` (2 h). See
|
||||
`rust/README.md`. The rest of this section describes the in-process path.
|
||||
joblib bundle. It runs after every collection cycle that brought newer
|
||||
readings and at least hourly, stores the rows in `forecast_history` (section 9's
|
||||
skill tracking) and `forecast_current`, and `GET /forecast` serves the stored
|
||||
row: one JSON row per station × horizon with the fields listed in section 1.
|
||||
A stored forecast older than `FORECAST_MAX_AGE_SECONDS` (2 h, i.e. the service
|
||||
has stopped forecasting) is still served, marked `X-Data-Stale: true`. See
|
||||
`rust/README.md`.
|
||||
|
||||
`GET /forecast` (`src/web_api.py`) returns one JSON row per station × horizon with
|
||||
the fields listed in section 1. Results are cached in-process for
|
||||
`FORECAST_TTL = 900` seconds (15 minutes), which matches the data cadence — the
|
||||
underlying readings do not update faster than hourly. Inference runs in a thread
|
||||
via `asyncio.to_thread` so it never blocks the event loop.
|
||||
**The issued history has every hour.** A run forecasts from the newest reading
|
||||
only, and RID publishes 22:00 to 06:00 in one batch in the morning (and now and
|
||||
then reports a single hour late), so those hours were never forecast from and
|
||||
the 24 h peak "as issued" had a gap every night. After each run the collector
|
||||
therefore forecasts the hours of the last 48 h (two weeks after a start) that
|
||||
have a RID reading but no model forecast, each as a run at that hour would
|
||||
have: gauges, HII fill and dam state up to that hour only. The exception is
|
||||
rain: it is fetched once, so a late hour's `rain_fc24` partly holds
|
||||
Open-Meteo's estimate of rain that has since fallen, as in training, where
|
||||
the forward rain is observed rain. Such rows are written only where no model
|
||||
row exists; `created_at` tells them apart (more than an hour after `as_of`).
|
||||
`ping-collector catch-up-forecasts --hours N` does the same further back (up
|
||||
to 60 days); with `--redo-from T` it first deletes the model forecasts issued
|
||||
from T on, to recompute them after `ping-collector refetch` corrected the
|
||||
readings they came from (a bad reading reaches the features for 72 h through
|
||||
`roll_max_72`).
|
||||
|
||||
Failure modes: **503** if the `src.ml` package cannot be imported (missing
|
||||
scikit-learn, say), **502** on any other exception.
|
||||
A model row in `forecast_history` is never replaced by a heuristic one. While
|
||||
RID is silent overnight each hourly run re-issues the 21:00 reading, and once
|
||||
it is 6 h old (`stale_after_h`) as the heuristic; that used to overwrite the
|
||||
21:00 model forecast every night.
|
||||
|
||||
One behaviour worth knowing, because the code comments suggest otherwise: the
|
||||
endpoint's `FileNotFoundError` ("No trained flood models found") and `RuntimeError`
|
||||
handlers are unreachable — nothing in `src/ml/` raises either, and
|
||||
`predict._forecast_station` checks `bundle_path.exists()` and falls back to the
|
||||
heuristic instead. So **before the first training run `/forecast` returns 200 with
|
||||
an all-heuristic payload**, not a 503, provided there is recent gauge data; you
|
||||
get an empty `200 []` only when there is no recent data at all. Judge deployment
|
||||
state by the `source` field, not the status code.
|
||||
A station without a model bundle, or whose bundle needs a feature the Rust side
|
||||
cannot build, gets the heuristic, as in Python. So **before the first training
|
||||
run `/forecast` returns 200 with an all-heuristic payload**, provided there is
|
||||
recent gauge data; you get an empty `200 []` only when there is no recent data
|
||||
at all. Judge deployment state by the `source` field, not the status code.
|
||||
|
||||
The Python serving path (`predict.get_latest_forecasts`) is not called by the
|
||||
service any more; `scripts/forecast_parity.py` runs both on the live database
|
||||
and `scripts/install.sh` does that on every install.
|
||||
|
||||
### Dashboard
|
||||
|
||||
The "Flood risk outlook" panel (`src/static/dashboard.html`, `loadForecasts()`)
|
||||
loads non-blocking after the map renders and **stays hidden unless `/forecast`
|
||||
returns a non-empty array** — a non-OK response, an empty array, or a thrown
|
||||
fetch all just leave the panel hidden, and the rest of the dashboard is
|
||||
unaffected. Per the note above, this means the panel appears with heuristic-only
|
||||
content once data is flowing but before any model is trained; the per-chip
|
||||
tooltip is what tells you so. Stations are sorted worst-risk first, each showing
|
||||
three chips (6/12/24 h) coloured by risk band, with the tooltip carrying the exact
|
||||
warning and danger percentages, the predicted peak level, and a "heuristic
|
||||
fallback" note when the row did not come from a model. The panel is labelled
|
||||
*experimental*.
|
||||
The dashboard (`src/static/dashboard.html`) loads `/forecast` with the gauges; if
|
||||
it fails or is empty, the page works without forecasts. Where the model shows:
|
||||
|
||||
- **Top bar.** Chiang Mai's verdict from P.1, in the classes of the gauge status:
|
||||
flooding (over the banks) at ≥ 4.20 m; high water at ≥ 3.70 m; "flooding likely" at
|
||||
a 24 h danger chance ≥ 50 %, "high water likely" at a warning chance ≥ 70 %;
|
||||
"watch" at a warning chance ≥ 20 % or a level ≥ 3.20 m. Next to it, the highest
|
||||
24 h warning chance of any gauge.
|
||||
- **Gauge list and map.** A gauge is "watch" from a 24 h warning chance of 25 %.
|
||||
- **Station panel.** Per horizon (6/12/24 h) the predicted peak and the warning and
|
||||
danger chances, the model version and when it was trained, or "Simple estimate" when
|
||||
the row came from the heuristic fallback. For P.1 the chance of reaching each of the
|
||||
five city flood stages. The history chart overlays the 24 h peaks as they were
|
||||
issued (`/api/forecast/history`), and "How accurate is the forecast?" loads
|
||||
`/api/forecast/skill`.
|
||||
- **Replay.** "Replay the October 2024 flood" plays `src/static/flood-2024-all.json`:
|
||||
the gauge readings of 2024 with what a model trained only on data before
|
||||
1 September 2024 said at each hour.
|
||||
|
||||
The page labels the forecast experimental.
|
||||
|
||||
### Training on the server
|
||||
|
||||
@@ -661,13 +692,14 @@ The server already has the PostgreSQL connection configured, so no host override
|
||||
is needed:
|
||||
|
||||
```bash
|
||||
cd /path/to/Northern-Thailand-Ping-River-Monitor
|
||||
python scripts/train_flood_model.py --stations all
|
||||
cd /opt/thailand-water-monitor
|
||||
uv run python scripts/train_flood_model.py --stations all
|
||||
```
|
||||
|
||||
`resolve_db_url()` picks up `Config.get_database_config()` automatically when
|
||||
`DB_TYPE=postgresql`. The run writes fifteen `models/flood_{station}.joblib`
|
||||
bundles plus `models/metrics.json`.
|
||||
`resolve_db_url()` picks up the `POSTGRES_*` settings from `.env`. The run writes
|
||||
fifteen `models/flood_{station}.joblib` bundles, their `.json.gz` exports for the
|
||||
service, and `models/metrics.json`. The monthly timer does the same through
|
||||
`scripts/retrain.sh` with a staged promote (section 7).
|
||||
|
||||
### Artifacts and dependencies
|
||||
|
||||
@@ -733,7 +765,10 @@ gap between fetch and first model could not be reconstructed from the surviving
|
||||
artifacts. Do not attribute it to per-station training cost. Either way the
|
||||
conclusion holds: **retraining is minutes, not tens of minutes.**
|
||||
|
||||
**Inference** (15 bundles, 16 stations × 3 horizons = 48 rows):
|
||||
**Inference in Python** (15 bundles, 16 stations × 3 horizons = 48 rows; the
|
||||
retired in-process serving path, measured before the Rust forecaster took
|
||||
over: there the whole run takes milliseconds, 0.25 ms per station against
|
||||
158 ms, and the 15 exported models need about 70 MB resident):
|
||||
|
||||
| Measurement | Value |
|
||||
|---|---|
|
||||
@@ -753,9 +788,9 @@ one-time lazy `sklearn.ensemble` import — the remaining fourteen average 159 m
|
||||
Of the ~640 ms warm compute, model prediction is ~525 ms, the hourly grid 47 ms,
|
||||
and feature building 71 ms across all sixteen stations.
|
||||
|
||||
**Live-endpoint measurements** (a second, independent benchmark run against a real
|
||||
uvicorn instance of the app, same day, same workstation, RSS summed over the
|
||||
process tree):
|
||||
**Live-endpoint measurements of the retired Python API** (a second, independent
|
||||
benchmark run against a real uvicorn instance of the app, same day, same
|
||||
workstation, RSS summed over the process tree):
|
||||
|
||||
| Measurement | Value |
|
||||
|---|---|
|
||||
@@ -775,14 +810,11 @@ Training shows the opposite (2.3× slower single-threaded), so set the variable
|
||||
per process, never globally.
|
||||
|
||||
**Server sizing, in plain terms:** this is a small workload and almost any server
|
||||
runs it. **RAM is the binding constraint, not CPU.** Budget about **1 GB for the
|
||||
API process** so the ~335 MB steady state has headroom on top of the rest of the
|
||||
app; training peaks at only ~210–315 MB and can share the same box. No GPU
|
||||
anywhere. Pin thread counts per process — `OMP_NUM_THREADS=1` in the serving
|
||||
unit, `OMP_NUM_THREADS=4` for retraining so it cannot monopolise every core while
|
||||
the API is serving. And since a cold call costs seconds against a 2.4 ms cache
|
||||
hit, consider warming `/forecast` once at startup rather than letting a user
|
||||
absorb it.
|
||||
runs it. The service (`ping-collector`, Rust) stays under 100 MB resident with
|
||||
every model loaded (its unit caps it at 512 MB); the monthly retrain peaks at
|
||||
~210–315 MB and shares the same box. No GPU anywhere. The retrain unit sets
|
||||
`OMP_NUM_THREADS=4` so it cannot monopolise every core while the site is
|
||||
serving.
|
||||
|
||||
## 7. Retraining policy
|
||||
|
||||
@@ -821,15 +853,15 @@ fires `water-monitor-retrain.service` on the 1st of every month at 03:30 server
|
||||
time (`Persistent=true`, so a missed run catches up at boot). The unit runs
|
||||
`scripts/retrain.sh` as the service user with `OMP_NUM_THREADS=4`, `Nice=15`:
|
||||
|
||||
1. trains all stations into `models/.staging/` (the API keeps serving the old
|
||||
bundles throughout);
|
||||
1. trains all stations into `models/.staging/` (the service keeps serving the
|
||||
old bundles throughout);
|
||||
2. refuses to promote unless `metrics.json` reports a `hgb-v3+` version and at
|
||||
least 14 trained stations (exit 3, staging discarded, old models untouched);
|
||||
3. renames the new bundles into `models/`, moving the previous generation to
|
||||
`models/.previous/` for rollback.
|
||||
|
||||
No API restart: `predict.py` reloads bundles by mtime on the next hourly
|
||||
precompute. `systemctl list-timers water-monitor-retrain.timer` shows the next
|
||||
No restart: ping-collector reloads changed `.json.gz` exports by mtime on its
|
||||
next forecast run. `systemctl list-timers water-monitor-retrain.timer` shows the next
|
||||
run; `sudo systemctl start water-monitor-retrain.service` runs it now (after a
|
||||
flood, say); `journalctl -u water-monitor-retrain` has the log. The installer
|
||||
(`scripts/install.sh`) enables the timer.
|
||||
@@ -912,9 +944,10 @@ non-model-backed, and all of them are visible in the payload:
|
||||
horizon's head is in the bundle's `skipped_heads`, almost always because the
|
||||
station had fewer than 30 positive examples for that threshold.
|
||||
- A whole station flipping to heuristic after a code change — the feature-name
|
||||
check in `_model_forecast` caught a mismatch between the live feature builder
|
||||
and the stored `feature_names`. The fix is to retrain; the log line names the
|
||||
missing columns.
|
||||
check caught a mismatch between the feature builder and the stored
|
||||
`feature_names` (in Python `_model_forecast`; in Rust a bundle needing a
|
||||
feature `rust/forecast` cannot build). The fix is to retrain or to port the
|
||||
feature; the log line names the missing columns.
|
||||
|
||||
A live example from the 2026-08-10 cache: of 48 forecast rows, 42 came from
|
||||
models and 6 were heuristic — three for P.4A, which has no bundle by design, and
|
||||
|
||||
@@ -1,275 +0,0 @@
|
||||
# Gap Filling and Data Integrity Guide
|
||||
|
||||
This guide explains the enhanced gap-filling functionality that addresses data gaps and missing timestamps in the Thailand Water Monitor.
|
||||
|
||||
## ✅ **Issues Resolved**
|
||||
|
||||
### **1. Data Gaps Problem**
|
||||
- **Before**: Tool only fetched current day data, leaving gaps in historical records
|
||||
- **After**: Automatically detects and fills missing timestamps for the last 7 days
|
||||
|
||||
### **2. Missing Midnight Timestamps**
|
||||
- **Before**: Jump from 23:00 to 01:00 (missing 00:00 midnight data)
|
||||
- **After**: Specifically checks for and fills midnight hour gaps
|
||||
|
||||
### **3. Changed Values**
|
||||
- **Before**: No mechanism to update existing data if values changed on the server
|
||||
- **After**: Compares existing data with fresh API data and updates changed values
|
||||
|
||||
## 🔧 **New Features**
|
||||
|
||||
### **Command Line Interface**
|
||||
```bash
|
||||
# Check for missing data gaps
|
||||
python water_scraper_v3.py --check-gaps [days]
|
||||
|
||||
# Fill missing data gaps
|
||||
python water_scraper_v3.py --fill-gaps [days]
|
||||
|
||||
# Update existing data with latest values
|
||||
python water_scraper_v3.py --update-data [days]
|
||||
|
||||
# Run single test cycle
|
||||
python water_scraper_v3.py --test
|
||||
|
||||
# Show help
|
||||
python water_scraper_v3.py --help
|
||||
```
|
||||
|
||||
### **Automatic Gap Detection**
|
||||
The system now automatically:
|
||||
- Generates expected hourly timestamps for the specified time range
|
||||
- Compares with existing database records
|
||||
- Identifies missing timestamps
|
||||
- Groups missing data by date for efficient API calls
|
||||
|
||||
### **Intelligent Gap Filling**
|
||||
- **Historical Data Fetching**: Retrieves data for specific dates to fill gaps
|
||||
- **Selective Insertion**: Only inserts data for actually missing timestamps
|
||||
- **API Rate Limiting**: Includes delays between API calls to be respectful
|
||||
- **Error Handling**: Continues processing even if some dates fail
|
||||
|
||||
### **Data Update Mechanism**
|
||||
- **Change Detection**: Compares water levels, discharge rates, and percentages
|
||||
- **Precision Checking**: Uses appropriate thresholds (0.001m for water level, 0.1 cms for discharge)
|
||||
- **Selective Updates**: Only updates records where values have actually changed
|
||||
|
||||
## 📊 **Test Results**
|
||||
|
||||
### **Before Enhancement**
|
||||
```
|
||||
Found 22 missing timestamps in the last 2 days:
|
||||
2025-07-23: Missing hours [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]
|
||||
2025-07-24: Missing hours [0, 20, 21, 22, 23]
|
||||
2025-07-25: Missing hours [0, 9]
|
||||
```
|
||||
|
||||
### **After Gap Filling**
|
||||
```
|
||||
Gap filling completed. Filled 96 missing data points
|
||||
|
||||
Remaining gaps:
|
||||
2025-07-24: Missing hours [10]
|
||||
2025-07-25: Missing hours [0, 10]
|
||||
```
|
||||
|
||||
**Improvement**: Reduced from 22 missing timestamps to 3 (86% improvement)
|
||||
|
||||
## 🚀 **Enhanced Scraping Cycle**
|
||||
|
||||
The regular scraping cycle now includes three phases:
|
||||
|
||||
### **Phase 1: Current Data Collection**
|
||||
```python
|
||||
# Fetch and save current data
|
||||
water_data = self.fetch_water_data()
|
||||
success = self.save_to_database(water_data)
|
||||
```
|
||||
|
||||
### **Phase 2: Gap Filling (Last 7 Days)**
|
||||
```python
|
||||
# Check for and fill missing data
|
||||
filled_count = self.fill_data_gaps(days_back=7)
|
||||
```
|
||||
|
||||
### **Phase 3: Data Updates (Last 2 Days)**
|
||||
```python
|
||||
# Update existing data with latest values
|
||||
updated_count = self.update_existing_data(days_back=2)
|
||||
```
|
||||
|
||||
## 🔧 **Technical Improvements**
|
||||
|
||||
### **Database Connection Handling**
|
||||
- **SQLite Optimization**: Added timeout and thread safety parameters
|
||||
- **Retry Logic**: Exponential backoff for database lock errors
|
||||
- **Transaction Management**: Proper use of `engine.begin()` for automatic commits
|
||||
|
||||
### **Error Recovery**
|
||||
```python
|
||||
# Retry logic with exponential backoff
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
success = self.db_adapter.save_measurements(water_data)
|
||||
if success:
|
||||
return True
|
||||
except Exception as e:
|
||||
if "database is locked" in str(e).lower():
|
||||
time.sleep(2 ** attempt) # 1s, 2s, 4s delays
|
||||
continue
|
||||
```
|
||||
|
||||
### **Memory Efficiency**
|
||||
- **Selective Data Processing**: Only processes data for missing timestamps
|
||||
- **Batch Processing**: Groups operations by date to minimize API calls
|
||||
- **Resource Management**: Proper cleanup and connection handling
|
||||
|
||||
## 📋 **Usage Examples**
|
||||
|
||||
### **Daily Maintenance**
|
||||
```bash
|
||||
# Check for gaps in the last week
|
||||
python water_scraper_v3.py --check-gaps 7
|
||||
|
||||
# Fill any found gaps
|
||||
python water_scraper_v3.py --fill-gaps 7
|
||||
|
||||
# Update recent data for accuracy
|
||||
python water_scraper_v3.py --update-data 2
|
||||
```
|
||||
|
||||
### **Historical Data Recovery**
|
||||
```bash
|
||||
# Check for gaps in the last month
|
||||
python water_scraper_v3.py --check-gaps 30
|
||||
|
||||
# Fill gaps for the last month (be patient, this takes time)
|
||||
python water_scraper_v3.py --fill-gaps 30
|
||||
```
|
||||
|
||||
### **Production Monitoring**
|
||||
```bash
|
||||
# Quick test to ensure system is working
|
||||
python water_scraper_v3.py --test
|
||||
|
||||
# Check for recent gaps
|
||||
python water_scraper_v3.py --check-gaps 1
|
||||
```
|
||||
|
||||
## 🔍 **Monitoring and Alerts**
|
||||
|
||||
### **Gap Detection Output**
|
||||
```
|
||||
Found 22 missing timestamps:
|
||||
2025-07-23: Missing hours [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]
|
||||
2025-07-24: Missing hours [0, 20, 21, 22, 23]
|
||||
2025-07-25: Missing hours [0, 9]
|
||||
```
|
||||
|
||||
### **Gap Filling Progress**
|
||||
```
|
||||
Fetching data for 2025-07-24 to fill 5 missing timestamps
|
||||
Successfully fetched 368 data points from API for 2025-07-24
|
||||
Filled 80 data points for 2025-07-24
|
||||
Gap filling completed. Filled 96 missing data points
|
||||
```
|
||||
|
||||
### **Update Detection**
|
||||
```
|
||||
Checking for updates on 2025-07-24
|
||||
Update needed for P.1 at 2025-07-24 15:00:00
|
||||
Updated 5 measurements for 2025-07-24
|
||||
Data update completed. Updated 5 measurements
|
||||
```
|
||||
|
||||
## ⚙️ **Configuration Options**
|
||||
|
||||
### **Environment Variables**
|
||||
```bash
|
||||
# Database configuration
|
||||
export DB_TYPE=sqlite
|
||||
export WATER_DB_PATH=water_monitoring.db
|
||||
|
||||
# Gap filling settings (can be added to config.py)
|
||||
export GAP_FILL_DAYS=7 # Days to check for gaps
|
||||
export UPDATE_DAYS=2 # Days to check for updates
|
||||
export API_DELAY=1 # Seconds between API calls
|
||||
export MAX_RETRIES=3 # Database retry attempts
|
||||
```
|
||||
|
||||
### **Customizable Parameters**
|
||||
- **Gap Check Period**: Default 7 days, configurable via command line
|
||||
- **Update Period**: Default 2 days, configurable via command line
|
||||
- **API Rate Limiting**: 1-second delay between calls (configurable)
|
||||
- **Retry Logic**: 3 attempts with exponential backoff (configurable)
|
||||
|
||||
## 🛠️ **Troubleshooting**
|
||||
|
||||
### **Common Issues**
|
||||
|
||||
#### **Database Locked Errors**
|
||||
```
|
||||
ERROR - Error saving to SQLITE: database is locked
|
||||
```
|
||||
**Solution**: The retry logic now handles this automatically with exponential backoff.
|
||||
|
||||
#### **API Rate Limiting**
|
||||
```
|
||||
WARNING - Too many requests to API
|
||||
```
|
||||
**Solution**: Increase delay between API calls or reduce the number of days processed at once.
|
||||
|
||||
#### **Missing Data Still Present**
|
||||
```
|
||||
Found X missing timestamps after gap filling
|
||||
```
|
||||
**Possible Causes**:
|
||||
- Data not available on the Thai government server for those timestamps
|
||||
- Network issues during API calls
|
||||
- API returned empty data for those specific times
|
||||
|
||||
### **Debug Commands**
|
||||
```bash
|
||||
# Enable debug logging
|
||||
export LOG_LEVEL=DEBUG
|
||||
python water_scraper_v3.py --check-gaps 1
|
||||
|
||||
# Test specific date range
|
||||
python water_scraper_v3.py --fill-gaps 1
|
||||
|
||||
# Check database directly
|
||||
sqlite3 water_monitoring.db "SELECT COUNT(*) FROM water_measurements;"
|
||||
sqlite3 water_monitoring.db "SELECT timestamp, COUNT(*) FROM water_measurements GROUP BY timestamp ORDER BY timestamp DESC LIMIT 10;"
|
||||
```
|
||||
|
||||
## 📈 **Performance Metrics**
|
||||
|
||||
### **Gap Filling Efficiency**
|
||||
- **API Calls**: Grouped by date to minimize requests
|
||||
- **Processing Speed**: ~100-400 data points per API call
|
||||
- **Success Rate**: 86% gap reduction in test case
|
||||
- **Resource Usage**: Minimal memory footprint with selective processing
|
||||
|
||||
### **Database Performance**
|
||||
- **SQLite Optimization**: Connection pooling and timeout handling
|
||||
- **Transaction Efficiency**: Batch inserts with proper transaction management
|
||||
- **Retry Success**: Automatic recovery from temporary lock conditions
|
||||
|
||||
## 🎯 **Best Practices**
|
||||
|
||||
### **Regular Maintenance**
|
||||
1. **Daily**: Run `--check-gaps 1` to monitor recent data quality
|
||||
2. **Weekly**: Run `--fill-gaps 7` to catch any missed data
|
||||
3. **Monthly**: Run `--update-data 7` to ensure data accuracy
|
||||
|
||||
### **Production Deployment**
|
||||
1. **Automated Scheduling**: Use cron or systemd timers for regular gap checks
|
||||
2. **Monitoring**: Set up alerts for excessive missing data
|
||||
3. **Backup**: Regular database backups before major gap-filling operations
|
||||
|
||||
### **Data Quality Assurance**
|
||||
1. **Validation**: Check for reasonable value ranges after gap filling
|
||||
2. **Comparison**: Compare filled data with nearby timestamps for consistency
|
||||
3. **Documentation**: Log all gap-filling activities for audit trails
|
||||
|
||||
This enhanced gap-filling system ensures comprehensive and accurate water level monitoring with minimal data loss and automatic recovery capabilities.
|
||||
@@ -1,300 +0,0 @@
|
||||
# 🔄 Gitea Actions Workflows - Northern Thailand Ping River Monitor
|
||||
|
||||
## 📋 Overview
|
||||
|
||||
This document describes the Gitea Actions workflows configured for the Northern Thailand Ping River Monitor project. These workflows provide comprehensive CI/CD, security scanning, and documentation generation.
|
||||
|
||||
## 🚀 Available Workflows
|
||||
|
||||
### 1. **CI/CD Pipeline** (`.gitea/workflows/ci.yml`)
|
||||
|
||||
**Triggers:**
|
||||
- Push to `main` or `develop` branches
|
||||
- Pull requests to `main`
|
||||
- Daily scheduled runs at 2 AM UTC
|
||||
|
||||
**Jobs:**
|
||||
- **Test Suite**: Multi-version Python testing (3.9-3.12)
|
||||
- **Code Quality**: Linting, formatting, and type checking
|
||||
- **Build**: Docker image creation and testing
|
||||
- **Integration Test**: Testing with VictoriaMetrics service
|
||||
- **Deploy Staging**: Automatic deployment to staging (develop branch)
|
||||
- **Deploy Production**: Manual deployment to production (main branch)
|
||||
- **Performance Test**: Load testing after production deployment
|
||||
|
||||
**Key Features:**
|
||||
- ✅ Multi-Python version testing
|
||||
- ✅ Docker multi-architecture builds (amd64, arm64)
|
||||
- ✅ Service integration testing
|
||||
- ✅ Automatic staging deployment
|
||||
- ✅ Manual production approval
|
||||
- ✅ Performance validation
|
||||
|
||||
### 2. **Security & Dependency Updates** (`.gitea/workflows/security.yml`)
|
||||
|
||||
**Triggers:**
|
||||
- Daily scheduled runs at 3 AM UTC
|
||||
- Manual dispatch
|
||||
- Changes to requirements files or Dockerfile
|
||||
|
||||
**Jobs:**
|
||||
- **Dependency Scan**: Safety, Bandit, Semgrep security scans
|
||||
- **Docker Security**: Trivy vulnerability scanning
|
||||
- **License Check**: License compliance verification
|
||||
- **Dependency Updates**: Automated update detection
|
||||
- **Code Quality**: Complexity and maintainability analysis
|
||||
|
||||
**Key Features:**
|
||||
- 🔒 Daily security scans
|
||||
- 📦 Dependency vulnerability detection
|
||||
- 📄 License compliance checking
|
||||
- 🔄 Automated update notifications
|
||||
- 📊 Code quality metrics
|
||||
|
||||
### 3. **Release Workflow** (`.gitea/workflows/release.yml`)
|
||||
|
||||
**Triggers:**
|
||||
- Git tags matching `v*.*.*` pattern
|
||||
- Manual dispatch with version input
|
||||
|
||||
**Jobs:**
|
||||
- **Create Release**: Automated release creation with changelog
|
||||
- **Test Release**: Comprehensive testing across Python versions
|
||||
- **Build Release**: Multi-architecture Docker images with proper tags
|
||||
- **Security Scan**: Trivy security scanning of release images
|
||||
- **Deploy Release**: Production deployment with health checks
|
||||
- **Validate Release**: Post-deployment validation and testing
|
||||
|
||||
**Key Features:**
|
||||
- 🏷️ Automated release creation
|
||||
- 📝 Changelog generation
|
||||
- 🐳 Multi-architecture Docker builds
|
||||
- 🔒 Security scanning
|
||||
- ✅ Comprehensive validation
|
||||
|
||||
### 4. **Documentation** (`.gitea/workflows/docs.yml`)
|
||||
|
||||
**Triggers:**
|
||||
- Changes to documentation files
|
||||
- Changes to Python source files
|
||||
- Manual dispatch
|
||||
|
||||
**Jobs:**
|
||||
- **Validate Docs**: Link checking and structure validation
|
||||
- **Generate API Docs**: OpenAPI specification generation
|
||||
- **Build Sphinx Docs**: Comprehensive API documentation
|
||||
- **Documentation Summary**: Build status and artifact summary
|
||||
|
||||
**Key Features:**
|
||||
- 📚 Automated API documentation
|
||||
- 🔗 Link validation
|
||||
- 📖 Sphinx documentation generation
|
||||
- ✅ Documentation completeness checking
|
||||
|
||||
## 🔧 Workflow Configuration
|
||||
|
||||
### **Required Secrets**
|
||||
|
||||
Configure these secrets in your Gitea repository settings:
|
||||
|
||||
```bash
|
||||
GITEA_TOKEN # Gitea access token for container registry
|
||||
GITHUB_TOKEN # GitHub personal access token for better rate limits
|
||||
SLACK_WEBHOOK_URL # Optional: Slack notifications
|
||||
STAGING_WEBHOOK_URL # Optional: Staging deployment webhook
|
||||
PRODUCTION_WEBHOOK_URL # Optional: Production deployment webhook
|
||||
```
|
||||
|
||||
**Setting up GitHub Token:**
|
||||
1. Go to GitHub → Settings → Developer settings → Personal access tokens
|
||||
2. Generate a new token with `public_repo` scope (for public repos) or `repo` scope (for private repos)
|
||||
3. Copy the token
|
||||
4. In your Gitea repository → Settings → Secrets → Add `GITHUB_TOKEN` with the copied value
|
||||
|
||||
### **Environment Variables**
|
||||
|
||||
Key environment variables used across workflows:
|
||||
|
||||
```yaml
|
||||
PYTHON_VERSION: '3.11' # Default Python version
|
||||
REGISTRY: git.b4l.co.th # Container registry
|
||||
IMAGE_NAME: grabowski/northern-thailand-ping-river-monitor
|
||||
```
|
||||
|
||||
## 📊 Workflow Status
|
||||
|
||||
### **CI/CD Pipeline Status**
|
||||
- **Test Coverage**: Multi-version Python testing
|
||||
- **Code Quality**: Automated linting and formatting
|
||||
- **Security**: Integrated security scanning
|
||||
- **Deployment**: Automated staging, manual production
|
||||
|
||||
### **Security Monitoring**
|
||||
- **Daily Scans**: Automated vulnerability detection
|
||||
- **Dependency Updates**: Proactive update notifications
|
||||
- **License Compliance**: Automated license checking
|
||||
- **Code Quality**: Continuous quality monitoring
|
||||
|
||||
### **Release Management**
|
||||
- **Automated Releases**: Tag-based release creation
|
||||
- **Multi-Architecture**: Support for amd64 and arm64
|
||||
- **Security Validation**: Pre-deployment security checks
|
||||
- **Health Monitoring**: Post-deployment validation
|
||||
|
||||
## 🚀 Usage Examples
|
||||
|
||||
### **Triggering Workflows**
|
||||
|
||||
**Manual CI/CD Run:**
|
||||
```bash
|
||||
# Push to trigger CI/CD
|
||||
git push origin main
|
||||
|
||||
# Create pull request to trigger testing
|
||||
git checkout -b feature/new-feature
|
||||
git push origin feature/new-feature
|
||||
# Create PR in Gitea UI
|
||||
```
|
||||
|
||||
**Manual Security Scan:**
|
||||
```bash
|
||||
# Trigger via Gitea Actions UI
|
||||
# Go to Actions → Security & Dependency Updates → Run workflow
|
||||
```
|
||||
|
||||
**Creating a Release:**
|
||||
```bash
|
||||
# Create and push a tag
|
||||
git tag v3.1.1
|
||||
git push origin v3.1.1
|
||||
|
||||
# Or use manual dispatch in Gitea Actions UI
|
||||
```
|
||||
|
||||
### **Monitoring Workflow Results**
|
||||
|
||||
**Check Workflow Status:**
|
||||
1. Navigate to your repository in Gitea
|
||||
2. Click on "Actions" tab
|
||||
3. View workflow runs and their status
|
||||
|
||||
**Download Artifacts:**
|
||||
1. Click on a completed workflow run
|
||||
2. Scroll to "Artifacts" section
|
||||
3. Download reports and logs
|
||||
|
||||
**View Security Reports:**
|
||||
1. Go to Security workflow runs
|
||||
2. Download security-reports artifacts
|
||||
3. Review JSON reports for vulnerabilities
|
||||
|
||||
## 🔍 Troubleshooting
|
||||
|
||||
### **Common Issues**
|
||||
|
||||
**Workflow Fails on Dependencies:**
|
||||
```bash
|
||||
# Check requirements.txt for version conflicts
|
||||
pip-compile requirements.in
|
||||
```
|
||||
|
||||
**Docker Build Fails:**
|
||||
```bash
|
||||
# Test Docker build locally
|
||||
make docker-build
|
||||
docker run --rm ping-river-monitor python run.py --test
|
||||
```
|
||||
|
||||
**Security Scan Failures:**
|
||||
```bash
|
||||
# Run security scans locally
|
||||
safety check -r requirements.txt
|
||||
bandit -r src/
|
||||
```
|
||||
|
||||
**Test Failures:**
|
||||
```bash
|
||||
# Run tests locally
|
||||
make test
|
||||
python tests/test_integration.py
|
||||
```
|
||||
|
||||
### **Debugging Workflows**
|
||||
|
||||
**Enable Debug Logging:**
|
||||
Add to workflow file:
|
||||
```yaml
|
||||
env:
|
||||
ACTIONS_STEP_DEBUG: true
|
||||
ACTIONS_RUNNER_DEBUG: true
|
||||
```
|
||||
|
||||
**Check Workflow Logs:**
|
||||
1. Go to failed workflow run
|
||||
2. Click on failed job
|
||||
3. Expand failed step to see detailed logs
|
||||
|
||||
**Validate Workflow Syntax:**
|
||||
```bash
|
||||
# Validate YAML syntax
|
||||
make validate-workflows
|
||||
```
|
||||
|
||||
## 📈 Performance Optimization
|
||||
|
||||
### **Caching Strategy**
|
||||
- **Pip Cache**: Cached across workflow runs
|
||||
- **Docker Layer Cache**: GitHub Actions cache for faster builds
|
||||
- **Dependency Cache**: Cached based on requirements.txt hash
|
||||
|
||||
### **Parallel Execution**
|
||||
- **Matrix Builds**: Multiple Python versions tested in parallel
|
||||
- **Independent Jobs**: Security scans run independently of tests
|
||||
- **Conditional Execution**: Jobs skip when not needed
|
||||
|
||||
### **Resource Management**
|
||||
- **Timeout Settings**: Prevent hanging workflows
|
||||
- **Resource Limits**: Appropriate runner sizing
|
||||
- **Artifact Cleanup**: Automatic cleanup of old artifacts
|
||||
|
||||
## 🔒 Security Best Practices
|
||||
|
||||
### **Secret Management**
|
||||
- Use Gitea repository secrets for sensitive data
|
||||
- Never commit secrets to repository
|
||||
- Rotate secrets regularly
|
||||
- Use least-privilege access tokens
|
||||
|
||||
### **Container Security**
|
||||
- Multi-stage Docker builds for smaller images
|
||||
- Non-root user in containers
|
||||
- Regular base image updates
|
||||
- Vulnerability scanning before deployment
|
||||
|
||||
### **Code Security**
|
||||
- Automated security scanning in CI/CD
|
||||
- Dependency vulnerability monitoring
|
||||
- License compliance checking
|
||||
- Code quality enforcement
|
||||
|
||||
## 📚 Additional Resources
|
||||
|
||||
### **Gitea Actions Documentation**
|
||||
- [Gitea Actions Overview](https://docs.gitea.io/en-us/usage/actions/)
|
||||
- [Workflow Syntax](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions)
|
||||
- [Available Actions](https://github.com/marketplace?type=actions)
|
||||
|
||||
### **Project-Specific Resources**
|
||||
- [Contributing Guide](../CONTRIBUTING.md)
|
||||
|
||||
### **Monitoring and Alerts**
|
||||
- Workflow status badges in README
|
||||
- Email notifications for failures
|
||||
- Slack/Discord integration for team updates
|
||||
- Grafana dashboards for deployment metrics
|
||||
|
||||
---
|
||||
|
||||
**Workflow Version**: v3.1.3
|
||||
**Last Updated**: 2025-08-12
|
||||
**Maintained By**: Ping River Monitor Team
|
||||
@@ -32,13 +32,9 @@ curl -X POST https://matrix.org/_matrix/client/v3/login \
|
||||
## Step 4: Configure .env File
|
||||
Add these to your `.env` file:
|
||||
```bash
|
||||
# Matrix Alerting Configuration
|
||||
MATRIX_HOMESERVER=https://matrix.org
|
||||
MATRIX_ACCESS_TOKEN=syt_your_access_token_here
|
||||
MATRIX_ROOM_ID=!your_room_id:matrix.org
|
||||
|
||||
# Grafana Integration (optional)
|
||||
GRAFANA_URL=http://localhost:3000
|
||||
```
|
||||
|
||||
## Step 5: Restart and check
|
||||
@@ -71,8 +67,8 @@ so a Matrix outage is retried on Matrix only and does not re-send ntfy.
|
||||
|
||||
The old `run.py --alert-check` cron job (every 15 minutes, repeating an alert
|
||||
for as long as a level stayed high) is replaced by this;
|
||||
`scripts/install_collector.sh` comments it out of the crontab once
|
||||
ping-collector sends Matrix, keeping a backup next to the app.
|
||||
`scripts/install.sh` comments it out of the crontab if it is still there,
|
||||
keeping a backup next to the app.
|
||||
|
||||
## Troubleshooting
|
||||
- **HTTP 401/403** in `journalctl -u ping-collector`: the access token is invalid
|
||||
|
||||
+52
-31
@@ -26,7 +26,8 @@ seventy-two. In a quiet season a subscriber hears nothing.
|
||||
|
||||
Station slugs are the code lowercased without the dot: `p1`, `p103`, `p67`.
|
||||
Thresholds are the ones in `src/ml/features.py` (`THRESHOLDS`): P.1 3.70 /
|
||||
4.20 m, P.103 5.95 / 6.75 m, and so on.
|
||||
4.20 m (half a metre below the banks / over the banks; the city floods in stages
|
||||
from 4.30 m), P.103 5.95 / 6.75 m, and so on.
|
||||
|
||||
The outlook topic is opt-in for a reason: it is model output, and the message
|
||||
says so. Observed-crossing topics only ever report a gauge reading.
|
||||
@@ -36,28 +37,38 @@ the public dashboard.
|
||||
|
||||
## How it runs
|
||||
|
||||
Since phase 2 of the Rust rewrite the notifier runs in `ping-collector`
|
||||
(`rust/collector/src/notify.rs`) when `FORECAST_IN_COLLECTOR=true`, after each
|
||||
forecast run. It is a port of `src/notify.py` with the same topics, texts and
|
||||
state keys (checked word for word against Python on 751 generated messages),
|
||||
so switching between the two sends nothing twice. One difference: the Rust
|
||||
notifier also uses ThaiWater (HII) telemetry. For the stations whose HII twin
|
||||
The notifier runs in `ping-collector` (`rust/collector/src/notify.rs`) when
|
||||
`FORECAST_IN_COLLECTOR=true`, after each forecast run. It is a port of the
|
||||
Python notifier it replaced in 2026-10, with the same topics, texts and state
|
||||
keys (checked word for word on 751 generated messages), so the switch sent
|
||||
nothing twice. One difference: it also uses ThaiWater (HII) telemetry. For the stations whose HII twin
|
||||
is the same sensor (P.1, P.103, P.20, P.4A, P.67, P.75, P.82, P.84, P.92, and
|
||||
P.81 bias-corrected), a reading newer than RID's drives the level alerts; the
|
||||
message then says "ThaiWater telemetry; the RID report is not in yet". RID's
|
||||
report regularly lags 2-10 h at night, which used to send "gauge feed stale"
|
||||
around midnight and "recovered" in the morning. Now the feed is stale only
|
||||
when neither source has a reading for 3 h, and RID alone lagging is reported
|
||||
once after `RID_STALE_ALERT_HOURS` (18, 0 = off). With `MATRIX_*` set, the
|
||||
message then says "ThaiWater telemetry; the RID report is not in yet". The feed
|
||||
counts as stale when no source has a reading for 3 h, and RID alone lagging is
|
||||
reported once after `RID_STALE_ALERT_HOURS` (18, 0 = off).
|
||||
|
||||
**RID's nightly pause.** RID publishes no hourly report overnight: on
|
||||
2026-10-17, -18 and -19 the 21:00 reading was the last one until the hours
|
||||
22:00-06:00 arrived together between 06:35 and 07:22. ThaiWater's copies of the
|
||||
RID gauges come from the same feed and arrive with the same delay (P.1's
|
||||
mirror had no value for 22:00-00:00 even the next morning), so they do not
|
||||
bridge the night. Without a rule for it, every night sent "gauge feed stale"
|
||||
around midnight and "feed recovered" around 07:00. So between 21:00 and
|
||||
`FEED_NIGHT_UNTIL_HOUR` (9, 0 = off) a quiet river's feed is not reported
|
||||
stale; a feed still without new readings at 09:00 is. While any station is at
|
||||
warning or danger, or the P.1 outlook is on, the 3 h rule holds all night:
|
||||
then a stalled dashboard matters. A feed that was already stale stays stale
|
||||
through the night, and a recovery is announced whenever it happens. With `MATRIX_*` set, the
|
||||
same transitions also go to a Matrix room, each once (per station, outlook,
|
||||
feed; not the per-topic summaries), with their own `matrix:` state keys.
|
||||
|
||||
With `FORECAST_IN_COLLECTOR=false`, `src/notify.py` is called once per collection cycle inside the API process
|
||||
(leader only), right after the forecast precompute, so it sees exactly the
|
||||
readings and forecasts the dashboard shows. Per-key last-sent state is stored
|
||||
in the `notification_state` table of the monitor's own database, so a restart
|
||||
or redeploy never re-sends and never misses a crossing that happened while
|
||||
the service was down (the next cycle compares against the persisted state).
|
||||
It runs in the same process that serves the dashboard, right after the
|
||||
forecast, so it sees exactly the readings and forecasts the dashboard shows.
|
||||
Per-key last-sent state is stored in the `notification_state` table of the
|
||||
monitor's own database, so a restart or redeploy never re-sends and never
|
||||
misses a crossing that happened while the service was down (the next cycle
|
||||
compares against the persisted state).
|
||||
|
||||
If ntfy is unreachable the transition is **not** recorded, so it is retried
|
||||
on the next cycle rather than silently lost. Any other failure in the notify
|
||||
@@ -86,8 +97,8 @@ token, and appends `NTFY_SERVER` (public URL for subscribers),
|
||||
and `NTFY_TOKEN` to `.env` if they are not there yet. Then:
|
||||
|
||||
```bash
|
||||
systemctl restart water-monitor
|
||||
journalctl -u water-monitor -n 20 | grep ntfy # "ntfy notifications: https://... topics ping-*"
|
||||
systemctl restart ping-collector
|
||||
journalctl -u ping-collector -n 20 | grep ntfy # "ntfy notifications: publish to ..., topics ping-*"
|
||||
curl -s 'https://ntfy.buildfor.life/ping-status/json?poll=1' # anonymous read works
|
||||
```
|
||||
|
||||
@@ -120,21 +131,31 @@ holds back an alert. Test the pipeline before the domain is live with
|
||||
| `NTFY_TOKEN` | *(empty)* | bearer token if the server requires auth to publish (it does, see above) |
|
||||
| `PUBLIC_URL` | `https://water.buildfor.life/` | click-through target in messages |
|
||||
|
||||
Tunables in `src/notify.py`: `CLEAR_MARGIN_M` (0.10), `OUTLOOK_ON` / `OUTLOOK_OFF`
|
||||
(0.50 / 0.25), stale feed threshold (3 h, argument to `evaluate`).
|
||||
| `RID_STALE_ALERT_HOURS` | `18` | report RID lagging this long while ThaiWater is current (0 = never) |
|
||||
| `FEED_NIGHT_UNTIL_HOUR` | `9` | end of the night window from 21:00 in which a quiet river's feed is not reported stale (0 = off) |
|
||||
| `MATRIX_HOMESERVER` / `MATRIX_ACCESS_TOKEN` / `MATRIX_ROOM_ID` | *(empty = off)* | the team room (docs/MATRIX_QUICK_START.md) |
|
||||
|
||||
Tunables in `src/ml/export.py` (compiled into ping-collector through
|
||||
`rust/forecast/flood_config.json`: regenerate with
|
||||
`uv run python -m src.ml.export --config` and rebuild): `CLEAR_MARGIN_M`
|
||||
(0.10), `OUTLOOK_ON` / `OUTLOOK_OFF` (0.50 / 0.25), `FEED_STALE_AFTER_H` (3),
|
||||
the capacity guard and the station names in the messages.
|
||||
|
||||
## Testing
|
||||
|
||||
`tests/test_notify.py` covers the state machine: quiet river sends nothing;
|
||||
crossing once, then silence while above, then all-clear; hysteresis on the way
|
||||
down; escalation to danger and back; basin digest grouping; outlook on/off;
|
||||
heuristic forecasts ignored; stale feed and recovery; state survives a restart
|
||||
through sqlite; a failed publish is retried next cycle.
|
||||
The tests in `rust/collector/src/notify.rs` cover the state machine: quiet
|
||||
river sends nothing; crossing once, then silence while above, then all-clear;
|
||||
hysteresis on the way down; escalation to danger and back; outlook on/off with
|
||||
heuristic forecasts ignored; HII readings replacing a lagging RID reading;
|
||||
stale feed only when both sources are; RID lagging reported once; a failed
|
||||
publish retried next cycle; the Matrix rendering; and the 652-step replay of
|
||||
the Python notifier's messages. `notification_state` round-trips against
|
||||
PostgreSQL in `db_tests.rs`.
|
||||
|
||||
To exercise the real path against a real ntfy locally: run `ntfy serve` (any
|
||||
platform, same binary), set `NTFY_SERVER`/`NTFY_TOKEN`, seed readings, and
|
||||
poll the topic JSON. `scripts/e2e_notify.py` does exactly that if you want a
|
||||
template.
|
||||
`ping-collector notify --dry-run` shows the readings judged and what would be
|
||||
sent right now, changing nothing. To exercise the real path against a real
|
||||
ntfy locally: run `ntfy serve` (any platform, same binary), point
|
||||
`NTFY_SERVER`/`NTFY_TOKEN` at it and poll the topic JSON.
|
||||
|
||||
## Why ntfy and not …
|
||||
|
||||
|
||||
+25
-10
@@ -1,9 +1,17 @@
|
||||
# A Rust rewrite: assessment
|
||||
# The Rust rewrite
|
||||
|
||||
Status: **in progress.** Phase 1 (collector) is in production since 2026-10-17,
|
||||
phase 2 (forecast + ntfy) and phase 3 (website/API + Matrix) since
|
||||
2026-10-18; see
|
||||
[rust/README.md](../rust/README.md). Spike: [spikes/rust-forecast](../spikes/rust-forecast/README.md).
|
||||
Status: **done** (2026-10-18). The service is `ping-collector`
|
||||
([rust/README.md](../rust/README.md)); Python keeps model training,
|
||||
evaluation and the archive backfills. Phases 1-3 replaced the Python
|
||||
collector, the forecast + notifier and the website/API one at a time, each
|
||||
checked against the Python side on production data before the switch;
|
||||
phase 4 removed the Python service code. The migration tooling
|
||||
(`scripts/collector_parity.py`, `scripts/api_parity.py`,
|
||||
`scripts/install_collector.sh` with its cutover/rollback modes, the
|
||||
`spikes/rust-forecast` spike) is in git history before the cleanup commit.
|
||||
|
||||
The assessment below was written before phase 1 and is kept as the record of
|
||||
why and how.
|
||||
|
||||
## Short version
|
||||
|
||||
@@ -87,7 +95,7 @@ This slows model iteration, which is the most active part of the repo.
|
||||
| 1 | `ping-collector`: RID, HII, dams, Open-Meteo | in production since 2026-10-17 23:09 (Python: `COLLECTION_ENABLED=false`) |
|
||||
| 2 | Forecast crate + ntfy | in production since 2026-10-18 15:08 (`FORECAST_IN_COLLECTOR=true`); live check on production data: 48/48 rows identical, 96/96 probabilities bit-identical |
|
||||
| 3 | Website + read API in `ping-collector`, Matrix like ntfy | in production since 2026-10-18 16:40 (`WEB_LISTEN`, `water-monitor.service` disabled); on production data 53 of 53 compared routes identical to FastAPI (`scripts/api_parity.py`) |
|
||||
| 4 | Cleanup | |
|
||||
| 4 | Cleanup: Python service code removed | done 2026-10-18 |
|
||||
|
||||
0. **Spike: done.** P.1 features + HGB inference, bit-identical.
|
||||
1. **Collector daemon** (`rust/collector`, ~1.5k LOC incl. tests): RID with the
|
||||
@@ -96,7 +104,7 @@ This slows model iteration, which is the most active part of the repo.
|
||||
API runs with `COLLECTION_ENABLED=false` and re-forecasts when the newest
|
||||
RID reading in the database advances. ntfy moved to phase 2: the outlook
|
||||
alert needs the forecasts, which are still Python. Cutover and rollback
|
||||
are one command each (`scripts/install_collector.sh`).
|
||||
were one command each (`scripts/install_collector.sh`, since removed).
|
||||
2. **Forecast crate + ntfy** (`rust/forecast`, run by `ping-collector` with
|
||||
`FORECAST_IN_COLLECTOR=true`): all stations, HII/rain/dam inputs, heuristic
|
||||
fallback, probabilities and stages, `forecast_history`, `forecast_current`
|
||||
@@ -130,9 +138,16 @@ This slows model iteration, which is the most active part of the repo.
|
||||
their own `matrix:` state keys seeded from ntfy's, replacing the
|
||||
`run.py --alert-check` cron job that repeated alerts every 15 minutes.
|
||||
Python keeps training, evaluation and the monthly retrain timer.
|
||||
4. **Cleanup:** drop FastAPI/uvicorn, the Python collector, notifier, CLI and
|
||||
the multi-backend adapters (SQLite, MySQL, InfluxDB, VictoriaMetrics) and
|
||||
demo databases; the Python environment shrinks to the training stack.
|
||||
4. **Cleanup** (done): FastAPI/uvicorn, the Python collector, notifier, Matrix
|
||||
alerting, CLI (`run.py`), the multi-backend adapters (SQLite, MySQL,
|
||||
InfluxDB, VictoriaMetrics), demo databases, Docker/Grafana files and the
|
||||
migration scripts are gone, about 17k lines. The Python environment is the
|
||||
training stack (pandas, scikit-learn, SQLAlchemy/psycopg2): `src/ml`, the
|
||||
backfill clients/stores (`src/hii_collector.py`, `src/rid_reservoir.py`)
|
||||
and `src/ml/predict.py` as the reference implementation the Rust forecaster
|
||||
is tested against. PostgreSQL only; the Python database tests run against
|
||||
a throwaway schema (`TEST_DATABASE_URL`, as the Rust ones). One installer,
|
||||
`scripts/install.sh`.
|
||||
|
||||
Decided before phase 3 (2026-10-18): Matrix stays and behaves like ntfy;
|
||||
`/docs` stays; SQLite dev mode goes; the monthly retrain stays on the same
|
||||
|
||||
@@ -1,241 +0,0 @@
|
||||
# 🏔️ Station Management Guide - Northern Thailand Ping River Monitor
|
||||
|
||||
## 🎯 **Overview**
|
||||
|
||||
The Northern Thailand Ping River Monitor now includes comprehensive station management capabilities, allowing you to dynamically add, update, and remove monitoring stations through the web API.
|
||||
|
||||
## 🌊 **Current Coverage**
|
||||
|
||||
The system currently monitors **16 water stations** along the Ping River Basin:
|
||||
|
||||
### **Upper Ping River (Chiang Mai Province)**
|
||||
- **P.20** - Ban Chiang Dao (บ้านเชียงดาว)
|
||||
- **P.75** - Ban Chai Lat (บ้านช่อแล)
|
||||
- **P.92** - Ban Muang Aut (บ้านเมืองกึ๊ด)
|
||||
- **P.4A** - Ban Mae Taeng (บ้านแม่แตง)
|
||||
- **P.67** - Ban Tae (บ้านแม่แต)
|
||||
- **P.21** - Ban Rim Tai (บ้านริมใต้)
|
||||
- **P.103** - Ring Bridge 3 (สะพานวงแหวนรอบ 3)
|
||||
|
||||
### **Middle Ping River**
|
||||
- **P.1** - Nawarat Bridge (สะพานนวรัฐ) - *Main reference station*
|
||||
- **P.82** - Ban Sob win (บ้านสบวิน)
|
||||
- **P.84** - Ban Panton (บ้านพันตน)
|
||||
- **P.81** - Ban Pong (บ้านโป่ง)
|
||||
- **P.5** - Tha Nang Bridge (สะพานท่านาง)
|
||||
|
||||
### **Lower Ping River**
|
||||
- **P.77** - Baan Sop Mae Sapuord (บ้านสบแม่สะป๊วด)
|
||||
- **P.87** - Ban Pa Sang (บ้านป่าซาง)
|
||||
- **P.76** - Banb Mae I Hai (บ้านแม่อีไฮ)
|
||||
- **P.85** - Baan Lai Kaew (บ้านหล่ายแก้ว)
|
||||
|
||||
## 🔧 **Station Management API**
|
||||
|
||||
### **List All Stations**
|
||||
```bash
|
||||
GET /stations
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
[
|
||||
{
|
||||
"station_id": 1,
|
||||
"station_code": "P.20",
|
||||
"thai_name": "บ้านเชียงดาว",
|
||||
"english_name": "Ban Chiang Dao",
|
||||
"latitude": 19.36731448032191,
|
||||
"longitude": 98.9688487015384,
|
||||
"geohash": null,
|
||||
"status": "active"
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
### **Get Specific Station**
|
||||
```bash
|
||||
GET /stations/{station_id}
|
||||
```
|
||||
|
||||
### **Add New Station**
|
||||
```bash
|
||||
POST /stations
|
||||
Content-Type: application/json
|
||||
|
||||
{
|
||||
"station_code": "P.NEW",
|
||||
"thai_name": "สถานีใหม่",
|
||||
"english_name": "New Station",
|
||||
"latitude": 18.7875,
|
||||
"longitude": 99.0045,
|
||||
"geohash": "w5q6uuhvfcfp25",
|
||||
"status": "active"
|
||||
}
|
||||
```
|
||||
|
||||
### **Update Station Information**
|
||||
```bash
|
||||
PUT /stations/{station_id}
|
||||
Content-Type: application/json
|
||||
|
||||
{
|
||||
"thai_name": "ชื่อใหม่",
|
||||
"english_name": "Updated Name",
|
||||
"latitude": 18.8000,
|
||||
"longitude": 99.0100
|
||||
}
|
||||
```
|
||||
|
||||
### **Delete Station**
|
||||
```bash
|
||||
DELETE /stations/{station_id}
|
||||
```
|
||||
|
||||
## 🧪 **Testing Station Management**
|
||||
|
||||
Use the provided test script to verify station management functionality:
|
||||
|
||||
```bash
|
||||
# Test all station management endpoints
|
||||
python test_station_management.py
|
||||
```
|
||||
|
||||
This will:
|
||||
1. List existing stations
|
||||
2. Create a test station
|
||||
3. Retrieve station details
|
||||
4. Update station information
|
||||
5. Verify changes
|
||||
6. Delete the test station
|
||||
7. Confirm deletion
|
||||
|
||||
## 📊 **Station Data Model**
|
||||
|
||||
### **Required Fields**
|
||||
- `station_code`: Unique identifier (e.g., "P.1", "P.20")
|
||||
- `thai_name`: Thai language name
|
||||
- `english_name`: English language name
|
||||
|
||||
### **Optional Fields**
|
||||
- `latitude`: GPS latitude coordinate (-90 to 90)
|
||||
- `longitude`: GPS longitude coordinate (-180 to 180)
|
||||
- `geohash`: Geohash string for location
|
||||
- `status`: Station status ("active", "inactive", "maintenance", "error")
|
||||
|
||||
### **Validation Rules**
|
||||
- Station codes must be unique
|
||||
- Latitude must be between -90 and 90
|
||||
- Longitude must be between -180 and 180
|
||||
- Names cannot be empty
|
||||
- Status must be valid enum value
|
||||
|
||||
## 🌐 **Web Interface**
|
||||
|
||||
Access the station management interface through the web dashboard:
|
||||
|
||||
1. **Start the API server:**
|
||||
```bash
|
||||
python run.py --web-api
|
||||
```
|
||||
|
||||
2. **Open your browser:**
|
||||
- Dashboard: http://localhost:8000
|
||||
- API Documentation: http://localhost:8000/docs
|
||||
|
||||
3. **Use the interactive API docs** to test station management endpoints
|
||||
|
||||
## 🔄 **Integration with Data Collection**
|
||||
|
||||
- **Dynamic Station Discovery**: New stations are automatically included in data collection
|
||||
- **Real-time Updates**: Station information changes are reflected immediately
|
||||
- **Data Continuity**: Historical data is preserved when updating station details
|
||||
- **Error Handling**: Invalid stations are skipped during data collection
|
||||
|
||||
## 📍 **Geographic Coverage**
|
||||
|
||||
The Ping River Basin monitoring network covers:
|
||||
|
||||
- **Total Distance**: ~400 km from Chiang Dao to Nakhon Sawan
|
||||
- **Elevation Range**: 300m to 1,200m above sea level
|
||||
- **Catchment Area**: ~25,000 km²
|
||||
- **Major Cities**: Chiang Mai, Lamphun, Tak, Nakhon Sawan
|
||||
|
||||
## 🚀 **Usage Examples**
|
||||
|
||||
### **Add a New Upstream Station**
|
||||
```bash
|
||||
curl -X POST "http://localhost:8000/stations" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"station_code": "P.UPSTREAM",
|
||||
"thai_name": "สถานีต้นน้ำ",
|
||||
"english_name": "Upstream Station",
|
||||
"latitude": 19.5000,
|
||||
"longitude": 98.9000,
|
||||
"status": "active"
|
||||
}'
|
||||
```
|
||||
|
||||
### **Update Station Coordinates**
|
||||
```bash
|
||||
curl -X PUT "http://localhost:8000/stations/1" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"latitude": 19.3700,
|
||||
"longitude": 98.9700
|
||||
}'
|
||||
```
|
||||
|
||||
### **Mark Station for Maintenance**
|
||||
```bash
|
||||
curl -X PUT "http://localhost:8000/stations/5" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"status": "maintenance"
|
||||
}'
|
||||
```
|
||||
|
||||
## 🔒 **Best Practices**
|
||||
|
||||
### **Station Naming**
|
||||
- Use consistent code format (P.XX)
|
||||
- Include both Thai and English names
|
||||
- Use descriptive location names
|
||||
|
||||
### **Coordinate Accuracy**
|
||||
- Use high-precision GPS coordinates (6+ decimal places)
|
||||
- Verify coordinates match actual station location
|
||||
- Include geohash for efficient spatial queries
|
||||
|
||||
### **Status Management**
|
||||
- Set status to "maintenance" during repairs
|
||||
- Use "inactive" for temporarily offline stations
|
||||
- Use "error" for stations with data quality issues
|
||||
|
||||
### **Data Integrity**
|
||||
- Test new stations before adding to production
|
||||
- Backup station configuration before major changes
|
||||
- Monitor data quality after station updates
|
||||
|
||||
## 🎯 **Future Enhancements**
|
||||
|
||||
Planned improvements for station management:
|
||||
|
||||
1. **Bulk Operations** - Import/export multiple stations
|
||||
2. **Station Groups** - Organize stations by river section
|
||||
3. **Automated Validation** - GPS coordinate verification
|
||||
4. **Historical Tracking** - Track station configuration changes
|
||||
5. **Alert Integration** - Notifications for station status changes
|
||||
6. **Map Interface** - Visual station management on interactive map
|
||||
|
||||
## 📞 **Support**
|
||||
|
||||
For station management issues:
|
||||
|
||||
1. Check the API documentation at `/docs`
|
||||
2. Run the test script: `python test_station_management.py`
|
||||
3. Review logs for error details
|
||||
4. Verify station data format and validation rules
|
||||
|
||||
The station management system provides flexible control over your monitoring network while maintaining data integrity and system reliability.
|
||||
@@ -1,443 +0,0 @@
|
||||
# VictoriaMetrics Setup Guide for Thailand Water Monitor
|
||||
|
||||
This guide provides comprehensive instructions for setting up VictoriaMetrics as the time-series database backend for the Thailand Water Monitor.
|
||||
|
||||
## Why VictoriaMetrics?
|
||||
|
||||
VictoriaMetrics is an excellent choice for water monitoring data because:
|
||||
|
||||
- **High Performance**: Up to 10x faster than InfluxDB
|
||||
- **Low Resource Usage**: Uses 10x less RAM than Prometheus
|
||||
- **Better Compression**: 70x better compression than Prometheus
|
||||
- **Prometheus Compatible**: Drop-in replacement for Prometheus
|
||||
- **Easy to Deploy**: Single binary, no dependencies
|
||||
- **Cost Effective**: Open source with commercial support available
|
||||
|
||||
## Quick Start
|
||||
|
||||
### 1. Environment Variables
|
||||
|
||||
Set these environment variables to configure VictoriaMetrics:
|
||||
|
||||
```bash
|
||||
# Windows (PowerShell)
|
||||
$env:DB_TYPE="victoriametrics"
|
||||
$env:VM_HOST="localhost"
|
||||
$env:VM_PORT="8428"
|
||||
|
||||
# Linux/Mac
|
||||
export DB_TYPE=victoriametrics
|
||||
export VM_HOST=localhost
|
||||
export VM_PORT=8428
|
||||
```
|
||||
|
||||
### 2. Start VictoriaMetrics with Docker
|
||||
|
||||
```bash
|
||||
# Simple setup
|
||||
docker run -d \
|
||||
--name victoriametrics \
|
||||
-p 8428:8428 \
|
||||
-v victoria-metrics-data:/victoria-metrics-data \
|
||||
victoriametrics/victoria-metrics:latest \
|
||||
--storageDataPath=/victoria-metrics-data \
|
||||
--retentionPeriod=2y \
|
||||
--httpListenAddr=:8428
|
||||
|
||||
# Verify it's running
|
||||
curl http://localhost:8428/health
|
||||
```
|
||||
|
||||
### 3. Run the Water Monitor
|
||||
|
||||
```bash
|
||||
python water_scraper_v3.py
|
||||
```
|
||||
|
||||
### 4. Access Grafana Dashboard
|
||||
|
||||
```bash
|
||||
# Start with Docker Compose (includes Grafana)
|
||||
docker-compose -f docker-compose.victoriametrics.yml up -d
|
||||
|
||||
# Access Grafana at http://localhost:3000
|
||||
# Username: admin
|
||||
# Password: admin_password
|
||||
```
|
||||
|
||||
## Production Setup
|
||||
|
||||
### Docker Compose Configuration
|
||||
|
||||
Use the provided `docker-compose.victoriametrics.yml` file:
|
||||
|
||||
```bash
|
||||
# Start the complete stack
|
||||
docker-compose -f docker-compose.victoriametrics.yml up -d
|
||||
|
||||
# Check status
|
||||
docker-compose -f docker-compose.victoriametrics.yml ps
|
||||
|
||||
# View logs
|
||||
docker-compose -f docker-compose.victoriametrics.yml logs -f
|
||||
```
|
||||
|
||||
### Manual VictoriaMetrics Configuration
|
||||
|
||||
#### High-Performance Configuration
|
||||
|
||||
```bash
|
||||
docker run -d \
|
||||
--name victoriametrics \
|
||||
-p 8428:8428 \
|
||||
-v victoria-metrics-data:/victoria-metrics-data \
|
||||
victoriametrics/victoria-metrics:latest \
|
||||
--storageDataPath=/victoria-metrics-data \
|
||||
--retentionPeriod=2y \
|
||||
--httpListenAddr=:8428 \
|
||||
--maxConcurrentInserts=32 \
|
||||
--search.maxQueryDuration=60s \
|
||||
--search.maxConcurrentRequests=16 \
|
||||
--dedup.minScrapeInterval=30s \
|
||||
--memory.allowedPercent=80 \
|
||||
--loggerLevel=INFO \
|
||||
--loggerFormat=json \
|
||||
--search.maxSeries=1000000 \
|
||||
--search.maxPointsPerTimeseries=100000
|
||||
```
|
||||
|
||||
#### Configuration Parameters Explained
|
||||
|
||||
| Parameter | Description | Recommended Value |
|
||||
|-----------|-------------|-------------------|
|
||||
| `--storageDataPath` | Data storage directory | `/victoria-metrics-data` |
|
||||
| `--retentionPeriod` | How long to keep data | `2y` (2 years) |
|
||||
| `--httpListenAddr` | HTTP listen address | `:8428` |
|
||||
| `--maxConcurrentInserts` | Max concurrent inserts | `32` |
|
||||
| `--search.maxQueryDuration` | Max query duration | `60s` |
|
||||
| `--search.maxConcurrentRequests` | Max concurrent queries | `16` |
|
||||
| `--dedup.minScrapeInterval` | Deduplication interval | `30s` |
|
||||
| `--memory.allowedPercent` | Max memory usage | `80` |
|
||||
| `--loggerLevel` | Log level | `INFO` |
|
||||
| `--search.maxSeries` | Max time series | `1000000` |
|
||||
|
||||
### Kubernetes Deployment
|
||||
|
||||
```yaml
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: victoriametrics
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: victoriametrics
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: victoriametrics
|
||||
spec:
|
||||
containers:
|
||||
- name: victoriametrics
|
||||
image: victoriametrics/victoria-metrics:latest
|
||||
ports:
|
||||
- containerPort: 8428
|
||||
args:
|
||||
- --storageDataPath=/victoria-metrics-data
|
||||
- --retentionPeriod=2y
|
||||
- --httpListenAddr=:8428
|
||||
- --maxConcurrentInserts=32
|
||||
volumeMounts:
|
||||
- name: storage
|
||||
mountPath: /victoria-metrics-data
|
||||
resources:
|
||||
requests:
|
||||
memory: "512Mi"
|
||||
cpu: "500m"
|
||||
limits:
|
||||
memory: "2Gi"
|
||||
cpu: "2000m"
|
||||
volumes:
|
||||
- name: storage
|
||||
persistentVolumeClaim:
|
||||
claimName: victoriametrics-pvc
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: victoriametrics
|
||||
spec:
|
||||
selector:
|
||||
app: victoriametrics
|
||||
ports:
|
||||
- port: 8428
|
||||
targetPort: 8428
|
||||
type: ClusterIP
|
||||
```
|
||||
|
||||
## Data Queries
|
||||
|
||||
### HTTP API Queries
|
||||
|
||||
VictoriaMetrics provides a Prometheus-compatible HTTP API:
|
||||
|
||||
```bash
|
||||
# Current water levels for all stations
|
||||
curl "http://localhost:8428/api/v1/query?query=water_level"
|
||||
|
||||
# Water levels for specific station
|
||||
curl "http://localhost:8428/api/v1/query?query=water_level{station_code=\"P.1\"}"
|
||||
|
||||
# Average discharge over last hour
|
||||
curl "http://localhost:8428/api/v1/query?query=avg_over_time(water_discharge[1h])"
|
||||
|
||||
# High discharge alerts (>80%)
|
||||
curl "http://localhost:8428/api/v1/query?query=water_discharge_percent>80"
|
||||
|
||||
# Time range query (last 6 hours)
|
||||
START=$(date -d '6 hours ago' +%s)
|
||||
END=$(date +%s)
|
||||
curl "http://localhost:8428/api/v1/query_range?query=water_level&start=${START}&end=${END}&step=300"
|
||||
```
|
||||
|
||||
### PromQL Examples
|
||||
|
||||
```promql
|
||||
# Current water levels
|
||||
water_level
|
||||
|
||||
# Water level trends (last 24h)
|
||||
water_level[24h]
|
||||
|
||||
# Discharge rates by station
|
||||
water_discharge{station_code="P.1"}
|
||||
|
||||
# Average discharge across all stations
|
||||
avg(water_discharge)
|
||||
|
||||
# Stations with high discharge (>80%)
|
||||
water_discharge_percent > 80
|
||||
|
||||
# Rate of change in water level
|
||||
rate(water_level[5m])
|
||||
|
||||
# Maximum water level in last hour
|
||||
max_over_time(water_level[1h])
|
||||
|
||||
# Stations with increasing water levels
|
||||
increase(water_level[1h]) > 0
|
||||
```
|
||||
|
||||
## Grafana Integration
|
||||
|
||||
### Data Source Configuration
|
||||
|
||||
1. **Add VictoriaMetrics as Prometheus Data Source**:
|
||||
- URL: `http://localhost:8428` (or `http://victoriametrics:8428` in Docker)
|
||||
- Access: Server (default)
|
||||
- HTTP Method: POST
|
||||
|
||||
2. **Import Dashboard**:
|
||||
- Use the provided `water-monitoring-dashboard.json`
|
||||
- Or create custom dashboards with the queries above
|
||||
|
||||
### Dashboard Panels
|
||||
|
||||
The included dashboard provides:
|
||||
|
||||
- **Time Series**: Water levels and discharge over time
|
||||
- **Table**: Current status of all stations
|
||||
- **Pie Chart**: Discharge percentage distribution
|
||||
- **Gauge**: Average discharge percentage
|
||||
- **Variables**: Filter by station
|
||||
|
||||
## Monitoring and Maintenance
|
||||
|
||||
### Health Checks
|
||||
|
||||
```bash
|
||||
# Check VictoriaMetrics health
|
||||
curl http://localhost:8428/health
|
||||
|
||||
# Check metrics endpoint
|
||||
curl http://localhost:8428/metrics
|
||||
|
||||
# Check configuration
|
||||
curl http://localhost:8428/api/v1/status/config
|
||||
```
|
||||
|
||||
### Performance Monitoring
|
||||
|
||||
```bash
|
||||
# Query performance stats
|
||||
curl http://localhost:8428/api/v1/status/tsdb
|
||||
|
||||
# Memory usage
|
||||
curl http://localhost:8428/api/v1/status/runtime
|
||||
|
||||
# Active queries
|
||||
curl http://localhost:8428/api/v1/status/active_queries
|
||||
```
|
||||
|
||||
### Backup and Restore
|
||||
|
||||
```bash
|
||||
# Create backup
|
||||
docker exec victoriametrics /usr/bin/vmbackup \
|
||||
-storageDataPath=/victoria-metrics-data \
|
||||
-dst=fs:///backup/$(date +%Y%m%d)
|
||||
|
||||
# Restore from backup
|
||||
docker exec victoriametrics /usr/bin/vmrestore \
|
||||
-src=fs:///backup/20250724 \
|
||||
-storageDataPath=/victoria-metrics-data
|
||||
```
|
||||
|
||||
### Log Analysis
|
||||
|
||||
```bash
|
||||
# View logs
|
||||
docker logs victoriametrics
|
||||
|
||||
# Follow logs
|
||||
docker logs -f victoriametrics
|
||||
|
||||
# Search for errors
|
||||
docker logs victoriametrics 2>&1 | grep ERROR
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Common Issues
|
||||
|
||||
1. **Connection Refused**:
|
||||
```bash
|
||||
# Check if VictoriaMetrics is running
|
||||
docker ps | grep victoriametrics
|
||||
|
||||
# Check port binding
|
||||
netstat -tlnp | grep 8428
|
||||
```
|
||||
|
||||
2. **High Memory Usage**:
|
||||
```bash
|
||||
# Reduce memory limit
|
||||
docker run ... --memory.allowedPercent=60 ...
|
||||
```
|
||||
|
||||
3. **Slow Queries**:
|
||||
```bash
|
||||
# Increase query timeout
|
||||
docker run ... --search.maxQueryDuration=120s ...
|
||||
```
|
||||
|
||||
4. **Data Not Appearing**:
|
||||
```bash
|
||||
# Check if data is being written
|
||||
curl "http://localhost:8428/api/v1/query?query=up"
|
||||
|
||||
# Check water monitor logs
|
||||
tail -f water_monitor.log
|
||||
```
|
||||
|
||||
### Performance Tuning
|
||||
|
||||
1. **For High Write Load**:
|
||||
```bash
|
||||
--maxConcurrentInserts=64
|
||||
--insert.maxQueueDuration=60s
|
||||
```
|
||||
|
||||
2. **For High Query Load**:
|
||||
```bash
|
||||
--search.maxConcurrentRequests=32
|
||||
--search.maxQueryDuration=120s
|
||||
```
|
||||
|
||||
3. **For Large Datasets**:
|
||||
```bash
|
||||
--search.maxSeries=10000000
|
||||
--search.maxPointsPerTimeseries=1000000
|
||||
```
|
||||
|
||||
## Security
|
||||
|
||||
### Authentication
|
||||
|
||||
VictoriaMetrics doesn't have built-in authentication. Use a reverse proxy:
|
||||
|
||||
```nginx
|
||||
server {
|
||||
listen 80;
|
||||
server_name victoriametrics.example.com;
|
||||
|
||||
auth_basic "VictoriaMetrics";
|
||||
auth_basic_user_file /etc/nginx/.htpasswd;
|
||||
|
||||
location / {
|
||||
proxy_pass http://localhost:8428;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### TLS/SSL
|
||||
|
||||
```bash
|
||||
# Use nginx or traefik for TLS termination
|
||||
# Or use VictoriaMetrics with TLS:
|
||||
docker run ... \
|
||||
-v /path/to/cert.pem:/cert.pem \
|
||||
-v /path/to/key.pem:/key.pem \
|
||||
victoriametrics/victoria-metrics:latest \
|
||||
--tls \
|
||||
--tlsCertFile=/cert.pem \
|
||||
--tlsKeyFile=/key.pem
|
||||
```
|
||||
|
||||
## Scaling
|
||||
|
||||
### Cluster Setup
|
||||
|
||||
For high availability and horizontal scaling:
|
||||
|
||||
```bash
|
||||
# Start multiple VictoriaMetrics instances
|
||||
docker run -d --name vm1 -p 8428:8428 victoriametrics/victoria-metrics:latest
|
||||
docker run -d --name vm2 -p 8429:8428 victoriametrics/victoria-metrics:latest
|
||||
|
||||
# Use load balancer to distribute queries
|
||||
# Use vminsert/vmselect/vmstorage for true clustering
|
||||
```
|
||||
|
||||
### Resource Requirements
|
||||
|
||||
| Data Points/Hour | RAM | CPU | Storage/Day |
|
||||
|------------------|-----|-----|-------------|
|
||||
| 1,000 | 100MB | 0.1 CPU | 10MB |
|
||||
| 10,000 | 500MB | 0.5 CPU | 100MB |
|
||||
| 100,000 | 2GB | 1 CPU | 1GB |
|
||||
| 1,000,000 | 8GB | 2 CPU | 10GB |
|
||||
|
||||
## Migration
|
||||
|
||||
### From InfluxDB
|
||||
|
||||
```bash
|
||||
# Export from InfluxDB
|
||||
influx -database water_monitoring -execute "SELECT * FROM water_data" -format csv > data.csv
|
||||
|
||||
# Import to VictoriaMetrics (convert to Prometheus format first)
|
||||
# Use vmctl tool for migration
|
||||
```
|
||||
|
||||
### From Prometheus
|
||||
|
||||
```bash
|
||||
# Use vmctl for direct migration
|
||||
vmctl prometheus --prom-snapshot=/path/to/prometheus/data --vm-addr=http://localhost:8428
|
||||
```
|
||||
|
||||
This comprehensive setup guide should help you configure VictoriaMetrics for optimal performance with the Thailand Water Monitor system.
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 122 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 522 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 201 KiB |
@@ -1,543 +0,0 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 0,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"liveNow": false,
|
||||
"panels": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "victoriametrics-uid"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"vis": false
|
||||
},
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 2,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "m"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 1,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "victoriametrics-uid"
|
||||
},
|
||||
"expr": "water_level",
|
||||
"interval": "",
|
||||
"legendFormat": "{{station_code}} - {{station_name_en}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Water Levels by Station",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "victoriametrics-uid"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"vis": false
|
||||
},
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 2,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "cms"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 0
|
||||
},
|
||||
"id": 2,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "victoriametrics-uid"
|
||||
},
|
||||
"expr": "water_discharge",
|
||||
"interval": "",
|
||||
"legendFormat": "{{station_code}} - {{station_name_en}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Water Discharge by Station",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "victoriametrics-uid"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"custom": {
|
||||
"align": "auto",
|
||||
"displayMode": "auto",
|
||||
"inspect": false
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "Water Level"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "unit",
|
||||
"value": "m"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "Discharge"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "unit",
|
||||
"value": "cms"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "Discharge %"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "unit",
|
||||
"value": "percent"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 8
|
||||
},
|
||||
"id": 3,
|
||||
"options": {
|
||||
"showHeader": true,
|
||||
"sortBy": [
|
||||
{
|
||||
"desc": false,
|
||||
"displayName": "Station"
|
||||
}
|
||||
]
|
||||
},
|
||||
"pluginVersion": "8.5.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "victoriametrics-uid"
|
||||
},
|
||||
"expr": "water_level",
|
||||
"format": "table",
|
||||
"instant": true,
|
||||
"interval": "",
|
||||
"legendFormat": "",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "victoriametrics-uid"
|
||||
},
|
||||
"expr": "water_discharge",
|
||||
"format": "table",
|
||||
"instant": true,
|
||||
"interval": "",
|
||||
"legendFormat": "",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "victoriametrics-uid"
|
||||
},
|
||||
"expr": "water_discharge_percent",
|
||||
"format": "table",
|
||||
"instant": true,
|
||||
"interval": "",
|
||||
"legendFormat": "",
|
||||
"refId": "C"
|
||||
}
|
||||
],
|
||||
"title": "Current Station Status",
|
||||
"transformations": [
|
||||
{
|
||||
"id": "merge",
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": "organize",
|
||||
"options": {
|
||||
"excludeByName": {
|
||||
"Time": true,
|
||||
"__name__": true,
|
||||
"job": true,
|
||||
"instance": true
|
||||
},
|
||||
"indexByName": {
|
||||
"station_code": 0,
|
||||
"station_name_en": 1,
|
||||
"station_name_th": 2,
|
||||
"Value #A": 3,
|
||||
"Value #B": 4,
|
||||
"Value #C": 5
|
||||
},
|
||||
"renameByName": {
|
||||
"Value #A": "Water Level",
|
||||
"Value #B": "Discharge",
|
||||
"Value #C": "Discharge %",
|
||||
"station_code": "Station",
|
||||
"station_name_en": "English Name",
|
||||
"station_name_th": "Thai Name"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"type": "table"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "victoriametrics-uid"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"vis": false
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"unit": "percent"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 16
|
||||
},
|
||||
"id": 4,
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "right"
|
||||
},
|
||||
"pieType": "pie",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "victoriametrics-uid"
|
||||
},
|
||||
"expr": "water_discharge_percent",
|
||||
"interval": "",
|
||||
"legendFormat": "{{station_code}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Discharge Percentage Distribution",
|
||||
"type": "piechart"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "victoriametrics-uid"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 70
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 90
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "percent"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 16
|
||||
},
|
||||
"id": 5,
|
||||
"options": {
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showThresholdLabels": false,
|
||||
"showThresholdMarkers": true,
|
||||
"text": {}
|
||||
},
|
||||
"pluginVersion": "8.5.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "victoriametrics-uid"
|
||||
},
|
||||
"expr": "avg(water_discharge_percent)",
|
||||
"interval": "",
|
||||
"legendFormat": "Average Discharge %",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Average Discharge Percentage",
|
||||
"type": "gauge"
|
||||
}
|
||||
],
|
||||
"refresh": "30s",
|
||||
"schemaVersion": 36,
|
||||
"style": "dark",
|
||||
"tags": [
|
||||
"water-monitoring",
|
||||
"thailand",
|
||||
"victoriametrics"
|
||||
],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"current": {
|
||||
"selected": false,
|
||||
"text": "All",
|
||||
"value": "$__all"
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "victoriametrics-uid"
|
||||
},
|
||||
"definition": "label_values(water_level, station_code)",
|
||||
"hide": 0,
|
||||
"includeAll": true,
|
||||
"label": "Station",
|
||||
"multi": true,
|
||||
"name": "station",
|
||||
"options": [],
|
||||
"query": {
|
||||
"query": "label_values(water_level, station_code)",
|
||||
"refId": "StandardVariableQuery"
|
||||
},
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"skipUrlSync": false,
|
||||
"sort": 1,
|
||||
"type": "query"
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": {
|
||||
"from": "now-6h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "Asia/Bangkok",
|
||||
"title": "Thailand Water Level Monitoring",
|
||||
"uid": "water-monitoring-dashboard",
|
||||
"version": 1,
|
||||
"weekStart": ""
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: 'water-monitoring'
|
||||
orgId: 1
|
||||
folder: 'Water Monitoring'
|
||||
type: file
|
||||
disableDeletion: false
|
||||
updateIntervalSeconds: 10
|
||||
allowUiUpdates: true
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
@@ -1,17 +0,0 @@
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: VictoriaMetrics
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://victoriametrics:8428
|
||||
isDefault: true
|
||||
editable: true
|
||||
jsonData:
|
||||
httpMethod: POST
|
||||
queryTimeout: 60s
|
||||
timeInterval: 30s
|
||||
customQueryParameters: ''
|
||||
secureJsonData: {}
|
||||
version: 1
|
||||
uid: victoriametrics-uid
|
||||
+10
-75
@@ -5,122 +5,57 @@ build-backend = "setuptools.build_meta"
|
||||
[project]
|
||||
name = "northern-thailand-ping-river-monitor"
|
||||
version = "3.1.3"
|
||||
description = "Real-time water level monitoring system for the Ping River Basin in Northern Thailand"
|
||||
description = "Ping River Monitor: flood-model training, evaluation and archive backfills (the service, ping-collector, is in rust/)"
|
||||
readme = "README.md"
|
||||
license = {text = "MIT"}
|
||||
authors = [
|
||||
{name = "Ping River Monitor Team", email = "contact@example.com"}
|
||||
{name = "buildfor.life"}
|
||||
]
|
||||
keywords = [
|
||||
"water monitoring",
|
||||
"hydrology",
|
||||
"flood forecasting",
|
||||
"thailand",
|
||||
"ping river",
|
||||
"environmental monitoring",
|
||||
"time series",
|
||||
"fastapi",
|
||||
"real-time data"
|
||||
]
|
||||
classifiers = [
|
||||
"Development Status :: 4 - Beta",
|
||||
"Development Status :: 5 - Production/Stable",
|
||||
"Intended Audience :: Science/Research",
|
||||
"Intended Audience :: System Administrators",
|
||||
"Topic :: Scientific/Engineering :: Hydrology",
|
||||
"Topic :: System :: Monitoring",
|
||||
"Programming Language :: Python :: 3",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Operating System :: OS Independent",
|
||||
"Environment :: Web Environment",
|
||||
"Framework :: FastAPI"
|
||||
]
|
||||
# 3.11 only: pandas 2.0.3 ships no 3.12 wheels; psycopg2-binary 2.9.9 breaks on 3.13
|
||||
requires-python = ">=3.11,<3.12"
|
||||
dependencies = [
|
||||
# Core dependencies
|
||||
"requests==2.34.2",
|
||||
"schedule==1.2.0",
|
||||
"pandas==2.0.3",
|
||||
"numpy>=1.24,<2",
|
||||
# Flood forecasting (ML)
|
||||
"numpy>=1.24,<2", # pandas 2.0.3 wheels are ABI-incompatible with numpy 2.x
|
||||
"scikit-learn==1.9.0",
|
||||
# Web API framework
|
||||
"fastapi==0.141.1",
|
||||
"uvicorn[standard]==0.52.4",
|
||||
"pydantic==2.13.5",
|
||||
# Database adapters
|
||||
"sqlalchemy==2.0.23",
|
||||
"influxdb==5.3.1",
|
||||
"pymysql==1.2.0",
|
||||
"psycopg2-binary==2.9.9",
|
||||
# Monitoring and metrics
|
||||
"psutil==5.9.6"
|
||||
# .env loading; rust/collector/src/envfile.rs ports its parser (the 614-case
|
||||
# fixture is identical for 1.1.1 and 1.2.2; regenerate it when bumping)
|
||||
"python-dotenv==1.2.2",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
# Testing
|
||||
"pytest==9.1.1",
|
||||
"pytest-cov==4.1.0",
|
||||
"pytest-asyncio==0.21.1",
|
||||
# Code formatting and linting
|
||||
"black==26.5.1",
|
||||
"flake8==6.1.0",
|
||||
"isort==5.12.0",
|
||||
"mypy==1.7.1",
|
||||
# Pre-commit hooks
|
||||
"pre-commit==3.5.0",
|
||||
# Development tools
|
||||
"ipython==8.17.2",
|
||||
"jupyter==1.0.0",
|
||||
# Type stubs
|
||||
"types-requests==2.33.0.20260906",
|
||||
"types-python-dateutil==2.8.19.14"
|
||||
]
|
||||
docs = [
|
||||
"sphinx==7.2.6",
|
||||
"sphinx-rtd-theme==1.3.0",
|
||||
"sphinx-autodoc-typehints==1.25.2"
|
||||
]
|
||||
all = [
|
||||
"influxdb==5.3.1",
|
||||
"pymysql==1.2.0",
|
||||
"psycopg2-binary==2.9.9"
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
ping-river-monitor = "src.main:main"
|
||||
ping-river-api = "src.web_api:main"
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor"
|
||||
Homepage = "https://water.buildfor.life/"
|
||||
Repository = "https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor"
|
||||
Issues = "https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/issues"
|
||||
Documentation = "https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/wiki"
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
# Testing
|
||||
"pytest==9.1.1",
|
||||
"pytest-cov==4.1.0",
|
||||
"pytest-asyncio==0.21.1",
|
||||
# Code formatting and linting
|
||||
"black==26.5.1",
|
||||
"flake8==6.1.0",
|
||||
"isort==5.12.0",
|
||||
"mypy==1.7.1",
|
||||
# Pre-commit hooks
|
||||
"pre-commit==3.5.0",
|
||||
# Development tools
|
||||
"ipython==8.17.2",
|
||||
"jupyter==1.0.0",
|
||||
# Type stubs
|
||||
"types-requests==2.33.0.20260906",
|
||||
"types-python-dateutil==2.8.19.14",
|
||||
# Documentation
|
||||
"sphinx==7.2.6",
|
||||
"sphinx-rtd-theme==1.3.0",
|
||||
"sphinx-autodoc-typehints==1.25.2",
|
||||
"pyinstaller>=6.16.0",
|
||||
]
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
|
||||
@@ -1,29 +1,8 @@
|
||||
# Development dependencies
|
||||
-r requirements.txt
|
||||
|
||||
# Testing
|
||||
pytest==9.1.1
|
||||
pytest-cov==4.1.0
|
||||
pytest-asyncio==0.21.1
|
||||
|
||||
# Code formatting and linting
|
||||
black==26.5.1
|
||||
flake8==6.1.0
|
||||
isort==5.12.0
|
||||
mypy==1.7.1
|
||||
|
||||
# Pre-commit hooks
|
||||
pre-commit==3.5.0
|
||||
|
||||
# Documentation
|
||||
sphinx==7.2.6
|
||||
sphinx-rtd-theme==1.3.0
|
||||
sphinx-autodoc-typehints==1.25.2
|
||||
|
||||
# Development tools
|
||||
ipython==8.17.2
|
||||
jupyter==1.0.0
|
||||
|
||||
# Type stubs
|
||||
types-requests==2.33.0.20260906
|
||||
types-python-dateutil==2.8.19.14
|
||||
+3
-28
@@ -1,34 +1,9 @@
|
||||
# Core dependencies
|
||||
# Python side: flood-model training, evaluation, backfills (pyproject.toml is
|
||||
# the source; uv.lock pins the rest). The service is ping-collector (rust/).
|
||||
requests==2.34.2
|
||||
schedule==1.2.0
|
||||
pandas==2.0.3
|
||||
numpy>=1.24,<2 # pandas 2.0.3 wheels are ABI-incompatible with numpy 2.x
|
||||
|
||||
# Flood forecasting (ML)
|
||||
scikit-learn==1.9.0
|
||||
|
||||
# Web API framework
|
||||
fastapi==0.141.1
|
||||
uvicorn[standard]==0.52.4
|
||||
pydantic==2.13.5
|
||||
|
||||
# Database adapters
|
||||
sqlalchemy==2.0.23
|
||||
influxdb==5.3.1
|
||||
pymysql==1.2.0
|
||||
psycopg2-binary==2.9.9
|
||||
|
||||
# Monitoring and metrics
|
||||
psutil==5.9.6
|
||||
|
||||
# Development dependencies (optional)
|
||||
pytest==9.1.1
|
||||
pytest-cov==4.1.0
|
||||
black==26.5.1
|
||||
flake8==6.1.0
|
||||
mypy==1.7.1
|
||||
pre-commit==3.5.0
|
||||
|
||||
# Documentation
|
||||
sphinx==7.2.6
|
||||
sphinx-rtd-theme==1.3.0
|
||||
python-dotenv==1.2.2
|
||||
@@ -1,15 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Simple startup script for Thailand Water Monitor
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
# Add src directory to Python path
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "src"))
|
||||
|
||||
if __name__ == "__main__":
|
||||
from src.main import main
|
||||
|
||||
main()
|
||||
+90
-125
@@ -1,163 +1,134 @@
|
||||
# Rust services
|
||||
# Rust: the service
|
||||
|
||||
Rewrite of the Ping River Monitor in Rust, one service at a time. The plan,
|
||||
costs and the parity rules are in [docs/RUST_REWRITE.md](../docs/RUST_REWRITE.md).
|
||||
Since October 2026 the whole Ping River Monitor service is one Rust binary,
|
||||
`ping-collector`; Python is left with model training and evaluation
|
||||
(`src/ml`) and the archive backfills. How it got here, and the parity rules
|
||||
the rewrite followed: [docs/RUST_REWRITE.md](../docs/RUST_REWRITE.md).
|
||||
|
||||
| Crate | Replaces | Status |
|
||||
| --- | --- | --- |
|
||||
| `collector` (`ping-collector`) | RID, HII/ThaiWater, RID dams and Open-Meteo collection in the Python web app | phase 1: in production since 2026-10-17 |
|
||||
| `forecast` (`ping-forecast`, library) | the flood forecast (`src/ml` serving path); run by `ping-collector` with ntfy | phase 2: in production since 2026-10-18 |
|
||||
| `collector`: `src/web` | the FastAPI app (`src/web_api.py`): dashboard, read API, `/docs` | phase 3: in production since 2026-10-18 |
|
||||
| Crate | What |
|
||||
| --- | --- |
|
||||
| `collector` (`ping-collector`) | collection (RID, HII/ThaiWater, RID dams, Open-Meteo), the forecast run, ntfy + Matrix notifications, the website and API (`collector/src/web`) |
|
||||
| `forecast` (`ping-forecast`, library) | flood-forecast inference from the models trained in Python: features, HII gap fill, gradient-boosted trees, heuristic fallback |
|
||||
|
||||
## ping-collector
|
||||
|
||||
Collects into the same PostgreSQL tables as the Python app, so the two can be
|
||||
swapped (and swapped back) with no data migration. The Python web app keeps
|
||||
serving the API, forecasting and sending ntfy alerts with
|
||||
`COLLECTION_ENABLED=false`, and re-forecasts whenever the newest RID reading in
|
||||
the database advances.
|
||||
|
||||
```text
|
||||
ping-collector run # the service: startup gap fill, RID at :05 (+10 min retries
|
||||
# while RID is behind), HII/dams/rain once per clock hour;
|
||||
# with FORECAST_IN_COLLECTOR=true also forecast + ntfy/Matrix;
|
||||
# with WEB_LISTEN the website and API
|
||||
ping-collector run # the service (systemd): startup gap fill, RID at :05 (+10 min
|
||||
# retries while RID is behind), HII/dams/rain once per clock
|
||||
# hour; FORECAST_IN_COLLECTOR=true: forecast + ntfy/Matrix
|
||||
# after each cycle with newer readings; WEB_LISTEN: website/API
|
||||
ping-collector serve [--listen A] [--no-analytics]
|
||||
# the website and API only (default 127.0.0.1:8001)
|
||||
# the website and API alone (default 127.0.0.1:8001)
|
||||
ping-collector once [--dry-run] # one cycle of every source; --dry-run writes nothing
|
||||
# and compares the RID fetch with the database
|
||||
ping-collector fill-gaps <days|all> # re-fetch RID reports with holes
|
||||
ping-collector refetch FROM [TO] # re-fetch the RID reports of FROM..=TO (YYYY-MM-DD) even
|
||||
# without holes, through the spike screening: corrects
|
||||
# readings RID revised, deletes stored spikes; prints the
|
||||
# catch-up-forecasts command when stored levels changed
|
||||
ping-collector check-db # connect, create missing tables, row counts
|
||||
ping-collector forecast [--json] [--save] [--now T] [--rain-file F]
|
||||
# forecast from the database and print it
|
||||
ping-collector catch-up-forecasts [--hours N] [--redo-from T]
|
||||
# forecast the hours whose readings came late (RID's
|
||||
# overnight batch) as a run then would have; default 336 h;
|
||||
# --redo-from first deletes the model forecasts from T on
|
||||
ping-collector notify --dry-run # the readings the notifier judges, and what it would send
|
||||
```
|
||||
|
||||
Configuration is the Python app's `.env` (read from the working directory, or
|
||||
`--env-file`) with python-dotenv's rules, verified on 614 generated inputs
|
||||
(`collector/testdata/`): an unparsable line is skipped with a warning giving
|
||||
only its line number, never its value. Variables already set (e.g. by systemd's
|
||||
`EnvironmentFile=`) win. Used keys:
|
||||
`POSTGRES_CONNECTION_STRING` (or `POSTGRES_HOST/PORT/DB/USER/PASSWORD`),
|
||||
`RID_POLL_OFFSET_MINUTES` (5), `RID_RETRY_MINUTES` (10),
|
||||
`GAP_FILL_DAYS_ON_START` (14), `ENABLE_HII_COLLECTION`,
|
||||
`ENABLE_RESERVOIR_COLLECTION`, `ENABLE_RAIN_COLLECTION`, `HII_BASIN_CODE` (6),
|
||||
`STATION_CONFIG_PATH` (falls back to the bundled `src/data/stations.json`),
|
||||
`SCRAPING_STATUS_FILE` (status JSON for `/scraping/status` when the website runs elsewhere),
|
||||
`LOG_LEVEL` / `RUST_LOG`; for phase 2 `FORECAST_IN_COLLECTOR` (false),
|
||||
`MODELS_DIR` (`models`), `NTFY_SERVER` / `NTFY_PUBLISH_URL` / `NTFY_TOPIC_PREFIX`
|
||||
/ `NTFY_TOKEN` / `PUBLIC_URL` (as the Python app), `RID_STALE_ALERT_HOURS` (18,
|
||||
0 = off). All timestamps are Asia/Bangkok wall-clock, whatever the server's
|
||||
time zone.
|
||||
Configuration: the environment, or `.env` in the working directory
|
||||
(`--env-file` for another), every key with its default in
|
||||
[.env.example](../.env.example). `.env` is parsed with python-dotenv's rules
|
||||
(`collector/src/envfile.rs`, verified on 614 generated inputs) because the
|
||||
Python tools read the same file: an unparsable line is skipped with a warning
|
||||
giving only its line number, never its value. Variables already set (e.g. by
|
||||
systemd's `EnvironmentFile=`) win. All timestamps are Asia/Bangkok wall-clock,
|
||||
whatever the server's time zone.
|
||||
|
||||
Not ported: the one-off archive backfills (`scripts/backfill_*.py`), which
|
||||
stay in Python.
|
||||
### Collection
|
||||
|
||||
### Verified
|
||||
The tables, upserts and parsers are the ones the Python collectors had (the
|
||||
DDL is in `collector/src/db.rs`; the Python backfill stores create the same).
|
||||
RID's evening report often lags 1-2 h: every poll stores every reading the
|
||||
report has, yesterday's report is re-fetched while it has holes, and the
|
||||
startup gap fill repairs the last `GAP_FILL_DAYS_ON_START` days.
|
||||
|
||||
- 20 tests: parsers, gap/schedule logic, and database tests against a real
|
||||
PostgreSQL (late reports, midnight recovery, upsert revisions, NUMERIC
|
||||
rounding, dam metadata preservation).
|
||||
- Live parity: the Python collectors and `ping-collector` run back-to-back
|
||||
against the real upstream APIs into two empty databases produced identical
|
||||
rows in all 9 tables (16 stations, 1,104 RID readings, 103 + 76 HII rows,
|
||||
35 dams x 2 days, 168 rain hours). Rerun with
|
||||
`scripts/collector_parity.py` after changing a parser, table or upsert on
|
||||
either side.
|
||||
- The Python API with `COLLECTION_ENABLED=false` served `/health`,
|
||||
`/measurements/latest`, `/api/stats`, `/scraping/status` and model
|
||||
forecasts from a database filled only by `ping-collector`.
|
||||
### Forecast and notifications
|
||||
|
||||
## Forecast and notifications (phase 2)
|
||||
After every cycle that brought newer RID or HII readings (and at least
|
||||
hourly) the forecast runs over the last 336 h, is stored in `forecast_history`
|
||||
(for skill tracking) and `forecast_current`, and the notifier compares the
|
||||
readings and forecasts with `notification_state`.
|
||||
|
||||
With `FORECAST_IN_COLLECTOR=true` in the shared `.env`, `ping-collector` runs
|
||||
the flood forecast after every cycle that brought newer RID or HII readings
|
||||
(and at least hourly), stores it in `forecast_history` and `forecast_current`
|
||||
(the Python API serves the latter as `/forecast`, falling back to computing
|
||||
itself when it is older than `FORECAST_MAX_AGE_SECONDS`), and sends the ntfy
|
||||
notifications. The Python app then neither forecasts nor notifies.
|
||||
|
||||
- **Models** stay trained in Python. `src/ml/export.py` writes
|
||||
- **Models** are trained in Python. `src/ml/export.py` writes
|
||||
`models/flood_<station>.json.gz` next to each joblib bundle (`train_all()`
|
||||
does it, `retrain.sh` promotes it, `python -m src.ml.export` exports
|
||||
existing bundles); the collector reloads changed files on its next run.
|
||||
- **Shared constants** (thresholds, flood stages, upstream leads, notify
|
||||
does it; `python -m src.ml.export` for existing bundles); the service
|
||||
reloads changed files on its next run, so a retrain needs no restart.
|
||||
- **Shared constants** (thresholds, flood stages, upstream leads, notifier
|
||||
settings) are compiled in from `forecast/flood_config.json`, generated by
|
||||
`uv run python -m src.ml.export --config`. `tests/test_model_export.py`
|
||||
fails while it differs from Python, and a change needs a collector rebuild.
|
||||
- **Notifications**: same topics, texts, hysteresis and `notification_state`
|
||||
keys as `src/notify.py`, so switching sends nothing twice. One change: for
|
||||
the stations whose ThaiWater twin is the same sensor, a reading newer than
|
||||
RID's (RID's report lags 2-10 h at night) drives the level alerts, marked
|
||||
"ThaiWater telemetry; the RID report is not in yet". "Gauge feed stale" now
|
||||
means no source has a reading for 3 h; RID alone lagging is reported once
|
||||
after `RID_STALE_ALERT_HOURS`.
|
||||
- Memory: 70 MB peak with all 15 models loaded (24-byte tree nodes).
|
||||
fails while it differs from Python; a change needs a rebuild.
|
||||
- **ntfy** topics, texts and hysteresis: [docs/NOTIFICATIONS.md](../docs/NOTIFICATIONS.md).
|
||||
For the stations whose ThaiWater twin is the same sensor, a reading newer
|
||||
than RID's drives the level alerts, marked as ThaiWater telemetry. "Gauge
|
||||
feed stale" means no source has a reading for 3 h, except during RID's
|
||||
nightly pause (21:00 to `FEED_NIGHT_UNTIL_HOUR`, 9) while the river is
|
||||
quiet; RID alone lagging is reported once after `RID_STALE_ALERT_HOURS`.
|
||||
- **Matrix** (`MATRIX_*`): the room gets the same transitions (per station,
|
||||
outlook, feed; not ntfy's per-topic summaries), each once, tracked under
|
||||
its own `matrix:` keys in `notification_state`, so a Matrix outage is
|
||||
retried on Matrix only. [docs/MATRIX_QUICK_START.md](../docs/MATRIX_QUICK_START.md).
|
||||
- Memory: under 100 MB resident in production, all 15 models loaded
|
||||
(24-byte tree nodes); the unit allows 512 MB.
|
||||
|
||||
Verified:
|
||||
### Website and API
|
||||
|
||||
- `forecast/tests/parity.rs` replays fixtures from the Python production path
|
||||
(`forecast/testdata/gen_forecast_fixtures.py`: the 2024 flood rising and at
|
||||
its peak with RID lagging behind HII, a quiet window with dam data, a
|
||||
stale-feed window, moved thresholds). HII fill, every feature and every
|
||||
level are bit-identical; with all 15 local models 377 of 384 probabilities
|
||||
are bit-identical and the rest within 2 ulp (`exp`). CI checks the P.1 model.
|
||||
- `scripts/forecast_parity.py` does the same on the live database: Python and
|
||||
`ping-collector forecast --json` at the same moment with the same rain
|
||||
series. The install script runs it before `--cutover-forecast`.
|
||||
- `collector/testdata/gen_notify_cases.py`: 652 hourly steps through
|
||||
`src/notify.py` (thresholds, capacity guard, outlook, feed gaps, ntfy
|
||||
outages); the Rust notifier sends the same 751 messages word for word.
|
||||
|
||||
## Website and API (phase 3)
|
||||
|
||||
`src/web` serves what the FastAPI app served: the dashboard
|
||||
(`STATIC_DIR`/dashboard.html, default `src/static`, loaded at start), the
|
||||
crawler files, `/static`, `/docs` + `/redoc` + `/openapi.json`, and every API
|
||||
route with the same parameters, JSON (field order included), status codes,
|
||||
`{"detail": ...}` errors, Cache-Control, gzip, CORS (`CORS_ALLOW_ORIGINS`),
|
||||
Umami API tracking (`UMAMI_*`) and response caches (`*_CACHE_TTL_SECONDS`).
|
||||
Admin routes need `ADMIN_API_KEY`. With `WEB_LISTEN=0.0.0.0:8000` it runs
|
||||
inside `ping-collector run` (one process collects, forecasts, notifies and
|
||||
serves; `/scraping/status` and `POST /scrape/trigger` then talk to the
|
||||
collection loop directly); `serve` runs it alone.
|
||||
`collector/src/web` (axum) serves the dashboard (`STATIC_DIR`/dashboard.html,
|
||||
default `src/static`, loaded at start), `/static`, the crawler files,
|
||||
`/docs` + `/redoc` + `/openapi.json`, and the read API, with gzip,
|
||||
Cache-Control per path, CORS (`CORS_ALLOW_ORIGINS`), Umami API tracking
|
||||
(`UMAMI_*`) and stale-while-revalidate response caches with byte budgets.
|
||||
Station editing, `/config` and `POST /scrape/trigger` need `ADMIN_API_KEY`
|
||||
(`X-API-Key` header).
|
||||
|
||||
- Parameters are declared once per route (`web/params.rs`): the same list
|
||||
validates the request (pydantic-style 422s) and builds the OpenAPI
|
||||
document (`web/openapi.rs`), so `/docs` cannot drift from what is accepted.
|
||||
The component schemas are FastAPI's (`web/openapi_components.json`).
|
||||
- Python's output conventions are kept where they show: `isoformat()`
|
||||
timestamps, `round()`, numpy's pairwise sums for the catchment means
|
||||
(`web/pyfmt.rs`), the skill metrics' summation order.
|
||||
- `scripts/api_parity.py --python URL --rust URL` compares the two live on
|
||||
one database; `install_collector.sh --check-api` / `--cutover-api` run it
|
||||
on the server against `ping-collector serve` on port 8001.
|
||||
- The response formats are the retired FastAPI app's, kept so clients did
|
||||
not notice the switch: field order, `isoformat()` timestamps, Python's
|
||||
`round()`, numpy's pairwise sums for the catchment means (`web/pyfmt.rs`),
|
||||
`{"detail": ...}` errors.
|
||||
- An old stored forecast is served marked `X-Data-Stale: true`; `/health`
|
||||
counts current ThaiWater telemetry of the same gauges while RID lags.
|
||||
|
||||
Matrix (`MATRIX_HOMESERVER`, `MATRIX_ACCESS_TOKEN`, `MATRIX_ROOM_ID`): the
|
||||
notifier sends the room the same transitions as ntfy (per station, outlook,
|
||||
feed; not ntfy's per-topic summaries), each once, with its own `matrix:` keys
|
||||
in `notification_state` (seeded from the ntfy ones on first use, so nothing
|
||||
already announced is repeated). It replaces the `run.py --alert-check` cron
|
||||
job, which the install script comments out.
|
||||
|
||||
### Develop
|
||||
## Develop
|
||||
|
||||
```bash
|
||||
cd rust
|
||||
cargo fmt --all && cargo clippy --all-targets -- -D warnings
|
||||
docker run -d --name ping-pg -e POSTGRES_PASSWORD=ping -e POSTGRES_DB=ping -p 55432:5432 postgres:16
|
||||
TEST_DATABASE_URL=postgres://postgres:ping@localhost:55432/ping cargo test
|
||||
CARGO_TARGET_DIR=target/msrv cargo +1.88 check --all-targets --locked # the minimum version
|
||||
```
|
||||
|
||||
Without `TEST_DATABASE_URL` the database tests skip. CI
|
||||
(`.gitea/workflows/rust.yml`) runs them against a PostgreSQL service and
|
||||
uploads the Linux binary as the `ping-collector-linux-x86_64` artifact.
|
||||
|
||||
- The forecast's parity with the Python reference: `forecast/tests/parity.rs`
|
||||
replays fixtures from the Python production path
|
||||
(`forecast/testdata/gen_forecast_fixtures.py`; regenerate after a change to
|
||||
`src/ml/features.py`, `predict.py`, the HII fill or the dam frame). Features
|
||||
and levels must stay bit-identical, probabilities within a few ulp.
|
||||
`scripts/forecast_parity.py` compares the two on a live database.
|
||||
- The notifier's texts are pinned by `collector/testdata/notify_python_cases.json.gz`
|
||||
(751 messages recorded from the Python notifier before it was retired).
|
||||
|
||||
HTTP uses native-tls (OpenSSL on Linux), not rustls: app.rid.go.th (dams)
|
||||
only offers TLS 1.2 CBC cipher suites, which rustls refuses. Building on
|
||||
Linux needs `pkg-config` and `libssl-dev`.
|
||||
|
||||
### Deploy
|
||||
## Deploy
|
||||
|
||||
Needs Rust >= 1.88. Distro `cargo` packages are older (`resolver 3 is not
|
||||
valid` / `edition 2024` errors), so install the toolchain once with rustup:
|
||||
@@ -169,21 +140,15 @@ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profil
|
||||
cargo --version # >= 1.88
|
||||
```
|
||||
|
||||
Then on the server, after `git pull` (the Python side of the switch ships in
|
||||
the same commit):
|
||||
Then on the server, in the checkout:
|
||||
|
||||
```bash
|
||||
git pull
|
||||
cargo build --release --locked --manifest-path rust/Cargo.toml # or BIN=<CI artifact>
|
||||
sudo bash scripts/install_collector.sh # install + checks, nothing switched
|
||||
sudo bash scripts/install_collector.sh --cutover # Rust collects from now on
|
||||
sudo bash scripts/install_collector.sh --cutover-forecast # Rust also forecasts and notifies
|
||||
sudo bash scripts/install_collector.sh --check-api # compare the Rust API with FastAPI
|
||||
sudo bash scripts/install_collector.sh --cutover-api # Rust serves the website on :8000
|
||||
sudo bash scripts/install_collector.sh --rollback-api # FastAPI serves the website again
|
||||
sudo bash scripts/install_collector.sh --rollback-forecast # forecasts + ntfy back to Python
|
||||
sudo bash scripts/install_collector.sh --rollback # Python does everything again
|
||||
sudo bash scripts/install.sh
|
||||
journalctl -u ping-collector -f
|
||||
```
|
||||
|
||||
Every install exports the models and runs `scripts/forecast_parity.py`;
|
||||
`--cutover-forecast` refuses to switch if it fails (`FORCE=1` overrides).
|
||||
`scripts/install.sh` installs the binary and the units, syncs the Python
|
||||
training environment, runs a dry run of every source and the forecast check,
|
||||
restarts the service and waits for the website (details in its header).
|
||||
@@ -1,5 +1,7 @@
|
||||
//! Configuration from the same environment / `.env` file as the Python app,
|
||||
//! so both services can share `/opt/thailand-water-monitor/.env`.
|
||||
//! Configuration from the environment / `.env` file
|
||||
//! (`/opt/thailand-water-monitor/.env`), which the Python training tools
|
||||
//! (src/config.py) read too. Names and defaults are the ones the Python app
|
||||
//! used.
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use std::env;
|
||||
@@ -33,10 +35,10 @@ pub struct Config {
|
||||
/// Days of RID holes re-fetched at startup (0 = off).
|
||||
pub gap_fill_days_on_start: u32,
|
||||
pub station_config_path: PathBuf,
|
||||
/// Status JSON read by the Python API's /scraping/status.
|
||||
/// Status JSON for /scraping/status when the website runs in a separate
|
||||
/// process (`ping-collector serve`).
|
||||
pub status_file: Option<PathBuf>,
|
||||
/// FORECAST_IN_COLLECTOR: forecast and notify here; the Python API then
|
||||
/// serves the stored forecast and does neither.
|
||||
/// FORECAST_IN_COLLECTOR: forecast and notify (off: collect only).
|
||||
pub forecast_enabled: bool,
|
||||
/// Exported bundles (`flood_<station>.json.gz`, src/ml/export.py).
|
||||
pub models_dir: PathBuf,
|
||||
@@ -48,6 +50,9 @@ pub struct Config {
|
||||
pub public_url: String,
|
||||
/// Report RID lagging this long while ThaiWater is current (0 = never).
|
||||
pub rid_stale_alert_hours: f64,
|
||||
/// FEED_NIGHT_UNTIL_HOUR: from 21:00 until this hour a quiet river's feed
|
||||
/// is not reported stale (RID publishes nothing overnight); 0 = off.
|
||||
pub feed_night_until_hour: u32,
|
||||
/// Public ntfy URL (what subscribers use), shown by /api/notifications.
|
||||
pub ntfy_server: Option<String>,
|
||||
/// MATRIX_*: the notifications also go to this room (off when unset).
|
||||
@@ -62,7 +67,7 @@ pub struct MatrixConfig {
|
||||
pub room_id: String,
|
||||
}
|
||||
|
||||
/// The website and read API (src/web_api.py's settings, same names).
|
||||
/// The website and read API (setting names as the retired Python app had them).
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct WebConfig {
|
||||
/// WEB_LISTEN: `run` also serves the website on this address
|
||||
@@ -209,6 +214,10 @@ impl Config {
|
||||
.var("PUBLIC_URL")
|
||||
.unwrap_or_else(|| "https://water.buildfor.life/".into()),
|
||||
rid_stale_alert_hours: v.number("RID_STALE_ALERT_HOURS", 18.0)?,
|
||||
feed_night_until_hour: match v.number("FEED_NIGHT_UNTIL_HOUR", 9u32)? {
|
||||
h if h < 21 => h,
|
||||
h => bail!("FEED_NIGHT_UNTIL_HOUR must be 0 (off) to 20, got {h}"),
|
||||
},
|
||||
ntfy_server: v.var("NTFY_SERVER"),
|
||||
matrix: match (v.var("MATRIX_ACCESS_TOKEN"), v.var("MATRIX_ROOM_ID")) {
|
||||
(Some(access_token), Some(room_id)) => Some(MatrixConfig {
|
||||
|
||||
@@ -30,8 +30,9 @@ pub async fn connect(url: &str, max_connections: u32) -> Result<PgPool> {
|
||||
.context("connecting to PostgreSQL")
|
||||
}
|
||||
|
||||
/// Idempotent DDL, identical to what the Python app creates (so either side
|
||||
/// can start first against an empty database).
|
||||
/// Idempotent DDL, identical to what the Python app created and the Python
|
||||
/// backfill stores (src/hii_collector.py, src/rid_reservoir.py) still create,
|
||||
/// so either can start first against an empty database.
|
||||
const SCHEMA: &[&str] = &[
|
||||
"CREATE TABLE IF NOT EXISTS stations (
|
||||
id SERIAL PRIMARY KEY,
|
||||
@@ -145,12 +146,12 @@ const SCHEMA: &[&str] = &[
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (as_of, station_code, horizon_hours))",
|
||||
"CREATE INDEX IF NOT EXISTS idx_forecast_history_station ON forecast_history(station_code, as_of)",
|
||||
// src/notify.py
|
||||
// notify.rs
|
||||
"CREATE TABLE IF NOT EXISTS notification_state (
|
||||
key VARCHAR(64) PRIMARY KEY, state VARCHAR(16) NOT NULL,
|
||||
value NUMERIC(8,3), updated_at TIMESTAMP NOT NULL)",
|
||||
// The current /forecast payload, read by the Python API when
|
||||
// FORECAST_IN_COLLECTOR=true (one row, id 'all').
|
||||
// The current /forecast payload (one row, id 'all'), served by
|
||||
// `ping-collector serve` when the website runs as its own process.
|
||||
"CREATE TABLE IF NOT EXISTS forecast_current (
|
||||
id VARCHAR(16) PRIMARY KEY,
|
||||
computed_at TIMESTAMP NOT NULL,
|
||||
@@ -236,6 +237,62 @@ pub async fn save_rid(pool: &PgPool, readings: &[Reading], stations: &Stations)
|
||||
Ok(rows.len())
|
||||
}
|
||||
|
||||
/// Stored RID levels of report columns `nums` in [start, end], for the
|
||||
/// screening (rid::screen).
|
||||
pub async fn rid_readings_between(
|
||||
pool: &PgPool,
|
||||
nums: &[u32],
|
||||
start: NaiveDateTime,
|
||||
end: NaiveDateTime,
|
||||
) -> Result<Vec<crate::rid::StoredReading>> {
|
||||
let rows: Vec<(NaiveDateTime, i32, f64, Option<f64>)> = sqlx::query_as(
|
||||
"SELECT timestamp, station_id, water_level::float8, discharge::float8
|
||||
FROM water_measurements
|
||||
WHERE station_id = ANY($1) AND timestamp >= $2 AND timestamp <= $3
|
||||
AND water_level IS NOT NULL",
|
||||
)
|
||||
.bind(nums.iter().map(|&n| n as i32).collect::<Vec<_>>())
|
||||
.bind(start)
|
||||
.bind(end)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.context("reading stored RID levels")?;
|
||||
Ok(rows
|
||||
.into_iter()
|
||||
.map(|(t, n, l, q)| (t, n as u32, l, q))
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Delete stored RID readings (timestamp, report column).
|
||||
pub async fn delete_rid_readings(pool: &PgPool, keys: &[(NaiveDateTime, u32)]) -> Result<usize> {
|
||||
if keys.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
let r = sqlx::query(
|
||||
"DELETE FROM water_measurements w
|
||||
USING UNNEST($1::timestamp[], $2::int4[]) AS k(ts, sid)
|
||||
WHERE w.timestamp = k.ts AND w.station_id = k.sid",
|
||||
)
|
||||
.bind(keys.iter().map(|(t, _)| *t).collect::<Vec<_>>())
|
||||
.bind(keys.iter().map(|(_, n)| *n as i32).collect::<Vec<_>>())
|
||||
.execute(pool)
|
||||
.await
|
||||
.context("deleting RID spikes")?;
|
||||
Ok(r.rows_affected() as usize)
|
||||
}
|
||||
|
||||
/// Delete the model rows of forecast_history issued at or after `from`, so
|
||||
/// a catch-up recomputes them (after the readings they were computed from
|
||||
/// changed).
|
||||
pub async fn delete_model_forecasts_since(pool: &PgPool, from: NaiveDateTime) -> Result<usize> {
|
||||
let r = sqlx::query("DELETE FROM forecast_history WHERE as_of >= $1 AND source = 'model'")
|
||||
.bind(from)
|
||||
.execute(pool)
|
||||
.await
|
||||
.context("deleting forecast_history rows")?;
|
||||
Ok(r.rows_affected() as usize)
|
||||
}
|
||||
|
||||
/// Day -> hours (0-23) that have at least one RID reading, for [start, end].
|
||||
pub async fn recorded_hours_by_day(
|
||||
pool: &PgPool,
|
||||
@@ -266,6 +323,34 @@ pub async fn earliest_rid_timestamp(pool: &PgPool) -> Result<Option<NaiveDateTim
|
||||
)
|
||||
}
|
||||
|
||||
/// (hour, station) pairs in `[start, end)` where one of `codes` has a RID
|
||||
/// level but no model forecast: hours that arrived together with a later
|
||||
/// one (RID's overnight batch, a late report), so no run forecast from them.
|
||||
pub async fn hours_without_model_forecast(
|
||||
pool: &PgPool,
|
||||
codes: &[String],
|
||||
start: NaiveDateTime,
|
||||
end: NaiveDateTime,
|
||||
) -> Result<Vec<(NaiveDateTime, String)>> {
|
||||
sqlx::query_as(
|
||||
"SELECT DISTINCT date_trunc('hour', m.timestamp) AS h, s.station_code::text
|
||||
FROM water_measurements m JOIN stations s ON s.id = m.station_id
|
||||
WHERE m.timestamp >= $1 AND m.timestamp < $2 AND m.water_level IS NOT NULL
|
||||
AND s.station_code = ANY($3)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM forecast_history f
|
||||
WHERE f.station_code = s.station_code
|
||||
AND f.as_of = date_trunc('hour', m.timestamp) AND f.source = 'model')
|
||||
ORDER BY 1, 2",
|
||||
)
|
||||
.bind(start)
|
||||
.bind(end)
|
||||
.bind(codes)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.context("finding hours without a model forecast")
|
||||
}
|
||||
|
||||
pub async fn latest_rid_timestamp(pool: &PgPool) -> Result<Option<NaiveDateTime>> {
|
||||
Ok(
|
||||
sqlx::query_scalar("SELECT MAX(timestamp) FROM water_measurements")
|
||||
@@ -664,6 +749,11 @@ pub async fn latest_rid_readings(pool: &PgPool) -> Result<Vec<LatestReading>> {
|
||||
}
|
||||
|
||||
/// `ForecastHistoryStore.save_rows`: one row per (as_of, station, horizon).
|
||||
/// Store issued forecasts; a later run's row for the same hour replaces the
|
||||
/// earlier one, except that a heuristic row never replaces a model row. The
|
||||
/// hourly run re-issues the newest reading's hour while RID is silent
|
||||
/// overnight, and after 6 h (`stale_after_h`) it does so as the heuristic:
|
||||
/// without the exception the 21:00 model forecast was overwritten each night.
|
||||
pub async fn save_forecast_history(
|
||||
pool: &PgPool,
|
||||
rows: &[ping_forecast::Forecast],
|
||||
@@ -682,7 +772,8 @@ pub async fn save_forecast_history(
|
||||
ON CONFLICT (as_of, station_code, horizon_hours) DO UPDATE SET
|
||||
predicted_max_level = EXCLUDED.predicted_max_level, p_warning = EXCLUDED.p_warning,
|
||||
p_danger = EXCLUDED.p_danger, current_level = EXCLUDED.current_level,
|
||||
model_version = EXCLUDED.model_version, source = EXCLUDED.source",
|
||||
model_version = EXCLUDED.model_version, source = EXCLUDED.source
|
||||
WHERE forecast_history.source IS DISTINCT FROM 'model' OR EXCLUDED.source = 'model'",
|
||||
)
|
||||
.bind(rows.iter().map(|r| r.as_of).collect::<Vec<_>>())
|
||||
.bind(
|
||||
@@ -727,7 +818,7 @@ pub async fn save_forecast_history(
|
||||
Ok(rows.len())
|
||||
}
|
||||
|
||||
/// The payload the Python API serves as /forecast.
|
||||
/// The payload /forecast serves (see `forecast_current` above).
|
||||
pub async fn save_forecast_current(
|
||||
pool: &PgPool,
|
||||
computed_at: NaiveDateTime,
|
||||
|
||||
@@ -191,6 +191,156 @@ async fn yesterdays_late_hours_are_recovered_after_midnight() {
|
||||
db.drop().await;
|
||||
}
|
||||
|
||||
/// A report source with fixed P.1 levels per hour of one report date.
|
||||
struct ScriptedRid(Mutex<HashMap<NaiveDate, Vec<(i64, f64)>>>);
|
||||
|
||||
impl RidSource for ScriptedRid {
|
||||
async fn fetch_report(&self, date: NaiveDate) -> Result<Vec<Reading>> {
|
||||
let script = self
|
||||
.0
|
||||
.lock()
|
||||
.unwrap()
|
||||
.get(&date)
|
||||
.cloned()
|
||||
.unwrap_or_default();
|
||||
Ok(script
|
||||
.into_iter()
|
||||
.map(|(h, level)| Reading {
|
||||
timestamp: date.and_time(NaiveTime::MIN) + Duration::hours(h),
|
||||
station_num: 8,
|
||||
water_level: level,
|
||||
discharge: Some(45.0),
|
||||
discharge_percent: None,
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
}
|
||||
|
||||
async fn p1_levels(pool: &PgPool) -> Vec<(NaiveDateTime, f64, Option<f64>)> {
|
||||
sqlx::query_as(
|
||||
"SELECT timestamp, water_level::float8, discharge::float8 FROM water_measurements
|
||||
WHERE station_id = 8 ORDER BY timestamp",
|
||||
)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn spikes_are_screened_and_refetch_repairs_the_store() {
|
||||
let Some(db) = TestDb::new().await else {
|
||||
return;
|
||||
};
|
||||
let stations = Stations::bundled();
|
||||
let day = d(2026, 10, 24);
|
||||
// what the store held before screening existed: the slip, level-only
|
||||
db::save_rid(
|
||||
&db.pool,
|
||||
&[
|
||||
Reading {
|
||||
timestamp: t(2026, 10, 24, 23, 0),
|
||||
station_num: 8,
|
||||
water_level: 1.04,
|
||||
discharge: None,
|
||||
discharge_percent: None,
|
||||
},
|
||||
Reading {
|
||||
timestamp: t(2026, 10, 25, 0, 0),
|
||||
station_num: 8,
|
||||
water_level: 10.6,
|
||||
discharge: None,
|
||||
discharge_percent: None,
|
||||
},
|
||||
Reading {
|
||||
timestamp: t(2026, 10, 25, 1, 0),
|
||||
station_num: 8,
|
||||
water_level: 1.07,
|
||||
discharge: Some(45.0),
|
||||
discharge_percent: None,
|
||||
},
|
||||
],
|
||||
&stations,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// RID still serves the slip: refetch deletes it and fills the discharge
|
||||
let src = ScriptedRid(Mutex::new(HashMap::from([(
|
||||
day,
|
||||
vec![(23, 1.04), (24, 10.6)],
|
||||
)])));
|
||||
let s = rid::refetch(&src, &db.pool, &stations, day, day)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!((s.saved, s.spikes, s.deleted, s.changed), (1, 1, 1, 2));
|
||||
assert_eq!(s.first_change, Some(t(2026, 10, 25, 0, 0)));
|
||||
assert_eq!(
|
||||
p1_levels(&db.pool).await,
|
||||
vec![
|
||||
(t(2026, 10, 24, 23, 0), 1.04, Some(45.0)),
|
||||
(t(2026, 10, 25, 1, 0), 1.07, Some(45.0)),
|
||||
]
|
||||
);
|
||||
|
||||
// RID corrected it: the hour comes back with the right value
|
||||
src.0
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert(day, vec![(23, 1.04), (24, 1.06)]);
|
||||
let s = rid::refetch(&src, &db.pool, &stations, day, day)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!((s.saved, s.spikes, s.deleted, s.changed), (2, 0, 0, 1));
|
||||
assert_eq!(s.first_change, None); // a new hour, nothing to redo
|
||||
assert_eq!(p1_levels(&db.pool).await.len(), 3);
|
||||
// RID revises a level: the forecasts from that hour on need a redo
|
||||
src.0
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert(day, vec![(23, 1.05), (24, 1.06)]);
|
||||
let s = rid::refetch(&src, &db.pool, &stations, day, day)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(s.first_change, Some(t(2026, 10, 24, 23, 0)));
|
||||
|
||||
// a live cycle: the newest hour jumps and waits for the next one
|
||||
let today = d(2026, 10, 25);
|
||||
src.0
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert(today, vec![(1, 1.07), (2, 9.9)]);
|
||||
// (yesterday's report comes along: it still has holes)
|
||||
let o = rid::run_cycle(&src, &db.pool, &stations, t(2026, 10, 25, 2, 10)).await;
|
||||
assert_eq!((o.fetched, o.saved), (4, 3));
|
||||
assert_eq!(
|
||||
p1_levels(&db.pool).await.last().unwrap().0,
|
||||
t(2026, 10, 25, 1, 0)
|
||||
);
|
||||
// 03:00 is back at 1.1: 02:00 was a spike and never stored
|
||||
src.0
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert(today, vec![(1, 1.07), (2, 9.9), (3, 1.1)]);
|
||||
let o = rid::run_cycle(&src, &db.pool, &stations, t(2026, 10, 25, 3, 10)).await;
|
||||
assert_eq!((o.fetched, o.saved), (5, 4));
|
||||
let levels: Vec<f64> = p1_levels(&db.pool).await.iter().map(|r| r.1).collect();
|
||||
assert_eq!(levels, vec![1.05, 1.06, 1.07, 1.1]);
|
||||
|
||||
// the forecasts issued from the changed hours can be redone
|
||||
let mut f = forecast_row("P.1", 24, 0.9);
|
||||
f.as_of = t(2026, 10, 25, 0, 0);
|
||||
let mut g = forecast_row("P.1", 24, 0.0);
|
||||
g.as_of = t(2026, 10, 24, 23, 0);
|
||||
db::save_forecast_history(&db.pool, &[f, g]).await.unwrap();
|
||||
assert_eq!(
|
||||
db::delete_model_forecasts_since(&db.pool, t(2026, 10, 25, 0, 0))
|
||||
.await
|
||||
.unwrap(),
|
||||
1
|
||||
);
|
||||
db.drop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rid_upsert_revises_values_and_rounds_like_numeric() {
|
||||
let Some(db) = TestDb::new().await else {
|
||||
@@ -404,6 +554,109 @@ async fn forecast_outputs_round_trip() {
|
||||
db.drop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heuristic_never_replaces_a_model_forecast() {
|
||||
let Some(db) = TestDb::new().await else {
|
||||
return;
|
||||
};
|
||||
// overnight the run re-issues the 21:00 reading, after 6 h as heuristic
|
||||
let model = forecast_row("P.1", 24, 0.2);
|
||||
let heuristic = ping_forecast::Forecast {
|
||||
p_warning: 0.0,
|
||||
predicted_max_level: 1.23,
|
||||
model_version: "heuristic-v1".into(),
|
||||
source: "heuristic".into(),
|
||||
..forecast_row("P.1", 24, 0.0)
|
||||
};
|
||||
let stored = async || -> (String, f64) {
|
||||
sqlx::query_as("SELECT source::text, predicted_max_level::float8 FROM forecast_history")
|
||||
.fetch_one(&db.pool)
|
||||
.await
|
||||
.unwrap()
|
||||
};
|
||||
db::save_forecast_history(&db.pool, std::slice::from_ref(&model))
|
||||
.await
|
||||
.unwrap();
|
||||
db::save_forecast_history(&db.pool, std::slice::from_ref(&heuristic))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(stored().await, ("model".into(), 3.123));
|
||||
// a model row does replace a heuristic one (and a newer model row)
|
||||
sqlx::query("DELETE FROM forecast_history")
|
||||
.execute(&db.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
db::save_forecast_history(&db.pool, &[heuristic])
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(stored().await, ("heuristic".into(), 1.23));
|
||||
db::save_forecast_history(&db.pool, &[model]).await.unwrap();
|
||||
assert_eq!(stored().await, ("model".into(), 3.123));
|
||||
db.drop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn hours_without_a_model_forecast() {
|
||||
let Some(db) = TestDb::new().await else {
|
||||
return;
|
||||
};
|
||||
let stations = Stations::bundled();
|
||||
let r = |h: u32, num: u32| Reading {
|
||||
timestamp: t(2026, 10, 18, h, 0),
|
||||
station_num: num, // 8 = P.1, 1 = P.20
|
||||
water_level: 1.1,
|
||||
discharge: None,
|
||||
discharge_percent: None,
|
||||
};
|
||||
// RID's morning batch: 11..13 arrive at once; only 13:00 (the newest)
|
||||
// and P.20's 11:00 were forecast, and 12:00 only by the heuristic
|
||||
db::save_rid(
|
||||
&db.pool,
|
||||
&[r(11, 8), r(12, 8), r(13, 8), r(11, 1), r(12, 1), r(13, 1)],
|
||||
&stations,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let issued = |code: &str, h: u32, source: &str| ping_forecast::Forecast {
|
||||
as_of: t(2026, 10, 18, h, 0),
|
||||
source: source.into(),
|
||||
..forecast_row(code, 24, 0.1)
|
||||
};
|
||||
db::save_forecast_history(
|
||||
&db.pool,
|
||||
&[
|
||||
issued("P.1", 13, "model"),
|
||||
issued("P.20", 13, "model"),
|
||||
issued("P.20", 11, "model"),
|
||||
issued("P.1", 12, "heuristic"),
|
||||
],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let codes = vec!["P.1".to_string(), "P.20".to_string()];
|
||||
let missing = db::hours_without_model_forecast(
|
||||
&db.pool,
|
||||
&codes,
|
||||
t(2026, 10, 18, 0, 0),
|
||||
t(2026, 10, 18, 14, 0),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let at = |h: u32, code: &str| (t(2026, 10, 18, h, 0), code.to_string());
|
||||
assert_eq!(missing, vec![at(11, "P.1"), at(12, "P.1"), at(12, "P.20")]);
|
||||
// stations without a model are not asked about
|
||||
let only_p20 = db::hours_without_model_forecast(
|
||||
&db.pool,
|
||||
&codes[1..],
|
||||
t(2026, 10, 18, 0, 0),
|
||||
t(2026, 10, 18, 14, 0),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(only_p20, vec![at(12, "P.20")]);
|
||||
db.drop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn notification_state_round_trip() {
|
||||
let Some(db) = TestDb::new().await else {
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
//! `.env` loading with python-dotenv 1.1.1 semantics (`load_dotenv()`:
|
||||
//! `.env` loading with python-dotenv semantics (1.1.1 and 1.2.2 parse the
|
||||
//! fixture identically; `load_dotenv()`:
|
||||
//! interpolate `${VAR}` / `${VAR:-default}`, never override variables that
|
||||
//! are already set).
|
||||
//!
|
||||
//! The Python app reads the same file, so the two must agree on every value.
|
||||
//! The Python tools (src/config.py) read the same file, so the two must agree
|
||||
//! on every value.
|
||||
//! dotenvy (the usual crate) is stricter: a stray `'` or `"` inside an
|
||||
//! unquoted value makes it swallow the rest of the file and abort, and its
|
||||
//! error message echoes the line, password included. Here an unparsable
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
//! Forecasting and notifications in the collector (FORECAST_IN_COLLECTOR=true),
|
||||
//! replacing the Python API's forecast_notify_task.
|
||||
//! Forecasting and notifications in the collector (FORECAST_IN_COLLECTOR=true);
|
||||
//! the port of the retired Python app's forecast_notify_task.
|
||||
//!
|
||||
//! After each collection cycle that brought newer RID or HII readings, and at
|
||||
//! least once per clock hour: load the last 336 h (RID plus the HII mirror
|
||||
//! fill), fetch the rain forecast, run every exported model, store the result
|
||||
//! (`forecast_history`, and `forecast_current` which the API serves as
|
||||
//! /forecast) and push ntfy transitions.
|
||||
//! /forecast) and push ntfy transitions. Then forecast the hours whose
|
||||
//! readings came late (`catch_up`).
|
||||
|
||||
use crate::config::Config;
|
||||
use crate::db;
|
||||
@@ -20,6 +21,7 @@ use ping_forecast::hii::{self, FillInfo};
|
||||
use ping_forecast::rain::RainSeries;
|
||||
use ping_forecast::{Forecast, ModelSet, ModelStore};
|
||||
use sqlx::PgPool;
|
||||
use std::collections::{BTreeMap, BTreeSet, HashSet};
|
||||
|
||||
/// Everything one forecast run used and produced.
|
||||
pub struct Snapshot {
|
||||
@@ -93,6 +95,87 @@ pub async fn compute(
|
||||
})
|
||||
}
|
||||
|
||||
/// How far back every run looks for hours no run forecast from; the first
|
||||
/// run after a start looks two weeks back (the nights before an upgrade).
|
||||
pub const CATCH_UP_HOURS: i64 = 48;
|
||||
pub const CATCH_UP_START_HOURS: i64 = 14 * 24;
|
||||
|
||||
/// What a catch-up did.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct CatchUp {
|
||||
/// hours computed (also those that produced no model row)
|
||||
pub hours: Vec<NaiveDateTime>,
|
||||
/// forecast_history rows stored
|
||||
pub rows: usize,
|
||||
}
|
||||
|
||||
/// Model forecasts for the hours whose readings arrived together with a
|
||||
/// later hour's. RID publishes 22:00..06:00 in one batch in the morning, so
|
||||
/// the hourly run only ever forecast from 06:00 and the night was missing
|
||||
/// from the issued history (also single hours RID reported late).
|
||||
///
|
||||
/// Each hour is computed as a run at that hour would have been: RID, the
|
||||
/// HII fill and the dam state up to that hour only. Rain is one fetch now,
|
||||
/// so a caught-up hour's `rain_fc24` is partly Open-Meteo's estimate of rain
|
||||
/// that has since fallen rather than a forecast (the training condition, not
|
||||
/// what a live run had). Only missing (hour, station) pairs are written, and
|
||||
/// a stored model row is never replaced. Hours in `skip` were tried before.
|
||||
pub async fn catch_up(
|
||||
pool: &PgPool,
|
||||
rain_client: &OpenMeteoClient,
|
||||
models: &ModelSet,
|
||||
now: NaiveDateTime,
|
||||
hours_back: i64,
|
||||
skip: &HashSet<NaiveDateTime>,
|
||||
) -> Result<CatchUp> {
|
||||
let codes: Vec<String> = models.keys().cloned().collect();
|
||||
let start = floor_hour(now) - Duration::hours(hours_back);
|
||||
let mut missing: BTreeMap<NaiveDateTime, BTreeSet<String>> = BTreeMap::new();
|
||||
for (hour, code) in db::hours_without_model_forecast(pool, &codes, start, now).await? {
|
||||
if !skip.contains(&hour) {
|
||||
missing.entry(hour).or_default().insert(code);
|
||||
}
|
||||
}
|
||||
if missing.is_empty() {
|
||||
return Ok(CatchUp::default());
|
||||
}
|
||||
// the rain windows (up to 72 h) of the oldest hour, plus a margin
|
||||
let past_days = (hours_back / 24 + 5).min(92) as u32;
|
||||
let rain: RainSeries = rain_client
|
||||
.fetch_forecast(past_days, 2)
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|r| (r.timestamp, r.catchment_mean.unwrap_or(f64::NAN)))
|
||||
.collect();
|
||||
let mut done = CatchUp::default();
|
||||
for (hour, stations) in missing {
|
||||
let snap = compute(pool, rain_client, models, hour, Some(rain.clone())).await?;
|
||||
let rows: Vec<Forecast> = snap
|
||||
.forecasts
|
||||
.into_iter()
|
||||
.filter(|f| {
|
||||
f.as_of == hour && f.source == "model" && stations.contains(&f.station_code)
|
||||
})
|
||||
.collect();
|
||||
done.rows += db::save_forecast_history(pool, &rows).await?;
|
||||
done.hours.push(hour);
|
||||
}
|
||||
Ok(done)
|
||||
}
|
||||
|
||||
pub fn describe_catch_up(c: &CatchUp) -> String {
|
||||
match (c.hours.first(), c.hours.last()) {
|
||||
(Some(first), Some(last)) => format!(
|
||||
"{} hours from {} to {}, {} rows stored",
|
||||
c.hours.len(),
|
||||
first.format("%d %H:%M"),
|
||||
last.format("%d %H:%M"),
|
||||
c.rows
|
||||
),
|
||||
_ => "nothing to catch up".into(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn summarize(s: &Snapshot) -> String {
|
||||
let model = s.forecasts.iter().filter(|f| f.source == "model").count();
|
||||
let p1 = s
|
||||
@@ -151,6 +234,7 @@ pub fn notifier(cfg: &Config, http: reqwest::Client) -> Option<Notifier> {
|
||||
settings: Settings {
|
||||
prefix: cfg.ntfy_topic_prefix.clone(),
|
||||
rid_stale_alert_hours: cfg.rid_stale_alert_hours,
|
||||
feed_night_until_hour: cfg.feed_night_until_hour,
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -170,6 +254,10 @@ pub struct Forecaster {
|
||||
notifier: Option<Notifier>,
|
||||
last_inputs: Option<(Option<NaiveDateTime>, Option<NaiveDateTime>)>,
|
||||
last_slot: Option<NaiveDateTime>,
|
||||
/// hours `catch_up` computed already (a station whose features cannot
|
||||
/// be built at that hour would otherwise be retried on every run)
|
||||
caught_up: HashSet<NaiveDateTime>,
|
||||
caught_up_since_start: bool,
|
||||
}
|
||||
|
||||
impl Forecaster {
|
||||
@@ -201,6 +289,8 @@ impl Forecaster {
|
||||
notifier,
|
||||
last_inputs: None,
|
||||
last_slot: None,
|
||||
caught_up: HashSet::new(),
|
||||
caught_up_since_start: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -271,6 +361,24 @@ impl Forecaster {
|
||||
Err(e) => tracing::warn!("notify cycle failed: {e:#}"),
|
||||
}
|
||||
}
|
||||
// after the notifications: they are about now, this is history
|
||||
let window = if self.caught_up_since_start {
|
||||
CATCH_UP_HOURS
|
||||
} else {
|
||||
CATCH_UP_START_HOURS
|
||||
};
|
||||
match catch_up(pool, rain, &models, now, window, &self.caught_up).await {
|
||||
Ok(c) => {
|
||||
if !c.hours.is_empty() {
|
||||
tracing::info!("forecast catch-up: {}", describe_catch_up(&c));
|
||||
}
|
||||
self.caught_up.extend(c.hours);
|
||||
self.caught_up_since_start = true;
|
||||
let oldest = floor_hour(now) - Duration::hours(CATCH_UP_HOURS);
|
||||
self.caught_up.retain(|h| *h >= oldest);
|
||||
}
|
||||
Err(e) => tracing::warn!("forecast catch-up failed: {e:#}"),
|
||||
}
|
||||
Ok(RunInfo {
|
||||
at: now,
|
||||
rows: snap.forecasts.len(),
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
//! ping-collector: the Ping River Monitor's data collection service.
|
||||
//! ping-collector: the Ping River Monitor service.
|
||||
//!
|
||||
//! Replaces the collection half of the Python web app (RID hourly gauges,
|
||||
//! HII/ThaiWater rain + water level, RID dams, Open-Meteo rain) and, with
|
||||
//! FORECAST_IN_COLLECTOR=true, the flood forecast (rust/forecast) and the
|
||||
//! ntfy/Matrix notifications, and with WEB_LISTEN the website and read API
|
||||
//! (the port of the FastAPI app). Writes the same PostgreSQL tables as the
|
||||
//! Python app did. See docs/RUST_REWRITE.md.
|
||||
//! Collects RID hourly gauges, HII/ThaiWater rain + water level, RID dams
|
||||
//! and Open-Meteo rain into PostgreSQL; with FORECAST_IN_COLLECTOR=true runs
|
||||
//! the flood forecast (rust/forecast, models trained in Python) and the
|
||||
//! ntfy/Matrix notifications; with WEB_LISTEN serves the website and API.
|
||||
//! It replaced the Python app in 2026-10 (same tables, routes and messages);
|
||||
//! see docs/RUST_REWRITE.md.
|
||||
|
||||
mod config;
|
||||
mod dams;
|
||||
@@ -62,6 +62,14 @@ enum Command {
|
||||
},
|
||||
/// Re-fetch RID reports that have holes: a number of days back, or "all"
|
||||
FillGaps { days: String },
|
||||
/// Re-fetch the RID reports of FROM..=TO (YYYY-MM-DD; day D's report holds
|
||||
/// D 01:00 to D+1 00:00) even without holes, through the spike screening:
|
||||
/// corrects readings RID has revised and removes stored spikes
|
||||
Refetch {
|
||||
from: chrono::NaiveDate,
|
||||
/// Last report date [default: FROM]
|
||||
to: Option<chrono::NaiveDate>,
|
||||
},
|
||||
/// Connect, create missing tables, print row counts and the newest RID reading
|
||||
CheckDb,
|
||||
/// Forecast from the database now and print it; stores nothing without --save
|
||||
@@ -80,6 +88,19 @@ enum Command {
|
||||
#[arg(long)]
|
||||
rain_file: Option<PathBuf>,
|
||||
},
|
||||
/// Forecast the hours no run forecast from (readings that arrived with a
|
||||
/// later hour's, e.g. RID's overnight batch) as a run at each hour would
|
||||
/// have, and store them; every run does this for the last 48 h
|
||||
CatchUpForecasts {
|
||||
/// How many hours back to look
|
||||
#[arg(long, default_value_t = 336)]
|
||||
hours: i64,
|
||||
/// First delete the model forecasts issued from this local time on
|
||||
/// (e.g. 2026-10-25T00:00) so they are recomputed: after `refetch`
|
||||
/// changed the readings they came from
|
||||
#[arg(long)]
|
||||
redo_from: Option<String>,
|
||||
},
|
||||
/// Show the readings the notifier judges and the ntfy transitions due now
|
||||
Notify {
|
||||
/// Show what would be published; publish nothing, change no state
|
||||
@@ -204,6 +225,43 @@ async fn run(cli: Cli) -> Result<()> {
|
||||
let rain = rain_file.as_deref().map(read_rain_file).transpose()?;
|
||||
collector.forecast(json, save, now, rain).await
|
||||
}
|
||||
Command::CatchUpForecasts { hours, redo_from } => {
|
||||
let redo_from = redo_from.as_deref().map(parse_local).transpose()?;
|
||||
collector.catch_up_forecasts(hours, redo_from).await
|
||||
}
|
||||
Command::Refetch { from, to } => {
|
||||
let to = to.unwrap_or(from);
|
||||
if to < from {
|
||||
bail!("refetch: TO ({to}) is before FROM ({from})");
|
||||
}
|
||||
db::ensure_schema(&collector.pool).await?;
|
||||
let s = rid::refetch(
|
||||
&collector.rid,
|
||||
&collector.pool,
|
||||
&collector.stations(),
|
||||
from,
|
||||
to,
|
||||
)
|
||||
.await?;
|
||||
println!(
|
||||
"{} readings saved ({} new or changed), {} spikes rejected, {} stored spikes deleted, {} held for want of the next reading",
|
||||
s.saved,
|
||||
s.changed - s.deleted,
|
||||
s.spikes,
|
||||
s.deleted,
|
||||
s.held
|
||||
);
|
||||
if let Some(t) = s.first_change {
|
||||
let hours = (util::bangkok_now() - t).num_hours() + 2;
|
||||
if hours <= 24 * 60 {
|
||||
println!(
|
||||
"stored levels changed from {t} on: the forecasts issued since came from wrong levels; recompute them with\n ping-collector catch-up-forecasts --hours {hours} --redo-from {}",
|
||||
t.format("%Y-%m-%dT%H:%M")
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Command::Notify { dry_run } => collector.notify(dry_run).await,
|
||||
Command::CheckDb => {
|
||||
db::ensure_schema(&collector.pool).await?;
|
||||
|
||||
+133
-23
@@ -1,21 +1,22 @@
|
||||
//! Public flood notifications over ntfy, and the same transitions to a
|
||||
//! Matrix room: the port of src/notify.py, run by the collector when
|
||||
//! FORECAST_IN_COLLECTOR=true. (Matrix replaces the old `run.py
|
||||
//! --alert-check` cron job, which repeated an alert on every run while a
|
||||
//! level stayed high.)
|
||||
//! Matrix room, run by the collector when FORECAST_IN_COLLECTOR=true. A port
|
||||
//! of the retired Python notifier (src/notify.py, removed after the 2026-10
|
||||
//! cutover); its settings still come from src/ml/export.py via
|
||||
//! flood_config.json. (Matrix replaced the old `run.py --alert-check` cron
|
||||
//! job, which repeated an alert on every run while a level stayed high.)
|
||||
//!
|
||||
//! Same topics, texts, hysteresis, capacity guard, outlook and persisted
|
||||
//! state keys (`notification_state`) as Python, so a cutover neither repeats
|
||||
//! nor drops a message. One change: for stations whose HII/ThaiWater twin is
|
||||
//! the same sensor, a reading newer than RID's (RID's report often lags 2-10 h
|
||||
//! at night) is used for the level alerts and the feed check. The feed is
|
||||
//! "stale" only when neither source has a reading for 3 h; RID lagging alone
|
||||
//! is reported separately after RID_STALE_ALERT_HOURS.
|
||||
//! state keys (`notification_state`) as Python had, so the cutover neither
|
||||
//! repeated nor dropped a message. Changes since: for stations whose HII/ThaiWater
|
||||
//! twin is the same sensor, a reading newer than RID's is used for the level
|
||||
//! alerts and the feed check; RID lagging alone is reported after
|
||||
//! RID_STALE_ALERT_HOURS; and RID's nightly pause is not reported (see
|
||||
//! [`NIGHT_FROM_HOUR`]).
|
||||
|
||||
use crate::config::MatrixConfig;
|
||||
use crate::db::{self, LatestReading};
|
||||
use anyhow::Result;
|
||||
use chrono::{Duration, NaiveDateTime};
|
||||
use chrono::{Duration, NaiveDateTime, Timelike};
|
||||
use ping_forecast::Forecast;
|
||||
use ping_forecast::config::config;
|
||||
use ping_forecast::hii::FillInfo;
|
||||
@@ -122,10 +123,28 @@ fn level_state(level: f64, warn: f64, danger: f64, prev: &str) -> &'static str {
|
||||
"clear"
|
||||
}
|
||||
|
||||
/// RID publishes no hourly report overnight: on 2026-10-17, -18 and -19 the
|
||||
/// 21:00 reading was the last until the 22:00..06:00 hours arrived together
|
||||
/// between 06:35 and 07:22 (ThaiWater's copies of the RID gauges arrive with
|
||||
/// the same delay, so they cannot bridge it). Between this hour and
|
||||
/// `Settings::feed_night_until_hour` a quiet river's feed is therefore not
|
||||
/// reported stale; a feed still stale at that hour is. While any station is at
|
||||
/// warning or danger, or the P.1 outlook is on, the 3 h rule holds all night.
|
||||
pub const NIGHT_FROM_HOUR: u32 = 21;
|
||||
|
||||
pub struct Settings {
|
||||
pub prefix: String,
|
||||
/// 0 = off
|
||||
pub rid_stale_alert_hours: f64,
|
||||
/// End of the night window (hour of day); 0 = no night window.
|
||||
pub feed_night_until_hour: u32,
|
||||
}
|
||||
|
||||
impl Settings {
|
||||
fn in_night_window(&self, now: NaiveDateTime) -> bool {
|
||||
let h = now.hour();
|
||||
self.feed_night_until_hour > 0 && (h >= NIGHT_FROM_HOUR || h < self.feed_night_until_hour)
|
||||
}
|
||||
}
|
||||
|
||||
impl Settings {
|
||||
@@ -175,6 +194,8 @@ pub fn evaluate(
|
||||
let (mut b_danger, mut b_warning, mut b_clear) = (vec![], vec![], vec![]);
|
||||
let mut latest: Option<NaiveDateTime> = None;
|
||||
let mut latest_rid: Option<NaiveDateTime> = None;
|
||||
// any station at warning/danger or the outlook on: the night window is off
|
||||
let mut river_high = false;
|
||||
|
||||
for r in readings {
|
||||
latest_rid = latest_rid.max(r.rid_timestamp);
|
||||
@@ -200,6 +221,7 @@ pub fn evaluate(
|
||||
);
|
||||
continue;
|
||||
}
|
||||
river_high |= cur != "clear";
|
||||
if cur == prev {
|
||||
continue;
|
||||
}
|
||||
@@ -301,6 +323,7 @@ pub fn evaluate(
|
||||
}
|
||||
|
||||
// model outlook for the city gauge (opt-in topic, experimental)
|
||||
river_high |= state("outlook:P.1") == Some("on");
|
||||
if let Some(p1) = forecasts.iter().find(|f| {
|
||||
f.station_code == "P.1" && f.horizon_hours == nc.outlook_horizon && f.source == "model"
|
||||
}) {
|
||||
@@ -312,6 +335,7 @@ pub fn evaluate(
|
||||
} else {
|
||||
"off"
|
||||
};
|
||||
river_high |= cur == "on";
|
||||
if cur != prev {
|
||||
let (warn, _) = cfg.thresholds("P.1");
|
||||
let notification = if cur == "on" {
|
||||
@@ -347,13 +371,22 @@ pub fn evaluate(
|
||||
}
|
||||
}
|
||||
|
||||
// feed health: stale only when no source has a recent reading
|
||||
// feed health: stale only when no source has a recent reading; during
|
||||
// RID's nightly pause a quiet river keeps its previous state (a recovery
|
||||
// is still announced, a new stale message waits for the morning)
|
||||
let mut feed_stale = false;
|
||||
if let Some(latest) = latest {
|
||||
let age_h = hours_between(now, latest);
|
||||
let prev = state("feed").unwrap_or("ok");
|
||||
feed_stale = age_h >= nc.feed_stale_after_h;
|
||||
let cur = if feed_stale { "stale" } else { "ok" };
|
||||
let deferred = !river_high && settings.in_night_window(now);
|
||||
let cur = if age_h < nc.feed_stale_after_h {
|
||||
"ok"
|
||||
} else if deferred {
|
||||
prev
|
||||
} else {
|
||||
"stale"
|
||||
};
|
||||
feed_stale = cur == "stale";
|
||||
if cur != prev {
|
||||
let notification = if feed_stale {
|
||||
n(
|
||||
@@ -742,6 +775,7 @@ mod tests {
|
||||
Settings {
|
||||
prefix: "ping".into(),
|
||||
rid_stale_alert_hours: 18.0,
|
||||
feed_night_until_hour: 9,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -995,6 +1029,81 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
fn at(day: u32, hour: u32, min: u32) -> NaiveDateTime {
|
||||
NaiveDate::from_ymd_opt(2026, 10, day)
|
||||
.unwrap()
|
||||
.and_hms_opt(hour, min, 0)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn reading_at(code: &str, level: f64, t: NaiveDateTime) -> Reading {
|
||||
Reading {
|
||||
timestamp: t,
|
||||
rid_timestamp: Some(t),
|
||||
..reading(code, level, 0)
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rids_nightly_pause_is_not_reported_on_a_quiet_river() {
|
||||
// 2026-10-19: last reading 21:00, nothing until the 06:00 batch at 06:55
|
||||
let mut st = HashMap::new();
|
||||
let evening = [reading_at("P.1", 1.26, at(18, 21, 0))];
|
||||
for t in [at(18, 23, 5), at(19, 0, 5), at(19, 3, 5), at(19, 6, 5)] {
|
||||
assert!(step(&evening, &[], &mut st, t).is_empty(), "at {t}");
|
||||
}
|
||||
let morning = [reading_at("P.1", 1.33, at(19, 6, 0))];
|
||||
assert!(step(&morning, &[], &mut st, at(19, 6, 55)).is_empty());
|
||||
// still nothing new at 09:00: reported then
|
||||
let sent = step(&evening, &[], &mut st, at(19, 9, 5));
|
||||
assert_eq!(
|
||||
titles(&sent),
|
||||
[("ping-status", "Ping River monitor: gauge feed stale")]
|
||||
);
|
||||
assert!(
|
||||
sent[0]
|
||||
.message
|
||||
.starts_with("No new readings for 12 h (last 18 Oct 21:00)")
|
||||
);
|
||||
// the window can be turned off
|
||||
let off = Settings {
|
||||
feed_night_until_hour: 0,
|
||||
..settings()
|
||||
};
|
||||
let plan = evaluate(&evening, &[], &HashMap::new(), &off, at(19, 0, 5));
|
||||
assert_eq!(
|
||||
plan[0].notification.title,
|
||||
"Ping River monitor: gauge feed stale"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn night_window_keeps_a_stale_feed_stale_and_announces_recovery() {
|
||||
let mut st = HashMap::new();
|
||||
let old = [reading_at("P.1", 1.2, at(18, 14, 0))];
|
||||
assert_eq!(step(&old, &[], &mut st, at(18, 18, 5)).len(), 1); // stale, daytime
|
||||
assert!(step(&old, &[], &mut st, at(18, 22, 5)).is_empty()); // not "recovered"
|
||||
let back = [reading_at("P.1", 1.2, at(18, 22, 0))];
|
||||
assert_eq!(
|
||||
step(&back, &[], &mut st, at(18, 23, 5))[0].title,
|
||||
"Ping River monitor: feed recovered"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_three_hour_rule_holds_at_night_while_the_river_is_high() {
|
||||
let mut st = HashMap::from([("level:P.1".to_string(), "warning".to_string())]);
|
||||
let high = [reading_at("P.1", 3.8, at(18, 21, 0))];
|
||||
assert_eq!(
|
||||
titles(&step(&high, &[], &mut st, at(19, 0, 5))),
|
||||
[("ping-status", "Ping River monitor: gauge feed stale")]
|
||||
);
|
||||
// the outlook being on counts too
|
||||
let mut st = HashMap::from([("outlook:P.1".to_string(), "on".to_string())]);
|
||||
let quiet = [reading_at("P.1", 1.3, at(18, 21, 0))];
|
||||
assert_eq!(step(&quiet, &[], &mut st, at(19, 0, 5)).len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn undelivered_transitions_are_retried() {
|
||||
let mut st = HashMap::new();
|
||||
@@ -1011,8 +1120,10 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// 652 hourly steps from src/notify.py itself (testdata/gen_notify_cases.py):
|
||||
/// the same notifications, word for word, in the same order.
|
||||
/// 652 hourly steps recorded from the retired Python notifier (generator:
|
||||
/// testdata/gen_notify_cases.py, in git history before the cleanup): the
|
||||
/// same notifications, word for word, in the same order. A deliberate
|
||||
/// text change updates the fixture with the new expected messages.
|
||||
#[test]
|
||||
fn matches_python_notify_on_generated_sequences() {
|
||||
use std::io::Read;
|
||||
@@ -1064,13 +1175,12 @@ mod tests {
|
||||
.map(|v| v.as_str().unwrap())
|
||||
.collect();
|
||||
let mut delivered = vec![];
|
||||
for p in evaluate(
|
||||
&readings,
|
||||
&forecasts,
|
||||
&states,
|
||||
&settings(),
|
||||
ts(&step["now"]),
|
||||
) {
|
||||
// the Python notifier had no night window
|
||||
let python = Settings {
|
||||
feed_night_until_hour: 0,
|
||||
..settings()
|
||||
};
|
||||
for p in evaluate(&readings, &forecasts, &states, &python, ts(&step["now"])) {
|
||||
if down.contains(&p.notification.topic.as_str()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
+401
-8
@@ -9,6 +9,12 @@
|
||||
//! every reading it gets (not only "the current hour"), and re-fetches
|
||||
//! yesterday's report while the store still has holes in it: hours 22-24 of a
|
||||
//! day are reachable only through that day's report.
|
||||
//!
|
||||
//! Before storing, readings are screened for single-hour spikes (see
|
||||
//! [`judge`]): RID's report sometimes carries a slipped decimal point, a
|
||||
//! flipped sign or a placeholder for one hour (P.1 10.6 m between 1.04 and
|
||||
//! 1.07 on 2026-10-25 00:00), which would raise a false alarm and
|
||||
//! distort the forecast features for three days.
|
||||
|
||||
use crate::db;
|
||||
use crate::stations::Stations;
|
||||
@@ -17,7 +23,7 @@ use anyhow::{Context, Result};
|
||||
use chrono::{Datelike, Duration, NaiveDate, NaiveDateTime, NaiveTime};
|
||||
use serde_json::Value;
|
||||
use sqlx::PgPool;
|
||||
use std::collections::{BTreeSet, HashMap, HashSet};
|
||||
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
|
||||
use std::future::Future;
|
||||
|
||||
pub const N_STATIONS: u32 = 16;
|
||||
@@ -96,6 +102,248 @@ pub fn parse_report(report_date: NaiveDate, payload: &Value) -> Vec<Reading> {
|
||||
out
|
||||
}
|
||||
|
||||
// ---- Screening -------------------------------------------------------------
|
||||
|
||||
/// Neighbours of a reading are looked for this many hours before and after it.
|
||||
const SCREEN_WINDOW_H: i64 = 3;
|
||||
|
||||
/// Gauges on flashy tributaries: genuine 1-h changes of 1.3-3.1 m are on
|
||||
/// record (P.76 0.09 -> 3.18 m on 2026-05-29). Elsewhere the largest genuine
|
||||
/// 1-h change since 2018 is 1.1 m (P.1: 0.56 m).
|
||||
const FLASHY: &[&str] = &["P.76", "P.77", "P.81", "P.82", "P.84", "P.85", "P.87"];
|
||||
|
||||
/// A change from the previous reading larger than this (m) needs the next
|
||||
/// reading to tell a spike from a real change.
|
||||
pub fn jump_limit(code: &str) -> f64 {
|
||||
if FLASHY.contains(&code) { 2.0 } else { 1.0 }
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Verdict {
|
||||
Keep,
|
||||
/// The newest reading jumped more than twice the limit: wait for the next
|
||||
/// one (an hour at most while RID is current) before storing it.
|
||||
Hold,
|
||||
/// Out and back within an hour or two: not stored.
|
||||
Spike,
|
||||
}
|
||||
|
||||
/// `prev`: the last accepted level within [`SCREEN_WINDOW_H`] before the
|
||||
/// reading, `next`: the following reading within the window, if any.
|
||||
///
|
||||
/// A spike leaves the previous level by more than `limit` and the next
|
||||
/// reading comes back, closer to the previous level than half of either
|
||||
/// jump. A rising or falling limb (prev < level < next) or a step the next
|
||||
/// reading confirms is kept. 2018-2026 history: 16 spikes (sign flips,
|
||||
/// slipped decimal points, -5/-6/-9 m placeholders) and no genuine change
|
||||
/// would have been rejected.
|
||||
pub fn judge(limit: f64, prev: Option<f64>, level: f64, next: Option<f64>) -> Verdict {
|
||||
let Some(p) = prev else {
|
||||
return Verdict::Keep;
|
||||
};
|
||||
let out = (level - p).abs();
|
||||
if out <= limit {
|
||||
return Verdict::Keep;
|
||||
}
|
||||
let Some(n) = next else {
|
||||
return if out > 2.0 * limit {
|
||||
Verdict::Hold
|
||||
} else {
|
||||
Verdict::Keep
|
||||
};
|
||||
};
|
||||
let back = (level - n).abs();
|
||||
if back > limit && (n - p).abs() < out.min(back) / 2.0 {
|
||||
Verdict::Spike
|
||||
} else {
|
||||
Verdict::Keep
|
||||
}
|
||||
}
|
||||
|
||||
/// A reading already in the database: (timestamp, report column, level, discharge).
|
||||
pub type StoredReading = (NaiveDateTime, u32, f64, Option<f64>);
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct Screened {
|
||||
pub keep: Vec<Reading>,
|
||||
pub held: Vec<Reading>,
|
||||
pub spikes: Vec<Reading>,
|
||||
/// Stored rows at a spike's hour that are spikes themselves: delete.
|
||||
pub delete: Vec<(NaiveDateTime, u32)>,
|
||||
}
|
||||
|
||||
/// Screen a batch against itself and the stored readings around it. A batch
|
||||
/// reading replaces the stored one at the same hour; stored readings count
|
||||
/// as accepted. `code` maps a report column to its station code.
|
||||
pub fn screen(
|
||||
readings: &[Reading],
|
||||
stored: &[StoredReading],
|
||||
code: impl Fn(u32) -> String,
|
||||
) -> Screened {
|
||||
#[derive(Clone, Copy)]
|
||||
enum Src {
|
||||
Batch(usize),
|
||||
Stored,
|
||||
}
|
||||
let mut out = Screened::default();
|
||||
let mut by_station: BTreeMap<u32, BTreeMap<NaiveDateTime, (f64, Src)>> = BTreeMap::new();
|
||||
for (t, n, level, _) in stored {
|
||||
by_station
|
||||
.entry(*n)
|
||||
.or_default()
|
||||
.insert(*t, (*level, Src::Stored));
|
||||
}
|
||||
let stored_at: HashMap<(NaiveDateTime, u32), f64> =
|
||||
stored.iter().map(|(t, n, l, _)| ((*t, *n), *l)).collect();
|
||||
// the last copy of an hour wins, as in the upsert
|
||||
for (i, r) in readings.iter().enumerate() {
|
||||
by_station
|
||||
.entry(r.station_num)
|
||||
.or_default()
|
||||
.insert(r.timestamp, (r.water_level, Src::Batch(i)));
|
||||
}
|
||||
let window = Duration::hours(SCREEN_WINDOW_H);
|
||||
for (n, series) in &by_station {
|
||||
let limit = jump_limit(&code(*n));
|
||||
let mut last: Option<(NaiveDateTime, f64)> = None;
|
||||
let entries: Vec<_> = series.iter().map(|(t, v)| (*t, *v)).collect();
|
||||
for (k, &(t, (level, src))) in entries.iter().enumerate() {
|
||||
let prev = last.filter(|(pt, _)| t - *pt <= window).map(|(_, l)| l);
|
||||
let i = match src {
|
||||
Src::Stored => {
|
||||
last = Some((t, level));
|
||||
continue;
|
||||
}
|
||||
Src::Batch(i) => i,
|
||||
};
|
||||
let next = entries
|
||||
.get(k + 1)
|
||||
.filter(|(nt, _)| *nt - t <= window)
|
||||
.map(|(_, (l, _))| *l);
|
||||
match judge(limit, prev, level, next) {
|
||||
Verdict::Keep => {
|
||||
out.keep.push(readings[i].clone());
|
||||
last = Some((t, level));
|
||||
}
|
||||
Verdict::Hold => out.held.push(readings[i].clone()),
|
||||
Verdict::Spike => {
|
||||
out.spikes.push(readings[i].clone());
|
||||
if let Some(&old) = stored_at.get(&(t, *n)) {
|
||||
if judge(limit, prev, old, next) == Verdict::Spike {
|
||||
out.delete.push((t, *n));
|
||||
} else {
|
||||
last = Some((t, old));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
out.keep.sort_by_key(|r| (r.timestamp, r.station_num));
|
||||
out
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct StoreOutcome {
|
||||
pub saved: usize,
|
||||
pub held: usize,
|
||||
pub spikes: usize,
|
||||
pub deleted: usize,
|
||||
/// Saved readings that were new or differ from the stored ones, plus deletions.
|
||||
pub changed: usize,
|
||||
/// The earliest hour whose stored level changed or was deleted: the
|
||||
/// forecasts issued from then on came from a wrong level. (New hours and
|
||||
/// added discharges do not count: the catch-up forecasts new hours, and
|
||||
/// a forecast issued without the discharge is still what was issued.)
|
||||
pub first_change: Option<NaiveDateTime>,
|
||||
}
|
||||
|
||||
/// Screen `readings` against the stored ones around them, then save them and
|
||||
/// delete stored spikes.
|
||||
pub async fn store(
|
||||
pool: &PgPool,
|
||||
readings: &[Reading],
|
||||
stations: &Stations,
|
||||
) -> Result<StoreOutcome> {
|
||||
let mut out = StoreOutcome::default();
|
||||
let (Some(first), Some(last)) = (
|
||||
readings.iter().map(|r| r.timestamp).min(),
|
||||
readings.iter().map(|r| r.timestamp).max(),
|
||||
) else {
|
||||
return Ok(out);
|
||||
};
|
||||
let window = Duration::hours(SCREEN_WINDOW_H);
|
||||
let nums: Vec<u32> = readings
|
||||
.iter()
|
||||
.map(|r| r.station_num)
|
||||
.collect::<BTreeSet<_>>()
|
||||
.into_iter()
|
||||
.collect();
|
||||
let stored = db::rid_readings_between(pool, &nums, first - window, last + window).await?;
|
||||
let code = |n: u32| stations.get(n).code;
|
||||
let s = screen(readings, &stored, code);
|
||||
for r in &s.spikes {
|
||||
tracing::warn!(
|
||||
"RID {} {}: {} m is a one-hour spike, not stored",
|
||||
r.timestamp,
|
||||
code(r.station_num),
|
||||
r.water_level
|
||||
);
|
||||
}
|
||||
for r in &s.held {
|
||||
tracing::warn!(
|
||||
"RID {} {}: {} m jumps from the previous reading; waiting for the next one",
|
||||
r.timestamp,
|
||||
code(r.station_num),
|
||||
r.water_level
|
||||
);
|
||||
}
|
||||
let before: HashMap<(NaiveDateTime, u32), (f64, Option<f64>)> = stored
|
||||
.iter()
|
||||
.map(|(t, n, l, q)| ((*t, *n), (*l, *q)))
|
||||
.collect();
|
||||
// NUMERIC(10,3) / (10,2): compare at the stored precision
|
||||
let differs = |a: f64, b: f64, places: i32| {
|
||||
let f = 10f64.powi(places);
|
||||
(a * f).round() != (b * f).round()
|
||||
};
|
||||
let mut changed: Vec<NaiveDateTime> = s
|
||||
.keep
|
||||
.iter()
|
||||
.filter(|r| match before.get(&(r.timestamp, r.station_num)) {
|
||||
None => true,
|
||||
Some((l, q)) => {
|
||||
differs(*l, r.water_level, 3)
|
||||
|| match (q, r.discharge) {
|
||||
(Some(a), Some(b)) => differs(*a, b, 2),
|
||||
(None, None) => false,
|
||||
_ => true,
|
||||
}
|
||||
}
|
||||
})
|
||||
.map(|r| r.timestamp)
|
||||
.collect();
|
||||
changed.extend(s.delete.iter().map(|(t, _)| *t));
|
||||
let level_changed = s
|
||||
.keep
|
||||
.iter()
|
||||
.filter(|r| {
|
||||
before
|
||||
.get(&(r.timestamp, r.station_num))
|
||||
.is_some_and(|(l, _)| differs(*l, r.water_level, 3))
|
||||
})
|
||||
.map(|r| r.timestamp)
|
||||
.chain(s.delete.iter().map(|(t, _)| *t))
|
||||
.min();
|
||||
out.saved = db::save_rid(pool, &s.keep, stations).await?;
|
||||
out.deleted = db::delete_rid_readings(pool, &s.delete).await?;
|
||||
out.held = s.held.len();
|
||||
out.spikes = s.spikes.len();
|
||||
out.changed = changed.len();
|
||||
out.first_change = level_changed;
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Anything that can produce a parsed RID report (HTTP in production,
|
||||
/// canned rows in tests).
|
||||
pub trait RidSource {
|
||||
@@ -248,12 +496,14 @@ pub async fn run_cycle(
|
||||
tracing::warn!("RID: no data fetched for {:?}", out.dates);
|
||||
return out;
|
||||
}
|
||||
if let Err(e) = db::save_rid(pool, &readings, stations).await {
|
||||
tracing::error!("RID save failed: {e:#}");
|
||||
out.error = Some(format!("save failed: {e:#}"));
|
||||
return out;
|
||||
match store(pool, &readings, stations).await {
|
||||
Ok(s) => out.saved = s.saved,
|
||||
Err(e) => {
|
||||
tracing::error!("RID save failed: {e:#}");
|
||||
out.error = Some(format!("save failed: {e:#}"));
|
||||
return out;
|
||||
}
|
||||
}
|
||||
out.saved = readings.len();
|
||||
let latest = readings
|
||||
.iter()
|
||||
.map(|r| r.timestamp)
|
||||
@@ -308,8 +558,7 @@ pub async fn fill_gaps(
|
||||
}
|
||||
match src.fetch_report(*d).await {
|
||||
Ok(readings) if !readings.is_empty() => {
|
||||
db::save_rid(pool, &readings, stations).await?;
|
||||
filled += readings.len();
|
||||
filled += store(pool, &readings, stations).await?.saved;
|
||||
}
|
||||
Ok(_) => {
|
||||
empty += 1;
|
||||
@@ -332,6 +581,52 @@ pub async fn fill_gaps(
|
||||
Ok(filled)
|
||||
}
|
||||
|
||||
/// Re-fetch the reports of `from..=to` (report dates: day D's report holds
|
||||
/// D 01:00 to D+1 00:00) whether or not they have holes, and store them
|
||||
/// through the screening: corrects readings RID has revised since (and
|
||||
/// fills the discharge of level-only hours), removes stored spikes.
|
||||
pub async fn refetch(
|
||||
src: &impl RidSource,
|
||||
pool: &PgPool,
|
||||
stations: &Stations,
|
||||
from: NaiveDate,
|
||||
to: NaiveDate,
|
||||
) -> Result<StoreOutcome> {
|
||||
let mut total = StoreOutcome::default();
|
||||
let mut day = from;
|
||||
while day <= to {
|
||||
if day > from {
|
||||
tokio::time::sleep(std::time::Duration::from_secs(1)).await; // be polite
|
||||
}
|
||||
match src.fetch_report(day).await {
|
||||
Ok(readings) => {
|
||||
let s = store(pool, &readings, stations).await?;
|
||||
tracing::info!(
|
||||
"refetch {day}: {} fetched, {} saved ({} new or changed), {} spikes rejected, {} stored spikes deleted, {} held",
|
||||
readings.len(),
|
||||
s.saved,
|
||||
s.changed - s.deleted,
|
||||
s.spikes,
|
||||
s.deleted,
|
||||
s.held
|
||||
);
|
||||
total.saved += s.saved;
|
||||
total.held += s.held;
|
||||
total.spikes += s.spikes;
|
||||
total.deleted += s.deleted;
|
||||
total.changed += s.changed;
|
||||
total.first_change = match (total.first_change, s.first_change) {
|
||||
(Some(a), Some(b)) => Some(a.min(b)),
|
||||
(a, b) => a.or(b),
|
||||
};
|
||||
}
|
||||
Err(e) => tracing::warn!("refetch {day}: {e:#}"),
|
||||
}
|
||||
day += Duration::days(1);
|
||||
}
|
||||
Ok(total)
|
||||
}
|
||||
|
||||
/// Next RID poll: `offset` minutes past the next hour when current, else in
|
||||
/// `retry` minutes (never later than that hourly poll).
|
||||
pub fn next_poll_time(
|
||||
@@ -407,6 +702,104 @@ mod tests {
|
||||
assert!(!by.contains_key(&5)); // negative discharge drops the reading
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn judge_spikes_steps_and_limbs() {
|
||||
use Verdict::*;
|
||||
// the P.1 slip of 2026-10-25 00:00, a sign flip, a placeholder
|
||||
assert_eq!(judge(1.0, Some(1.04), 10.6, Some(1.07)), Spike);
|
||||
assert_eq!(judge(1.0, Some(-1.18), 1.18, Some(-1.18)), Spike);
|
||||
assert_eq!(judge(2.0, Some(-0.05), -5.0, Some(-0.05)), Spike);
|
||||
// small changes, no neighbour, a confirmed step, rising limbs
|
||||
assert_eq!(judge(1.0, Some(1.0), 1.9, Some(1.0)), Keep);
|
||||
assert_eq!(judge(1.0, None, 10.6, Some(1.07)), Keep);
|
||||
assert_eq!(judge(1.0, Some(1.0), 2.5, Some(2.6)), Keep);
|
||||
assert_eq!(judge(2.0, Some(0.04), 1.61, Some(2.67)), Keep);
|
||||
assert_eq!(judge(1.0, Some(0.82), 1.78, Some(2.82)), Keep);
|
||||
// a peak that falls back less than halfway in the next hour is real
|
||||
assert_eq!(judge(1.0, Some(1.0), 3.0, Some(1.8)), Keep);
|
||||
// the newest reading: a jump over twice the limit waits
|
||||
assert_eq!(judge(1.0, Some(1.04), 10.6, None), Hold);
|
||||
assert_eq!(judge(1.0, Some(1.0), 2.5, None), Keep);
|
||||
assert_eq!(judge(2.0, Some(0.09), 3.18, None), Keep);
|
||||
assert_eq!(jump_limit("P.76"), 2.0);
|
||||
assert_eq!(jump_limit("P.1"), 1.0);
|
||||
}
|
||||
|
||||
fn rd(h: u32, n: u32, level: f64) -> Reading {
|
||||
Reading {
|
||||
timestamp: t(2026, 10, 25, h, 0),
|
||||
station_num: n,
|
||||
water_level: level,
|
||||
discharge: None,
|
||||
discharge_percent: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn code(n: u32) -> String {
|
||||
if n == 8 {
|
||||
"P.1".into()
|
||||
} else {
|
||||
format!("P.{}", 19 + n)
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn screen_rejects_a_spike_and_deletes_its_stored_copy() {
|
||||
let stored = vec![
|
||||
(t(2026, 10, 24, 23, 0), 8, 1.04, None),
|
||||
(t(2026, 10, 25, 0, 0), 8, 10.6, None),
|
||||
];
|
||||
// the report re-fetched, RID still has the slip
|
||||
let batch = vec![rd(0, 8, 10.6), rd(1, 8, 1.07), rd(2, 8, 1.09)];
|
||||
let s = screen(&batch, &stored, code);
|
||||
assert_eq!(s.spikes, vec![rd(0, 8, 10.6)]);
|
||||
assert_eq!(s.delete, vec![(t(2026, 10, 25, 0, 0), 8)]);
|
||||
assert_eq!(s.keep, vec![rd(1, 8, 1.07), rd(2, 8, 1.09)]);
|
||||
// RID corrected it: the new value replaces the stored spike
|
||||
let s = screen(&[rd(0, 8, 1.06), rd(1, 8, 1.07)], &stored, code);
|
||||
assert!(s.spikes.is_empty() && s.delete.is_empty());
|
||||
assert_eq!(s.keep, vec![rd(0, 8, 1.06), rd(1, 8, 1.07)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn screen_keeps_a_good_stored_copy_and_judges_the_next_hour_by_it() {
|
||||
let stored = vec![
|
||||
(t(2026, 10, 25, 0, 0), 8, 1.06, None),
|
||||
(t(2026, 10, 25, 1, 0), 8, 1.07, None),
|
||||
];
|
||||
let s = screen(&[rd(1, 8, 10.7), rd(2, 8, 1.09)], &stored, code);
|
||||
assert_eq!(s.spikes, vec![rd(1, 8, 10.7)]);
|
||||
assert!(s.delete.is_empty());
|
||||
assert_eq!(s.keep, vec![rd(2, 8, 1.09)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn screen_holds_the_newest_jump_until_the_next_reading() {
|
||||
let stored = vec![(t(2026, 10, 25, 13, 0), 8, 1.04, None)];
|
||||
let s = screen(&[rd(14, 8, 10.6)], &stored, code);
|
||||
assert_eq!(s.held, vec![rd(14, 8, 10.6)]);
|
||||
assert!(s.keep.is_empty() && s.spikes.is_empty());
|
||||
// next poll: the report again, now with 15:00
|
||||
let s = screen(&[rd(14, 8, 10.6), rd(15, 8, 1.06)], &stored, code);
|
||||
assert_eq!(s.spikes, vec![rd(14, 8, 10.6)]);
|
||||
assert_eq!(s.keep, vec![rd(15, 8, 1.06)]);
|
||||
// a real jump the next reading confirms is stored then
|
||||
let s = screen(&[rd(14, 8, 3.5), rd(15, 8, 3.6)], &stored, code);
|
||||
assert_eq!(s.keep, vec![rd(14, 8, 3.5), rd(15, 8, 3.6)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn screen_window_and_stations_are_separate() {
|
||||
// previous reading 4 h back: nothing to compare with
|
||||
let stored = vec![(t(2026, 10, 25, 9, 0), 8, 1.0, None)];
|
||||
let s = screen(&[rd(13, 8, 5.0), rd(13, 1, 0.5)], &stored, code);
|
||||
assert_eq!(s.keep.len(), 2);
|
||||
// a jump at one gauge does not involve the others
|
||||
let stored = vec![(t(2026, 10, 25, 12, 0), 1, 0.1, None)];
|
||||
let s = screen(&[rd(13, 8, 1.0), rd(13, 1, 0.12)], &stored, code);
|
||||
assert_eq!(s.keep.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_rows_key_is_empty() {
|
||||
assert!(parse_report(d(2026, 7, 22), &json!({"unexpected": true})).is_empty());
|
||||
|
||||
@@ -398,12 +398,56 @@ impl Collector {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `ping-collector catch-up-forecasts`: `forecasting::catch_up` over
|
||||
/// `hours`, e.g. once after an upgrade for the nights before it.
|
||||
pub async fn catch_up_forecasts(
|
||||
&self,
|
||||
hours: i64,
|
||||
redo_from: Option<NaiveDateTime>,
|
||||
) -> Result<()> {
|
||||
if !(1..=24 * 60).contains(&hours) {
|
||||
bail!("--hours must be between 1 and 1440 (Open-Meteo keeps 92 days of past rain)");
|
||||
}
|
||||
let now = bangkok_now();
|
||||
if let Some(from) = redo_from
|
||||
&& from < floor_hour(now) - Duration::hours(hours)
|
||||
{
|
||||
bail!(
|
||||
"--redo-from {from} is more than --hours {hours} back: those hours would not be recomputed"
|
||||
);
|
||||
}
|
||||
let mut store = ModelStore::new(&self.cfg.models_dir);
|
||||
let models = store.refresh()?;
|
||||
if models.is_empty() {
|
||||
bail!(
|
||||
"no exported models in {} (run `python -m src.ml.export`)",
|
||||
self.cfg.models_dir.display()
|
||||
);
|
||||
}
|
||||
db::ensure_schema(&self.pool).await?;
|
||||
if let Some(from) = redo_from {
|
||||
let n = db::delete_model_forecasts_since(&self.pool, from).await?;
|
||||
eprintln!("deleted {n} model forecasts issued since {from}; recomputing");
|
||||
}
|
||||
let c = forecasting::catch_up(
|
||||
&self.pool,
|
||||
&self.rain,
|
||||
&models,
|
||||
now,
|
||||
hours,
|
||||
&Default::default(),
|
||||
)
|
||||
.await?;
|
||||
eprintln!("forecast catch-up: {}", forecasting::describe_catch_up(&c));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `ping-collector notify`: what would be (or, without `dry_run`, is)
|
||||
/// published right now against the stored notification state.
|
||||
pub async fn notify(&self, dry_run: bool) -> Result<()> {
|
||||
if !dry_run && !self.cfg.forecast_enabled {
|
||||
bail!(
|
||||
"the Python app notifies while FORECAST_IN_COLLECTOR is not true; \
|
||||
"notifications are off while FORECAST_IN_COLLECTOR is not true; \
|
||||
use --dry-run"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
//! Stale-while-revalidate response cache, as `_cached_swr` in src/web_api.py:
|
||||
//! Stale-while-revalidate response cache, as the retired FastAPI app's
|
||||
//! `_cached_swr`:
|
||||
//!
|
||||
//! - fresh entry: served;
|
||||
//! - expired entry (no older than `max_stale`): served at once, marked stale,
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
//! The website and read API: the port of the FastAPI app (src/web_api.py).
|
||||
//! The website and read API: the port of the retired FastAPI app.
|
||||
//!
|
||||
//! Same routes, parameters, JSON fields, status codes, cache headers and
|
||||
//! response caches as the Python app, checked route by route against it by
|
||||
//! scripts/api_parity.py. It runs inside `ping-collector run` (WEB_LISTEN)
|
||||
//! response caches as the Python app had; before the cutover (2026-10-18) a
|
||||
//! route-by-route comparison of the two on production data found no
|
||||
//! difference. It runs inside `ping-collector run` (WEB_LISTEN)
|
||||
//! or alone (`ping-collector serve`). /docs is the same Swagger UI, fed by an
|
||||
//! OpenAPI document built from the route table that also validates the
|
||||
//! parameters (openapi.rs, params.rs).
|
||||
|
||||
@@ -58,8 +58,8 @@ fn check(status: &'static str, message: impl Into<String>) -> Check {
|
||||
|
||||
/// Data freshness. As the Python check it judges the newest reading of the
|
||||
/// first station (`get_latest_measurements(limit=1)`: P.1) against 2 h. One
|
||||
/// change, as in the notifier: while RID's report lags (2-10 h at night) but
|
||||
/// the ThaiWater telemetry of the same gauges is current, the data is current.
|
||||
/// change, as in the notifier: while RID's report lags but the ThaiWater
|
||||
/// telemetry of the same gauges is current, the data is current.
|
||||
async fn database_check(st: &Shared) -> Check {
|
||||
let latest: Result<Option<(chrono::NaiveDateTime,)>, _> = sqlx::query_as(
|
||||
"SELECT m.timestamp
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
//! Route tests: the router as the server runs it, against a throwaway
|
||||
//! schema (TEST_DATABASE_URL; see db_tests.rs) or, for the routes that do
|
||||
//! not touch the database, a pool that never connects. Output formats are
|
||||
//! the FastAPI app's (scripts/api_parity.py compares the two live).
|
||||
//! the retired FastAPI app's (field order, isoformat, errors).
|
||||
|
||||
use super::{AppState, router};
|
||||
use crate::config::Config;
|
||||
|
||||
-127
@@ -1,127 +0,0 @@
|
||||
"""Notification cases for rust/collector/src/notify.rs, from src/notify.py itself.
|
||||
|
||||
uv run python rust/collector/testdata/gen_notify_cases.py
|
||||
|
||||
Random hourly sequences around the thresholds (capacity guard, hysteresis,
|
||||
danger escalation, the P.1 outlook, feed gaps, ntfy outages for some topics)
|
||||
run through notify.evaluate with an in-memory state; every delivered
|
||||
notification is recorded. The Rust test replays the same steps (RID readings
|
||||
only, so the ThaiWater additions stay out of play) and must deliver exactly
|
||||
the same messages.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import gzip
|
||||
import json
|
||||
import random
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE.parents[2]))
|
||||
|
||||
from src import notify # noqa: E402
|
||||
|
||||
|
||||
class Publisher(notify.NtfyPublisher):
|
||||
def __init__(self):
|
||||
super().__init__("http://ntfy.invalid", prefix="ping")
|
||||
self.down = set()
|
||||
|
||||
def publish(self, n):
|
||||
return n.topic not in self.down
|
||||
|
||||
|
||||
STATIONS = { # around (warning, danger)
|
||||
"P.1": (3.7, 4.2),
|
||||
"P.77": (4.3, 4.9),
|
||||
"P.21": (3.2, 3.6),
|
||||
"P.20": (2.35, 2.8),
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
rng = random.Random(20261018)
|
||||
sequences = []
|
||||
for _ in range(40):
|
||||
state = notify.InMemoryState()
|
||||
pub = Publisher()
|
||||
t = datetime.datetime(2026, 9, 1) + datetime.timedelta(
|
||||
hours=rng.randrange(2000)
|
||||
)
|
||||
codes = rng.sample(sorted(STATIONS), rng.randint(1, 4))
|
||||
levels = {c: STATIONS[c][0] + rng.uniform(-0.6, 0.6) for c in codes}
|
||||
p_out = rng.random()
|
||||
steps = []
|
||||
for _ in range(rng.randint(8, 25)):
|
||||
gap = rng.choice([1, 1, 1, 1, 2, 4, 6])
|
||||
t += datetime.timedelta(hours=gap)
|
||||
# the clock sometimes runs ahead of the data (a stale feed)
|
||||
now = t + datetime.timedelta(minutes=rng.choice([5, 5, 30, 200, 300]))
|
||||
readings = []
|
||||
for c in codes:
|
||||
levels[c] += rng.uniform(-0.25, 0.25)
|
||||
pct = rng.choice([None, rng.uniform(5, 120)])
|
||||
level = round(levels[c], 2) if rng.random() > 0.05 else None
|
||||
readings.append(
|
||||
{
|
||||
"station_code": c,
|
||||
"timestamp": t,
|
||||
"water_level": level,
|
||||
"discharge_percent": None if pct is None else round(pct, 2),
|
||||
}
|
||||
)
|
||||
p_out = min(1.0, max(0.0, p_out + rng.uniform(-0.3, 0.3)))
|
||||
forecasts = [
|
||||
{
|
||||
"station_code": "P.1",
|
||||
"horizon_hours": rng.choice([24, 24, 24, 12]),
|
||||
"p_warning": p_out,
|
||||
"predicted_max_level": round(rng.uniform(2.5, 5.0), 3),
|
||||
"source": rng.choice(["model", "model", "model", "heuristic"]),
|
||||
}
|
||||
]
|
||||
pub.down = set(
|
||||
rng.sample(
|
||||
[
|
||||
"ping-p1-warning",
|
||||
"ping-warning",
|
||||
"ping-status",
|
||||
"ping-p1-outlook",
|
||||
],
|
||||
rng.choice([0, 0, 0, 1]),
|
||||
)
|
||||
)
|
||||
sent = notify.evaluate(readings, forecasts, state, pub, now=now)
|
||||
steps.append(
|
||||
{
|
||||
"now": now.isoformat(),
|
||||
"readings": [
|
||||
{**r, "timestamp": r["timestamp"].isoformat()} for r in readings
|
||||
],
|
||||
"forecasts": forecasts,
|
||||
"down": sorted(pub.down),
|
||||
"sent": [
|
||||
{
|
||||
"topic": n.topic,
|
||||
"title": n.title,
|
||||
"message": n.message,
|
||||
"priority": n.priority,
|
||||
"tags": n.tags or [],
|
||||
}
|
||||
for n in sent
|
||||
],
|
||||
}
|
||||
)
|
||||
sequences.append(steps)
|
||||
out = HERE / "notify_python_cases.json.gz"
|
||||
with open(out, "wb") as raw, gzip.GzipFile(fileobj=raw, mode="wb", mtime=0) as gz:
|
||||
gz.write(json.dumps({"sequences": sequences}).encode())
|
||||
total = sum(len(s["sent"]) for seq in sequences for s in seq)
|
||||
print(
|
||||
f"{len(sequences)} sequences, {sum(map(len, sequences))} steps, {total} notifications"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -73,38 +73,28 @@
|
||||
"P.1": [
|
||||
{
|
||||
"stage": 1,
|
||||
"level": 3.7,
|
||||
"discharge_cms": 405.0
|
||||
},
|
||||
{
|
||||
"stage": 2,
|
||||
"level": 3.9,
|
||||
"discharge_cms": 438.0
|
||||
},
|
||||
{
|
||||
"stage": 3,
|
||||
"level": 4.0,
|
||||
"discharge_cms": 458.0
|
||||
},
|
||||
{
|
||||
"stage": 4,
|
||||
"level": 4.1,
|
||||
"discharge_cms": 478.0
|
||||
},
|
||||
{
|
||||
"stage": 5,
|
||||
"level": 4.2,
|
||||
"discharge_cms": 493.0
|
||||
},
|
||||
{
|
||||
"stage": 6,
|
||||
"level": 4.3,
|
||||
"discharge_cms": 508.0
|
||||
},
|
||||
{
|
||||
"stage": 7,
|
||||
"level": 4.6,
|
||||
"discharge_cms": 558.0
|
||||
"stage": 2,
|
||||
"level": 4.5,
|
||||
"discharge_cms": 538.0
|
||||
},
|
||||
{
|
||||
"stage": 3,
|
||||
"level": 4.7,
|
||||
"discharge_cms": 564.0
|
||||
},
|
||||
{
|
||||
"stage": 4,
|
||||
"level": 5.0,
|
||||
"discharge_cms": 604.0
|
||||
},
|
||||
{
|
||||
"stage": 5,
|
||||
"level": 5.3,
|
||||
"discharge_cms": 656.0
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -2,8 +2,9 @@
|
||||
//!
|
||||
//! For stations whose HII twin is the same sensor, hours missing from the RID
|
||||
//! series (or carrying no level) are filled from HII, shifted by the median
|
||||
//! offset between the two series over their overlap. This is also what keeps
|
||||
//! the forecast current at night while RID's report lags.
|
||||
//! offset between the two series over their overlap. ThaiWater relays RID's
|
||||
//! own telemetry, so this repairs holes in RID's report but cannot bridge
|
||||
//! RID's overnight pause (the mirror arrives with the same delay).
|
||||
|
||||
use crate::config::config;
|
||||
use crate::grid::{Row, floor_hour};
|
||||
|
||||
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
@@ -1,389 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare the Rust website/API with the FastAPI app, route by route.
|
||||
|
||||
Phase 3 of docs/RUST_REWRITE.md: before the Rust service takes over port
|
||||
8000, both answer the same requests against the same database and every
|
||||
response is compared: status, content type, Cache-Control and the JSON body
|
||||
(floats exactly; lists of database rows without ORDER BY as sets). Static
|
||||
files and the dashboard byte for byte.
|
||||
|
||||
# the FastAPI app on :8000, `ping-collector serve --listen 127.0.0.1:8001`
|
||||
uv run python scripts/api_parity.py \\
|
||||
--python http://127.0.0.1:8000 --rust http://127.0.0.1:8001
|
||||
|
||||
Query parameters vary per run (limits, hours) so the FastAPI app's response
|
||||
caches (up to 15 min) answer with fresh data, not an older copy. Data the
|
||||
collector writes during the run can still make a route differ once: a
|
||||
mismatch is retried after a pause before it counts.
|
||||
|
||||
Intended differences (the Rust side may answer differently):
|
||||
/health a lagging RID report with current ThaiWater telemetry for
|
||||
the same gauges is healthy (as the notifier, phase 2)
|
||||
/scraping/status extra fields (collector version, forecast run)
|
||||
/metrics process counters: compared by shape only
|
||||
/sensors/thaiwater live upstream: compared, a difference only warns
|
||||
/api/hii/rainfall/catchment correlation within 1e-9 (numpy uses BLAS)
|
||||
/api/stats newer counts on the Rust side are accepted (the FastAPI app
|
||||
caches this one for 5 minutes)
|
||||
*.geojson content type application/geo+json (FastAPI: octet-stream)
|
||||
|
||||
Exit status 0 when every hard check passed.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import random
|
||||
import sys
|
||||
import time
|
||||
from typing import Any, Callable, List, Optional, Tuple
|
||||
|
||||
import requests
|
||||
|
||||
SOFT = "soft" # a difference warns, does not fail
|
||||
|
||||
|
||||
def get(base: str, path: str, method: str = "GET", **kw) -> requests.Response:
|
||||
last = None
|
||||
for _ in range(3):
|
||||
try:
|
||||
return requests.request(method, base + path, timeout=120, **kw)
|
||||
except requests.RequestException as e: # connection reset etc.
|
||||
last = e
|
||||
time.sleep(1)
|
||||
raise last
|
||||
|
||||
|
||||
def first_diff(a: Any, b: Any, path: str = "$", rel: float = 0.0) -> Optional[str]:
|
||||
"""Where two decoded JSON values differ (None if equal)."""
|
||||
if isinstance(a, bool) or isinstance(b, bool):
|
||||
return None if a is b else f"{path}: {a!r} != {b!r}"
|
||||
if isinstance(a, (int, float)) and isinstance(b, (int, float)):
|
||||
if a == b:
|
||||
return None
|
||||
if rel and abs(a - b) <= rel * max(abs(a), abs(b)):
|
||||
return None
|
||||
return f"{path}: {a!r} != {b!r}"
|
||||
if type(a) is not type(b):
|
||||
return f"{path}: {type(a).__name__} {a!r} != {type(b).__name__} {b!r}"
|
||||
if isinstance(a, dict):
|
||||
if list(a.keys()) != list(b.keys()):
|
||||
if set(a.keys()) != set(b.keys()):
|
||||
return f"{path}: keys {sorted(set(a) ^ set(b))} only on one side"
|
||||
for k in a:
|
||||
d = first_diff(a[k], b[k], f"{path}.{k}", rel)
|
||||
if d:
|
||||
return d
|
||||
return None
|
||||
if isinstance(a, list):
|
||||
if len(a) != len(b):
|
||||
return f"{path}: {len(a)} items != {len(b)} items"
|
||||
for i, (x, y) in enumerate(zip(a, b)):
|
||||
d = first_diff(x, y, f"{path}[{i}]", rel)
|
||||
if d:
|
||||
return d
|
||||
return None
|
||||
return None if a == b else f"{path}: {a!r} != {b!r}"
|
||||
|
||||
|
||||
def media_type(r: requests.Response) -> str:
|
||||
return r.headers.get("content-type", "").split(";")[0].strip()
|
||||
|
||||
|
||||
class Case:
|
||||
def __init__(
|
||||
self,
|
||||
path: str,
|
||||
method: str = "GET",
|
||||
compare: Optional[Callable] = None,
|
||||
sort_key: Optional[str] = None,
|
||||
soft: bool = False,
|
||||
body: Any = None,
|
||||
headers: Optional[dict] = None,
|
||||
):
|
||||
self.path, self.method, self.compare = path, method, compare
|
||||
self.sort_key, self.soft, self.body = sort_key, soft, body
|
||||
self.headers = headers or {}
|
||||
|
||||
def run(self, py: str, rs: str) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""(hard difference, warning)"""
|
||||
kw = {"headers": self.headers}
|
||||
if self.body is not None:
|
||||
kw["json"] = self.body
|
||||
a = get(py, self.path, self.method, **kw)
|
||||
b = get(rs, self.path, self.method, **kw)
|
||||
if a.status_code != b.status_code:
|
||||
return self._fail(
|
||||
f"status {a.status_code} != {b.status_code} "
|
||||
f"({a.text[:120]!r} / {b.text[:120]!r})"
|
||||
)
|
||||
if media_type(a) != media_type(b):
|
||||
warn = f"content-type {media_type(a)!r} != {media_type(b)!r}"
|
||||
if media_type(a).startswith("application/json") or media_type(b).startswith(
|
||||
"application/json"
|
||||
):
|
||||
return self._fail(warn)
|
||||
return None, warn
|
||||
if self.method == "GET" and a.status_code == 200:
|
||||
ca, cb = a.headers.get("cache-control"), b.headers.get("cache-control")
|
||||
if ca != cb:
|
||||
return self._fail(f"Cache-Control {ca!r} != {cb!r}")
|
||||
if self.compare is not None:
|
||||
d = self.compare(a, b)
|
||||
elif media_type(a) == "application/json":
|
||||
ja, jb = a.json(), b.json()
|
||||
if a.status_code == 422:
|
||||
# pydantic's messages are approximated; type and location must agree
|
||||
ja = [(e["type"], e["loc"]) for e in ja["detail"]]
|
||||
jb = [(e["type"], e["loc"]) for e in jb["detail"]]
|
||||
if any(t.startswith("date") for t, _ in ja):
|
||||
ja, jb = [loc for _, loc in ja], [loc for _, loc in jb]
|
||||
elif self.sort_key and isinstance(ja, list):
|
||||
ja = sorted(ja, key=lambda r: str(r.get(self.sort_key)))
|
||||
jb = sorted(jb, key=lambda r: str(r.get(self.sort_key)))
|
||||
d = first_diff(ja, jb)
|
||||
else:
|
||||
d = (
|
||||
None
|
||||
if a.content == b.content
|
||||
else f"bodies differ ({len(a.content)} / {len(b.content)} bytes)"
|
||||
)
|
||||
return self._fail(d) if d else (None, None)
|
||||
|
||||
def _fail(self, d: str) -> Tuple[Optional[str], Optional[str]]:
|
||||
return (None, d) if self.soft else (d, None)
|
||||
|
||||
|
||||
def compare_health(a, b) -> Optional[str]:
|
||||
ja, jb = a.json(), b.json()
|
||||
if set(ja["checks"]) != set(jb["checks"]):
|
||||
return f"checks {sorted(ja['checks'])} != {sorted(jb['checks'])}"
|
||||
for name, ca in ja["checks"].items():
|
||||
cb = jb["checks"][name]
|
||||
if ca["status"] == cb["status"]:
|
||||
continue
|
||||
# the intended change: RID lagging, ThaiWater current
|
||||
if (
|
||||
name == "database"
|
||||
and ca["status"] == "degraded"
|
||||
and cb["status"] == "healthy"
|
||||
and "ThaiWater telemetry current" in cb["message"]
|
||||
):
|
||||
continue
|
||||
if name in ("api", "memory"):
|
||||
continue # probes of the RID API / own process, compared loosely
|
||||
return f"{name}: {ca} != {cb}"
|
||||
return None
|
||||
|
||||
|
||||
def compare_metrics(a, b) -> Optional[str]:
|
||||
ja, jb = a.json(), b.json()
|
||||
return None if set(ja) == set(jb) else f"keys {sorted(ja)} != {sorted(jb)}"
|
||||
|
||||
|
||||
def compare_status(a, b) -> Optional[str]:
|
||||
ja, jb = a.json(), b.json()
|
||||
for k, v in ja.items():
|
||||
if jb.get(k) != v:
|
||||
return f"{k}: {v!r} != {jb.get(k)!r}"
|
||||
return None
|
||||
|
||||
|
||||
def compare_skill(a, b) -> Optional[str]:
|
||||
ja, jb = a.json(), b.json()
|
||||
va, vb = ja.pop("verified_until"), jb.pop("verified_until")
|
||||
gap = abs(
|
||||
time.mktime(time.strptime(va[:19], "%Y-%m-%dT%H:%M:%S"))
|
||||
- time.mktime(time.strptime(vb[:19], "%Y-%m-%dT%H:%M:%S"))
|
||||
)
|
||||
if gap > 300:
|
||||
return f"verified_until {va} != {vb}"
|
||||
return first_diff(ja, jb)
|
||||
|
||||
|
||||
def compare_stats(a, b) -> Optional[str]:
|
||||
"""Exact, except that the FastAPI app may answer from its 5-minute cache
|
||||
while the collector has written newer rows since."""
|
||||
ja, jb = a.json(), b.json()
|
||||
if ja == jb:
|
||||
return None
|
||||
for k, v in ja.items():
|
||||
w = jb.get(k)
|
||||
if k.endswith("_measurements") or k == "days_spanned":
|
||||
if not isinstance(w, int) or w < v:
|
||||
return f"{k}: {v!r} > {w!r}"
|
||||
elif k == "last_timestamp":
|
||||
if not isinstance(w, str) or w < v:
|
||||
return f"{k}: {v!r} > {w!r}"
|
||||
elif k == "coverage_percent":
|
||||
if not isinstance(w, (int, float)) or abs(w - v) > 0.2:
|
||||
return f"{k}: {v!r} != {w!r}"
|
||||
elif v != w:
|
||||
return f"{k}: {v!r} != {w!r}"
|
||||
return None
|
||||
|
||||
|
||||
def compare_catchment(a, b) -> Optional[str]:
|
||||
return first_diff(a.json(), b.json(), rel=1e-9)
|
||||
|
||||
|
||||
def compare_openapi(a, b) -> Optional[str]:
|
||||
ja, jb = a.json(), b.json()
|
||||
if ja["info"]["title"] != jb["info"]["title"]:
|
||||
return "title differs"
|
||||
ops = lambda spec: { # noqa: E731
|
||||
(m, p): op for p, item in spec["paths"].items() for m, op in item.items()
|
||||
}
|
||||
oa, ob = ops(ja), ops(jb)
|
||||
if set(oa) != set(ob):
|
||||
return f"operations only on one side: {sorted(set(oa) ^ set(ob))}"
|
||||
for k, op in oa.items():
|
||||
other = ob[k]
|
||||
for field in ("operationId", "summary", "parameters", "requestBody"):
|
||||
d = first_diff(op.get(field), other.get(field), f"{k}.{field}")
|
||||
if d:
|
||||
return d
|
||||
if set(op["responses"]) != set(other["responses"]):
|
||||
return f"{k}: responses {sorted(op['responses'])} != {sorted(other['responses'])}"
|
||||
for name in ja["components"]["schemas"]:
|
||||
if name not in jb["components"]["schemas"]:
|
||||
return f"schema {name} missing"
|
||||
return None
|
||||
|
||||
|
||||
def compare_html_docs(a, b) -> Optional[str]:
|
||||
return (
|
||||
None
|
||||
if "swagger-ui" in b.text and "/openapi.json" in b.text
|
||||
else "no Swagger UI"
|
||||
)
|
||||
|
||||
|
||||
def cases(admin_key: Optional[str]) -> List[Case]:
|
||||
r = random.Random(time.time())
|
||||
hours = r.randint(150, 190) # fresh keys for the Python app's caches
|
||||
limit = r.randint(40000, 49999)
|
||||
feed_hours = r.randint(20, 30)
|
||||
skill_station = r.choice(
|
||||
["P.67", "P.103", "P.20", "P.75", "P.92", "P.21", "P.5", "P.82"]
|
||||
)
|
||||
day = time.strftime("%Y-%m-%d", time.localtime(time.time() - 86400 * 3))
|
||||
day2 = time.strftime("%Y-%m-%d", time.localtime(time.time() - 86400))
|
||||
out = [
|
||||
Case("/"),
|
||||
Case("/robots.txt"),
|
||||
Case("/llms.txt"),
|
||||
Case("/sitemap.xml"),
|
||||
Case("/static/ping-river-network.geojson", soft=True),
|
||||
Case("/static/custom-markers.json"),
|
||||
Case("/static/flood-zones-p1.geojson", soft=True),
|
||||
Case("/static/flood-2024-all.json"),
|
||||
Case("/static/nope.json"),
|
||||
Case("/openapi.json", compare=compare_openapi),
|
||||
Case("/docs", compare=compare_html_docs),
|
||||
Case("/health", compare=compare_health),
|
||||
Case("/metrics", compare=compare_metrics),
|
||||
Case("/stations"),
|
||||
Case("/stations/8"),
|
||||
Case("/stations/1"),
|
||||
Case("/stations/999"),
|
||||
Case("/stations/abc"),
|
||||
Case(f"/measurements/latest?limit={r.randint(200, 499)}"),
|
||||
Case(f"/measurements/latest?limit={r.randint(1, 15)}"),
|
||||
Case(f"/measurements/history/P.1?hours={hours}&limit={limit}"),
|
||||
Case(f"/measurements/history/P.67?hours={hours + 1}"),
|
||||
Case(f"/measurements/history/P.1?start={day}&end={day2}&limit={limit - 1}"),
|
||||
Case(f"/measurements/history/P.1?start={day}&limit={r.randint(5, 50)}"),
|
||||
Case(f"/measurements/history/P.1?start={day2}&end={day}"),
|
||||
Case(f"/measurements/history/NOPE?hours={hours}"),
|
||||
Case("/measurements/history/P.1?hours=0"),
|
||||
Case("/measurements/history/P.1?limit=100001"),
|
||||
Case("/measurements/history/P.1?start=yesterday"),
|
||||
Case(f"/measurements/station/P.1?hours={r.randint(20, 48)}"),
|
||||
Case("/measurements/station/P.103?hours=6&limit=2"),
|
||||
Case(f"/api/hii/rainfall/latest?hours={feed_hours}", sort_key="station_id"),
|
||||
Case(
|
||||
f"/api/hii/waterlevel/latest?hours={feed_hours + 1}", sort_key="station_id"
|
||||
),
|
||||
Case("/api/hii/waterlevel/latest?hours=500"),
|
||||
Case(
|
||||
f"/api/hii/rainfall/catchment?days={r.randint(15, 30)}",
|
||||
compare=compare_catchment,
|
||||
),
|
||||
Case("/sensors/thaiwater", sort_key="id", soft=True),
|
||||
Case("/forecast"),
|
||||
Case(f"/api/forecast/history/P.1?hours={hours}&horizon=24"),
|
||||
Case(f"/api/forecast/history/P.103?hours={feed_hours}"),
|
||||
Case(f"/api/forecast/history/P.1?start={day}&end={day2}&horizon=6"),
|
||||
Case("/api/forecast/history/P.1?horizon=99"),
|
||||
Case("/api/notifications"),
|
||||
# a station/horizon pair the dashboard does not ask for: not in the
|
||||
# FastAPI app's 15-minute cache; the dashboard's own one may be
|
||||
Case(
|
||||
f"/api/forecast/skill?station_code={skill_station}"
|
||||
f"&horizon={r.randint(1, 48)}",
|
||||
compare=compare_skill,
|
||||
),
|
||||
Case(
|
||||
"/api/forecast/skill?station_code=P.1&horizon=24",
|
||||
compare=compare_skill,
|
||||
soft=True,
|
||||
),
|
||||
Case("/api/forecast/skill?horizon=0"),
|
||||
Case("/api/stats", compare=compare_stats),
|
||||
Case("/scraping/status", compare=compare_status),
|
||||
Case("/nope"),
|
||||
Case("/forecast", method="DELETE"),
|
||||
Case("/config"),
|
||||
Case("/scrape/trigger", method="POST"),
|
||||
Case("/stations", method="POST", body={"station_code": "X"}),
|
||||
Case("/stations/8", method="DELETE"),
|
||||
]
|
||||
if admin_key:
|
||||
key = {"X-API-Key": admin_key}
|
||||
out += [
|
||||
Case("/config", headers={"X-API-Key": "wrong"}),
|
||||
Case("/stations", method="POST", body={"thai_name": 1}, headers=key),
|
||||
Case("/stations/999", method="PUT", body={}, headers=key),
|
||||
]
|
||||
return out
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
|
||||
ap.add_argument("--python", default="http://127.0.0.1:8000")
|
||||
ap.add_argument("--rust", default="http://127.0.0.1:8001")
|
||||
ap.add_argument(
|
||||
"--admin-key", help="ADMIN_API_KEY, to compare the admin errors too"
|
||||
)
|
||||
ap.add_argument("--retries", type=int, default=2)
|
||||
ap.add_argument("--pause", type=float, default=30.0, help="seconds before a retry")
|
||||
args = ap.parse_args()
|
||||
|
||||
failed, warned, passed = [], [], 0
|
||||
for case in cases(args.admin_key):
|
||||
label = f"{case.method} {case.path}"
|
||||
for attempt in range(args.retries + 1):
|
||||
try:
|
||||
hard, warn = case.run(args.python, args.rust)
|
||||
except Exception as e: # noqa: BLE001 - report and go on
|
||||
hard, warn = f"{type(e).__name__}: {e}", None
|
||||
if not hard or attempt == args.retries:
|
||||
break
|
||||
print(f" retry {label}: {hard}", flush=True)
|
||||
time.sleep(args.pause)
|
||||
if hard:
|
||||
failed.append((label, hard))
|
||||
print(f"FAIL {label}: {hard}", flush=True)
|
||||
elif warn:
|
||||
warned.append((label, warn))
|
||||
print(f"warn {label}: {warn}", flush=True)
|
||||
else:
|
||||
passed += 1
|
||||
print(
|
||||
f"{passed} identical, {len(warned)} warnings, {len(failed)} different "
|
||||
f"({args.python} vs {args.rust})"
|
||||
)
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -2,7 +2,7 @@
|
||||
"""Backfill the openmeteo_rain table with the full 2021+ catchment history.
|
||||
|
||||
Usage:
|
||||
uv run scripts/backfill_rain_db.py # DB from Config/.env
|
||||
uv run scripts/backfill_rain_db.py # DB from POSTGRES_* / .env
|
||||
uv run scripts/backfill_rain_db.py --db-url postgresql://...
|
||||
"""
|
||||
|
||||
@@ -13,7 +13,7 @@ import sys
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
|
||||
|
||||
from src.config import Config
|
||||
from src.config import database_url
|
||||
from src.ml.rain import backfill_db
|
||||
|
||||
|
||||
@@ -25,21 +25,17 @@ def main(argv=None) -> int:
|
||||
logging.basicConfig(
|
||||
level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s"
|
||||
)
|
||||
if args.db_url:
|
||||
# postgresql+psycopg2://... -> postgresql (driver suffix is not a dialect)
|
||||
connection_string = args.db_url
|
||||
db_type = args.db_url.split(":", 1)[0].split("+", 1)[0]
|
||||
else:
|
||||
cfg = Config.get_database_config()
|
||||
if cfg["type"] not in ("sqlite", "postgresql", "mysql"):
|
||||
print(f"requires a SQL database, got {cfg['type']}", file=sys.stderr)
|
||||
return 1
|
||||
connection_string, db_type = cfg["connection_string"], cfg["type"]
|
||||
connection_string = args.db_url or database_url()
|
||||
if not connection_string:
|
||||
print(
|
||||
"no database: set POSTGRES_CONNECTION_STRING or POSTGRES_*", file=sys.stderr
|
||||
)
|
||||
return 1
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
|
||||
engine = create_engine(connection_string, pool_pre_ping=True)
|
||||
saved = backfill_db(engine, db_type)
|
||||
saved = backfill_db(engine)
|
||||
print(f"backfilled {saved} hourly rows into openmeteo_rain")
|
||||
return 0 if saved else 1
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
|
||||
Two paths, both idempotent and both skipping what is already stored, so a
|
||||
rerun repairs holes left by transient failures and is safe alongside the
|
||||
hourly live collector:
|
||||
live collector (ping-collector, today + yesterday):
|
||||
|
||||
--dam-id (default: Mae Ngat) one dam, whole range, via api/dam — a handful
|
||||
of requests for the entire 2009-today archive
|
||||
@@ -26,13 +26,8 @@ import sys
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
|
||||
|
||||
from src.config import Config
|
||||
from src.rid_reservoir import (
|
||||
MAE_NGAT_DAM_ID,
|
||||
RidReservoirStore,
|
||||
backfill,
|
||||
backfill_dam,
|
||||
)
|
||||
from src.config import database_url
|
||||
from src.rid_reservoir import MAE_NGAT_DAM_ID, RidReservoirStore, backfill, backfill_dam
|
||||
|
||||
DEFAULT_START = datetime.date(2018, 8, 1) # start of the water_measurements grid
|
||||
|
||||
@@ -71,18 +66,14 @@ def main(argv=None) -> int:
|
||||
logging.basicConfig(
|
||||
level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s"
|
||||
)
|
||||
if args.db_url:
|
||||
# postgresql+psycopg2://... -> postgresql (driver suffix is not a dialect)
|
||||
connection_string = args.db_url
|
||||
db_type = args.db_url.split(":", 1)[0].split("+", 1)[0]
|
||||
else:
|
||||
cfg = Config.get_database_config()
|
||||
if cfg["type"] not in ("sqlite", "postgresql", "mysql"):
|
||||
print(f"requires a SQL database, got {cfg['type']}", file=sys.stderr)
|
||||
return 1
|
||||
connection_string, db_type = cfg["connection_string"], cfg["type"]
|
||||
connection_string = args.db_url or database_url()
|
||||
if not connection_string:
|
||||
print(
|
||||
"no database: set POSTGRES_CONNECTION_STRING or POSTGRES_*", file=sys.stderr
|
||||
)
|
||||
return 1
|
||||
|
||||
store = RidReservoirStore(connection_string, db_type)
|
||||
store = RidReservoirStore(connection_string)
|
||||
if not store.connect():
|
||||
print(
|
||||
"database connection failed — check the connection string",
|
||||
|
||||
+188
-75
@@ -14,7 +14,7 @@ non-zero exit if the model gives less than 12 h of warning before the first
|
||||
3.70 m crossing of the 2024 event.
|
||||
|
||||
Usage:
|
||||
python scripts/backtest_render.py # uses FLOOD_ML_DB_URL/Config
|
||||
python scripts/backtest_render.py # uses FLOOD_ML_DB_URL / POSTGRES_*
|
||||
python scripts/backtest_render.py --db-url postgresql://...
|
||||
"""
|
||||
|
||||
@@ -60,7 +60,11 @@ def fit_backtest_model(df_long: pd.DataFrame, train_end: str, use_dam: bool = Fa
|
||||
rain_series = rain_mod.catchment_mean(rain_mod.load_history())
|
||||
dam_frame = dam_mod.load_history() if use_dam else None
|
||||
X, Y, _meta = features.build_matrix(
|
||||
df_long, STATION, (HORIZON,), stats_end=train_end, rain=rain_series,
|
||||
df_long,
|
||||
STATION,
|
||||
(HORIZON,),
|
||||
stats_end=train_end,
|
||||
rain=rain_series,
|
||||
dam=dam_frame,
|
||||
)
|
||||
train_mask = X.index <= pd.Timestamp(train_end)
|
||||
@@ -92,6 +96,7 @@ def event_series(df_long, X, reg, clf, window_start: str, window_end: str):
|
||||
# whichever is more alarmed. The classifier alone proved unreliable on
|
||||
# out-of-distribution extremes (silent on the 2024 record flood).
|
||||
import numpy as np
|
||||
|
||||
p_clf = clf.predict_proba(Xw)[:, 1]
|
||||
p_sig = 1.0 / (1.0 + np.exp(-(forecasts["pred_max"] - STAGE1) / 0.15))
|
||||
forecasts["p_flood"] = np.maximum(p_clf, p_sig)
|
||||
@@ -109,55 +114,107 @@ def _style_axes(ax):
|
||||
ax.set_axisbelow(True)
|
||||
|
||||
|
||||
def render(observed, forecasts, flood_start, first_alert, out_path, *,
|
||||
title, subtitle, detail=False, show_stage7=False, peak_note=None):
|
||||
def render(
|
||||
observed,
|
||||
forecasts,
|
||||
flood_start,
|
||||
first_alert,
|
||||
out_path,
|
||||
*,
|
||||
title,
|
||||
subtitle,
|
||||
detail=False,
|
||||
show_stage7=False,
|
||||
peak_note=None,
|
||||
):
|
||||
fig, (ax, axp) = plt.subplots(
|
||||
2, 1, figsize=(12.6, 7.6), sharex=True,
|
||||
2,
|
||||
1,
|
||||
figsize=(12.6, 7.6),
|
||||
sharex=True,
|
||||
gridspec_kw={"height_ratios": [2.2, 1], "hspace": 0.12},
|
||||
)
|
||||
fig.patch.set_facecolor("white")
|
||||
|
||||
marker = dict(marker="o", markersize=3) if detail else {}
|
||||
ax.plot(observed.index, observed.values, color=BLUE, linewidth=2.2,
|
||||
label="Observed level" + (" (hourly)" if detail else ""), **marker)
|
||||
ax.plot(
|
||||
observed.index,
|
||||
observed.values,
|
||||
color=BLUE,
|
||||
linewidth=2.2,
|
||||
label="Observed level" + (" (hourly)" if detail else ""),
|
||||
**marker,
|
||||
)
|
||||
marker = dict(marker="s", markersize=3) if detail else {}
|
||||
ax.plot(forecasts.index, forecasts["pred_max"], color=AMBER, linewidth=2,
|
||||
linestyle="--", label="Predicted 24 h peak (issued at that hour)", **marker)
|
||||
ax.plot(
|
||||
forecasts.index,
|
||||
forecasts["pred_max"],
|
||||
color=AMBER,
|
||||
linewidth=2,
|
||||
linestyle="--",
|
||||
label="Predicted 24 h peak (issued at that hour)",
|
||||
**marker,
|
||||
)
|
||||
|
||||
ax.axhline(STAGE1, color=RED, linewidth=1, alpha=0.65)
|
||||
ax.annotate(f"{STAGE1:.2f} m · stage 1 · flooding begins", xy=(0.06, STAGE1),
|
||||
xycoords=("axes fraction", "data"), xytext=(0, 5),
|
||||
textcoords="offset points", color=RED, fontsize=10.5)
|
||||
ax.annotate(
|
||||
f"{STAGE1:.2f} m · stage 1 · flooding begins",
|
||||
xy=(0.06, STAGE1),
|
||||
xycoords=("axes fraction", "data"),
|
||||
xytext=(0, 5),
|
||||
textcoords="offset points",
|
||||
color=RED,
|
||||
fontsize=10.5,
|
||||
)
|
||||
if show_stage7:
|
||||
ax.axhline(STAGE7, color=RED, linewidth=1, alpha=0.65)
|
||||
ax.annotate(f"{STAGE7:.2f} m · stage 7 · widespread", xy=(0.06, STAGE7),
|
||||
xycoords=("axes fraction", "data"), xytext=(0, 5),
|
||||
textcoords="offset points", color=RED, fontsize=10.5)
|
||||
ax.annotate(
|
||||
f"{STAGE7:.2f} m · stage 7 · widespread",
|
||||
xy=(0.06, STAGE7),
|
||||
xycoords=("axes fraction", "data"),
|
||||
xytext=(0, 5),
|
||||
textcoords="offset points",
|
||||
color=RED,
|
||||
fontsize=10.5,
|
||||
)
|
||||
|
||||
if peak_note:
|
||||
peak_ts = observed.idxmax()
|
||||
ax.annotate(peak_note, xy=(peak_ts, observed.max()),
|
||||
xytext=(12, 10), textcoords="offset points",
|
||||
color=BLUE, fontsize=11.5, fontweight="bold")
|
||||
ax.annotate(
|
||||
peak_note,
|
||||
xy=(peak_ts, observed.max()),
|
||||
xytext=(12, 10),
|
||||
textcoords="offset points",
|
||||
color=BLUE,
|
||||
fontsize=11.5,
|
||||
fontweight="bold",
|
||||
)
|
||||
|
||||
ax.set_ylabel("P.1 water level (m)", color=INK, fontsize=11.5)
|
||||
ax.legend(loc="upper left", frameon=False, fontsize=10.5)
|
||||
_style_axes(ax)
|
||||
|
||||
axp.plot(forecasts.index, forecasts["p_flood"], color=AMBER, linewidth=1.8)
|
||||
axp.fill_between(forecasts.index, 0, forecasts["p_flood"],
|
||||
color=AMBER, alpha=0.28)
|
||||
axp.fill_between(forecasts.index, 0, forecasts["p_flood"], color=AMBER, alpha=0.28)
|
||||
axp.axhline(0.5, color=INK, linewidth=0.9, linestyle=":", alpha=0.6)
|
||||
axp.set_ylim(-0.02, 1.1)
|
||||
axp.set_ylabel(f"P(flooding within {HORIZON} h)", color=INK, fontsize=11.5)
|
||||
_style_axes(axp)
|
||||
|
||||
if first_alert is not None:
|
||||
lead_h = None if flood_start is None else \
|
||||
int((flood_start - first_alert).total_seconds() // 3600)
|
||||
lead_txt = "" if lead_h is None else (
|
||||
f"\n({lead_h} h before flooding began)" if lead_h >= 0
|
||||
else f"\n({-lead_h} h after flooding began)"
|
||||
lead_h = (
|
||||
None
|
||||
if flood_start is None
|
||||
else int((flood_start - first_alert).total_seconds() // 3600)
|
||||
)
|
||||
lead_txt = (
|
||||
""
|
||||
if lead_h is None
|
||||
else (
|
||||
f"\n({lead_h} h before flooding began)"
|
||||
if lead_h >= 0
|
||||
else f"\n({-lead_h} h after flooding began)"
|
||||
)
|
||||
)
|
||||
if detail and flood_start is not None:
|
||||
for a in (ax, axp):
|
||||
@@ -166,34 +223,61 @@ def render(observed, forecasts, flood_start, first_alert, out_path, *,
|
||||
# Anchor labels away from each other in chronological order so a
|
||||
# late alert (alert AFTER crossing) cannot overprint the labels.
|
||||
events = sorted(
|
||||
[(first_alert, "model alert", AMBER), (flood_start, "flooding begins", BLUE)]
|
||||
[
|
||||
(first_alert, "model alert", AMBER),
|
||||
(flood_start, "flooding begins", BLUE),
|
||||
]
|
||||
)
|
||||
for (ts, label, color), (offset, align) in zip(events, ((-8, "right"), (8, "left"))):
|
||||
ax.annotate(f"{label}\n{ts:%d %b %H:%M}",
|
||||
xy=(ts, observed.min()), xytext=(offset, 18),
|
||||
textcoords="offset points", ha=align,
|
||||
color=color, fontsize=11, fontweight="bold")
|
||||
for (ts, label, color), (offset, align) in zip(
|
||||
events, ((-8, "right"), (8, "left"))
|
||||
):
|
||||
ax.annotate(
|
||||
f"{label}\n{ts:%d %b %H:%M}",
|
||||
xy=(ts, observed.min()),
|
||||
xytext=(offset, 18),
|
||||
textcoords="offset points",
|
||||
ha=align,
|
||||
color=color,
|
||||
fontsize=11,
|
||||
fontweight="bold",
|
||||
)
|
||||
mid_y = observed.min() + (observed.max() - observed.min()) * 0.28
|
||||
ax.annotate("", xy=(flood_start, mid_y), xytext=(first_alert, mid_y),
|
||||
arrowprops=dict(arrowstyle="<->", color=INK, lw=1.3))
|
||||
ax.annotate(
|
||||
"",
|
||||
xy=(flood_start, mid_y),
|
||||
xytext=(first_alert, mid_y),
|
||||
arrowprops=dict(arrowstyle="<->", color=INK, lw=1.3),
|
||||
)
|
||||
arrow_label = (
|
||||
f"{lead_h} h warning" if lead_h >= 0 else f"alert {-lead_h} h late"
|
||||
)
|
||||
ax.annotate(arrow_label,
|
||||
xy=(first_alert + (flood_start - first_alert) / 2, mid_y),
|
||||
xytext=(0, 8), textcoords="offset points", ha="center",
|
||||
color=INK, fontsize=11.5, fontweight="bold")
|
||||
ax.annotate(
|
||||
arrow_label,
|
||||
xy=(first_alert + (flood_start - first_alert) / 2, mid_y),
|
||||
xytext=(0, 8),
|
||||
textcoords="offset points",
|
||||
ha="center",
|
||||
color=INK,
|
||||
fontsize=11.5,
|
||||
fontweight="bold",
|
||||
)
|
||||
else:
|
||||
axp.annotate(f"first alert · {first_alert:%d %b %H:%M}{lead_txt}",
|
||||
xy=(first_alert, 0.62), xytext=(10, 0),
|
||||
textcoords="offset points", color=RED, fontsize=10.5,
|
||||
bbox=dict(facecolor="white", alpha=0.75, edgecolor="none"))
|
||||
axp.annotate(
|
||||
f"first alert · {first_alert:%d %b %H:%M}{lead_txt}",
|
||||
xy=(first_alert, 0.62),
|
||||
xytext=(10, 0),
|
||||
textcoords="offset points",
|
||||
color=RED,
|
||||
fontsize=10.5,
|
||||
bbox=dict(facecolor="white", alpha=0.75, edgecolor="none"),
|
||||
)
|
||||
|
||||
locator = mdates.DayLocator(interval=1 if detail else 3)
|
||||
axp.xaxis.set_major_locator(locator)
|
||||
axp.xaxis.set_major_formatter(mdates.DateFormatter("%d %b"))
|
||||
fig.suptitle(f"{title}\n{subtitle}", x=0.07, y=0.985, ha="left",
|
||||
fontsize=15, color=INK)
|
||||
fig.suptitle(
|
||||
f"{title}\n{subtitle}", x=0.07, y=0.985, ha="left", fontsize=15, color=INK
|
||||
)
|
||||
fig.subplots_adjust(top=0.885, left=0.07, right=0.97, bottom=0.07)
|
||||
fig.savefig(out_path, dpi=110)
|
||||
plt.close(fig)
|
||||
@@ -204,16 +288,20 @@ def main(argv=None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--db-url", default=None)
|
||||
parser.add_argument("--out-dir", default=os.path.join("docs", "img"))
|
||||
parser.add_argument("--dam", action="store_true",
|
||||
help="ablation: include Mae Ngat reservoir features "
|
||||
"(2026-08 result: costs 1-3 h of alert lead)")
|
||||
parser.add_argument("--no-hii-fill", action="store_true",
|
||||
help="ablation: load without the HII gap-fill merge")
|
||||
parser.add_argument(
|
||||
"--dam",
|
||||
action="store_true",
|
||||
help="ablation: include Mae Ngat reservoir features "
|
||||
"(2026-08 result: costs 1-3 h of alert lead)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-hii-fill",
|
||||
action="store_true",
|
||||
help="ablation: load without the HII gap-fill merge",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
df = data.load_measurements(
|
||||
db_url=args.db_url, hii_fill=not args.no_hii_fill
|
||||
)
|
||||
df = data.load_measurements(db_url=args.db_url, hii_fill=not args.no_hii_fill)
|
||||
if df.empty:
|
||||
print("no measurement data available", file=sys.stderr)
|
||||
return 1
|
||||
@@ -222,41 +310,66 @@ def main(argv=None) -> int:
|
||||
# --- October 2024 record flood: trained only on data before 1 Sep 2024 ---
|
||||
X, reg, clf = fit_backtest_model(df, "2024-08-31", use_dam=args.dam)
|
||||
obs, fc, flood_start, first_alert = event_series(
|
||||
df, X, reg, clf, "2024-09-10", "2024-10-14 23:00")
|
||||
df, X, reg, clf, "2024-09-10", "2024-10-14 23:00"
|
||||
)
|
||||
peak = float(obs.max())
|
||||
render(obs, fc, flood_start, first_alert,
|
||||
os.path.join(args.out_dir, "backtest-2024-p1.png"),
|
||||
title="October 2024 flood: what the model saw coming",
|
||||
subtitle="P.1 Nawarat Bridge — model trained only on data before 1 Sep 2024",
|
||||
show_stage7=True, peak_note=f"record peak {peak:.2f} m")
|
||||
render(
|
||||
obs,
|
||||
fc,
|
||||
flood_start,
|
||||
first_alert,
|
||||
os.path.join(args.out_dir, "backtest-2024-p1.png"),
|
||||
title="October 2024 flood: what the model saw coming",
|
||||
subtitle="P.1 Nawarat Bridge — model trained only on data before 1 Sep 2024",
|
||||
show_stage7=True,
|
||||
peak_note=f"record peak {peak:.2f} m",
|
||||
)
|
||||
|
||||
obs_d, fc_d, flood_d, alert_d = event_series(
|
||||
df, X, reg, clf, "2024-09-21 18:00", "2024-09-28 06:00")
|
||||
df, X, reg, clf, "2024-09-21 18:00", "2024-09-28 06:00"
|
||||
)
|
||||
lead_h = None
|
||||
if alert_d is not None and flood_d is not None:
|
||||
lead_h = int((flood_d - alert_d).total_seconds() // 3600)
|
||||
render(obs_d, fc_d, flood_d, alert_d,
|
||||
os.path.join(args.out_dir, "backtest-2024-p1-detail.png"),
|
||||
title="Detection in detail: 22–28 September 2024, hour by hour",
|
||||
subtitle=(
|
||||
f"the model alerts {lead_h} h before the river crosses the flooding line"
|
||||
if lead_h is not None and lead_h > 0
|
||||
else "model alert vs the river crossing the flooding line"
|
||||
),
|
||||
detail=True)
|
||||
render(
|
||||
obs_d,
|
||||
fc_d,
|
||||
flood_d,
|
||||
alert_d,
|
||||
os.path.join(args.out_dir, "backtest-2024-p1-detail.png"),
|
||||
title="Detection in detail: 22–28 September 2024, hour by hour",
|
||||
subtitle=(
|
||||
f"the model alerts {lead_h} h before the river crosses the flooding line"
|
||||
if lead_h is not None and lead_h > 0
|
||||
else "model alert vs the river crossing the flooding line"
|
||||
),
|
||||
detail=True,
|
||||
)
|
||||
|
||||
# --- September 2025 flood: the deployed configuration (trained <= 2024) ---
|
||||
X25, reg25, clf25 = fit_backtest_model(df, "2024-12-31", use_dam=args.dam)
|
||||
obs25, fc25, flood25, alert25 = event_series(
|
||||
df, X25, reg25, clf25, "2025-09-22", "2025-10-02 12:00")
|
||||
pred_at_alert = float(fc25.loc[alert25:, "pred_max"].iloc[:24].max()) if alert25 is not None else None
|
||||
df, X25, reg25, clf25, "2025-09-22", "2025-10-02 12:00"
|
||||
)
|
||||
pred_at_alert = (
|
||||
float(fc25.loc[alert25:, "pred_max"].iloc[:24].max())
|
||||
if alert25 is not None
|
||||
else None
|
||||
)
|
||||
note = f"peak {float(obs25.max()):.2f} m" + (
|
||||
f" (predicted {pred_at_alert:.2f} m)" if pred_at_alert is not None else "")
|
||||
render(obs25, fc25, flood25, alert25,
|
||||
os.path.join(args.out_dir, "backtest-2025-p1.png"),
|
||||
title="The September 2025 flood — as forecast by the deployed configuration",
|
||||
subtitle="model trained only on data through 2024; this event was never seen in training",
|
||||
detail=True, peak_note=note)
|
||||
f" (predicted {pred_at_alert:.2f} m)" if pred_at_alert is not None else ""
|
||||
)
|
||||
render(
|
||||
obs25,
|
||||
fc25,
|
||||
flood25,
|
||||
alert25,
|
||||
os.path.join(args.out_dir, "backtest-2025-p1.png"),
|
||||
title="The September 2025 flood — as forecast by the deployed configuration",
|
||||
subtitle="model trained only on data through 2024; this event was never seen in training",
|
||||
detail=True,
|
||||
peak_note=note,
|
||||
)
|
||||
|
||||
print(f"2024: flooding began {flood_start}, first alert {first_alert}")
|
||||
print(f"2025: flooding began {flood25}, first alert {alert25}")
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Build src/static/flood-marks-2024.json from CMU Water Center's flood marks.
|
||||
|
||||
After the October 2024 flood (P.1 peaked at 5.30 m on 5 October), CMU Water
|
||||
Center marked the high-water line at about 5,000 places in Chiang Mai and
|
||||
Lamphun (CMFlood, "Flood marks 2024"): the depth above the road surface, in
|
||||
cm. This keeps position and depth only, in a compact array form:
|
||||
|
||||
uv run python scripts/build_flood_marks.py # fetch from CMU
|
||||
uv run python scripts/build_flood_marks.py --geojson F # or a saved copy
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parents[1]
|
||||
OUT = REPO / "src" / "static" / "flood-marks-2024.json"
|
||||
URL = "https://watercenter.cmu.ac.th/data/New_floodMark_2025_fixed_like_pole.geojson"
|
||||
SOURCE = (
|
||||
"CMU Water Center (ศูนย์วิชาการสนับสนุนการบริหารจัดการน้ำ มหาวิทยาลัยเชียงใหม่), "
|
||||
"flood marks of October 2024, "
|
||||
"https://watercenter.cmu.ac.th/chiangmai/cmflood/warning/floodmark"
|
||||
)
|
||||
|
||||
|
||||
def load(path: Path | None) -> dict:
|
||||
if path:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
req = urllib.request.Request(URL, headers={"User-Agent": "Mozilla/5.0"})
|
||||
with urllib.request.urlopen(req, timeout=120) as r:
|
||||
return json.loads(r.read().decode("utf-8"))
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument("--geojson", type=Path, help="read CMU's GeoJSON from here")
|
||||
args = ap.parse_args()
|
||||
|
||||
marks, skipped = [], 0
|
||||
for f in load(args.geojson)["features"]:
|
||||
p = f.get("properties") or {}
|
||||
lon, lat, depth = p.get("longitude"), p.get("latitude"), p.get("water_level")
|
||||
if lon is None or lat is None or depth is None:
|
||||
skipped += 1
|
||||
continue
|
||||
# 18-19.5 N, 98.5-99.5 E: a swapped or empty coordinate is skipped
|
||||
if not (18.0 < lat < 19.5 and 98.5 < lon < 99.5) or depth <= 0:
|
||||
skipped += 1
|
||||
continue
|
||||
marks.append([round(lon, 5), round(lat, 5), round(depth)])
|
||||
marks.sort(key=lambda m: (m[1], m[0]))
|
||||
doc = {
|
||||
"source": SOURCE,
|
||||
"p1_level": 5.30,
|
||||
"date": "2024-10-05",
|
||||
"fields": ["lon", "lat", "depth_cm_above_road"],
|
||||
"marks": marks,
|
||||
}
|
||||
OUT.write_text(
|
||||
json.dumps(doc, ensure_ascii=False, separators=(",", ":")) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(
|
||||
f"wrote {OUT.relative_to(REPO)}: {len(marks)} marks, {skipped} skipped, "
|
||||
f"{OUT.stat().st_size / 1024:.0f} KB"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,151 @@
|
||||
# /// script
|
||||
# requires-python = ">=3.11"
|
||||
# dependencies = ["shapely>=2.0"]
|
||||
# ///
|
||||
"""Build src/static/flood-zones-p1.geojson from CMU Water Center's flood map.
|
||||
|
||||
CMU's Chiang Mai flood hazard map (watercenter.cmu.ac.th, CMFlood, built from
|
||||
the October 2024 flood) publishes the flooded area at five P.1 levels as KML:
|
||||
/data/KML5Layer/L1.kml (P.1 = 4.30 m) to L5.kml (5.30 m, the 2024 record).
|
||||
Each file is the whole extent at that level. The dashboard wants one ring per
|
||||
stage instead: zone k is what floods first when P.1 reaches stage k's level
|
||||
(the extent at level k minus everything flooded at a lower level).
|
||||
|
||||
uv run scripts/build_flood_zones.py # fetch the KML from CMU
|
||||
uv run scripts/build_flood_zones.py --kml-dir D # or read L1..L5.kml from D
|
||||
|
||||
The levels must match P1_FLOOD_STAGES in src/ml/features.py.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
import shapely
|
||||
from shapely import make_valid
|
||||
from shapely.geometry import MultiPolygon, Polygon, mapping
|
||||
from shapely.ops import unary_union
|
||||
|
||||
REPO = Path(__file__).resolve().parents[1]
|
||||
OUT = REPO / "src" / "static" / "flood-zones-p1.geojson"
|
||||
BASE = "https://watercenter.cmu.ac.th/data/KML5Layer"
|
||||
LEVELS = [4.30, 4.50, 4.70, 5.00, 5.30] # L1..L5, P.1 gauge level in m
|
||||
# ~3 m at Chiang Mai: invisible at the zoom levels the map allows, and it
|
||||
# keeps the file small. Slivers below MIN_AREA (about 300 m²) come from the
|
||||
# layers' borders not lining up exactly and are dropped.
|
||||
SIMPLIFY_DEG = 3e-5
|
||||
MIN_AREA_DEG2 = 2.5e-8
|
||||
GRID = 1e-5 # coordinates snapped to 5 decimals (~1 m) with valid geometry
|
||||
SOURCE = (
|
||||
"CMU Water Center (ศูนย์วิชาการสนับสนุนการบริหารจัดการน้ำ มหาวิทยาลัยเชียงใหม่), "
|
||||
"Chiang Mai flood hazard map from the October 2024 flood, "
|
||||
"https://watercenter.cmu.ac.th/chiangmai/cmflood/warning/floodmap"
|
||||
)
|
||||
|
||||
|
||||
def fetch(i: int, kml_dir: Path | None) -> str:
|
||||
if kml_dir:
|
||||
return (kml_dir / f"L{i}.kml").read_text(encoding="utf-8")
|
||||
req = urllib.request.Request(
|
||||
f"{BASE}/L{i}.kml", headers={"User-Agent": "Mozilla/5.0"}
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=60) as r:
|
||||
return r.read().decode("utf-8")
|
||||
|
||||
|
||||
def ring(text: str) -> list:
|
||||
pts = []
|
||||
for tok in text.split():
|
||||
x, y, *_ = tok.split(",")
|
||||
pts.append((float(x), float(y)))
|
||||
return pts
|
||||
|
||||
|
||||
def kml_extent(text: str):
|
||||
polys = []
|
||||
for body in re.findall(r"<Polygon>(.*?)</Polygon>", text, re.S):
|
||||
outer = re.search(
|
||||
r"<outerBoundaryIs>.*?<coordinates>(.*?)</coordinates>", body, re.S
|
||||
)
|
||||
inner = re.findall(
|
||||
r"<innerBoundaryIs>.*?<coordinates>(.*?)</coordinates>", body, re.S
|
||||
)
|
||||
polys.append(
|
||||
make_valid(Polygon(ring(outer.group(1)), [ring(s) for s in inner]))
|
||||
)
|
||||
return unary_union(polys)
|
||||
|
||||
|
||||
def polygons(geom) -> list:
|
||||
if isinstance(geom, Polygon):
|
||||
return [geom]
|
||||
return [
|
||||
g for g in getattr(geom, "geoms", []) if isinstance(g, (Polygon, MultiPolygon))
|
||||
]
|
||||
|
||||
|
||||
def clean(geom):
|
||||
parts = []
|
||||
for g in polygons(geom):
|
||||
for p in polygons(g):
|
||||
if p.area >= MIN_AREA_DEG2:
|
||||
parts.append(p)
|
||||
return unary_union(parts)
|
||||
|
||||
|
||||
def rounded(coords):
|
||||
# the geometry is already on the grid; this only drops float noise
|
||||
if isinstance(coords, (list, tuple)) and coords and isinstance(coords[0], float):
|
||||
return [round(coords[0], 5), round(coords[1], 5)]
|
||||
return [rounded(c) for c in coords]
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument("--kml-dir", type=Path, help="read L1..L5.kml from here")
|
||||
args = ap.parse_args()
|
||||
|
||||
flooded = None
|
||||
features = []
|
||||
for i, level in enumerate(LEVELS, 1):
|
||||
extent = kml_extent(fetch(i, args.kml_dir))
|
||||
extent = shapely.set_precision(
|
||||
make_valid(extent.simplify(SIMPLIFY_DEG, preserve_topology=True)), GRID
|
||||
)
|
||||
if flooded is None:
|
||||
new, flooded = extent, extent
|
||||
else:
|
||||
new = shapely.difference(extent, flooded, grid_size=GRID)
|
||||
flooded = shapely.union(flooded, extent, grid_size=GRID)
|
||||
zone = shapely.set_precision(clean(new), GRID)
|
||||
assert zone.is_valid, f"zone {i} is not a valid geometry"
|
||||
geo = mapping(zone)
|
||||
features.append(
|
||||
{
|
||||
"type": "Feature",
|
||||
"properties": {"zone": i, "level": level},
|
||||
"geometry": {
|
||||
"type": geo["type"],
|
||||
"coordinates": rounded(geo["coordinates"]),
|
||||
},
|
||||
}
|
||||
)
|
||||
print(f"zone {i} (P.1 {level:.2f} m): {len(polygons(zone))} polygons")
|
||||
doc = {
|
||||
"type": "FeatureCollection",
|
||||
"source": SOURCE,
|
||||
"note": "Zone k floods first when P.1 reaches its level; zones are disjoint.",
|
||||
# highest zone first, so the zones that flood first are drawn on top
|
||||
"features": features[::-1],
|
||||
}
|
||||
OUT.write_text(
|
||||
json.dumps(doc, ensure_ascii=False, separators=(",", ":")) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(f"wrote {OUT.relative_to(REPO)}: {OUT.stat().st_size / 1024:.0f} KB")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,150 +0,0 @@
|
||||
"""Live parity check: Python collectors vs the Rust ping-collector.
|
||||
|
||||
Runs one collection of every source with the Python code into PY_DB and with
|
||||
`ping-collector` into RS_DB (both should start empty), back to back against the
|
||||
real upstream APIs, then diffs all nine collector tables row by row (ids and
|
||||
created/updated timestamps ignored). Exit status 1 on any difference.
|
||||
|
||||
uv run python scripts/collector_parity.py \
|
||||
--py-db postgresql://postgres:ping@localhost:55432/py \
|
||||
--rs-db postgresql://postgres:ping@localhost:55432/rs \
|
||||
--collector rust/target/debug/ping-collector
|
||||
|
||||
Upstream snapshots can move between the two runs (a new HII hour, a revised
|
||||
RID value); rerun before treating a small difference as a bug.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(REPO))
|
||||
|
||||
TABLES = {
|
||||
"stations": (
|
||||
"id",
|
||||
"id, station_code, thai_name, english_name, latitude, longitude, geohash",
|
||||
),
|
||||
"water_measurements": (
|
||||
"timestamp, station_id",
|
||||
"timestamp, station_id, water_level, discharge, discharge_percent, status",
|
||||
),
|
||||
"hii_rain_stations": (
|
||||
"id",
|
||||
"id, oldcode, name_th, name_en, latitude, longitude, sub_basin_id, agency",
|
||||
),
|
||||
"hii_rainfall": (
|
||||
"station_id, timestamp",
|
||||
"station_id, timestamp, rain_1h, rain_24h",
|
||||
),
|
||||
"hii_wl_stations": (
|
||||
"id",
|
||||
"id, oldcode, rid_code, name_th, name_en, latitude, longitude, agency, "
|
||||
"river_name, offset_msl, ground_level_msl, min_bank_msl, "
|
||||
"critical_level_msl, critical_level_m, qmax, is_key_station",
|
||||
),
|
||||
"hii_waterlevel": (
|
||||
"station_id, timestamp",
|
||||
"station_id, timestamp, wl_msl, wl_m, discharge, flow_rate, "
|
||||
"storage_percent, situation_level, diff_wl_bank",
|
||||
),
|
||||
"rid_dams": (
|
||||
"dam_id",
|
||||
"dam_id, region, name_th, latitude, longitude, capacity_max_mcm, "
|
||||
"capacity_normal_mcm",
|
||||
),
|
||||
"rid_reservoir_daily": (
|
||||
"dam_id, date",
|
||||
"dam_id, date, storage_mcm, storage_pct, inflow_mcm, outflow_mcm, level_msl",
|
||||
),
|
||||
"openmeteo_rain": (
|
||||
"timestamp",
|
||||
"timestamp, chiang_dao, mae_taeng, mae_ngat, mae_rim, chiang_mai, "
|
||||
"catchment_mean",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def run_python(db_url: str, gap_days: int):
|
||||
os.environ.update(
|
||||
DB_TYPE="postgresql", POSTGRES_CONNECTION_STRING=db_url, LOG_LEVEL="WARNING"
|
||||
)
|
||||
from sqlalchemy import create_engine
|
||||
|
||||
from src.config import Config
|
||||
from src.hii_collector import HiiCollector
|
||||
from src.ml import rain
|
||||
from src.rid_reservoir import RidReservoirCollector
|
||||
from src.water_scraper_v3 import EnhancedWaterMonitorScraper
|
||||
|
||||
cfg = Config.get_database_config()
|
||||
scraper = EnhancedWaterMonitorScraper(cfg)
|
||||
scraper.fill_data_gaps(gap_days)
|
||||
scraper.run_scraping_cycle()
|
||||
HiiCollector(cfg).run_cycle()
|
||||
RidReservoirCollector(cfg).run_cycle()
|
||||
rain.save_to_db(rain.fetch_forecast(), create_engine(db_url), "postgresql")
|
||||
|
||||
|
||||
def run_rust(db_url: str, collector: str, gap_days: int):
|
||||
env = {**os.environ, "POSTGRES_CONNECTION_STRING": db_url, "DB_TYPE": "postgresql"}
|
||||
for args in (["fill-gaps", str(gap_days)], ["once"]):
|
||||
subprocess.run([collector, *args], env=env, check=True)
|
||||
|
||||
|
||||
def table_rows(db_url: str, table: str, key: str, cols: str) -> dict:
|
||||
import psycopg2
|
||||
|
||||
with psycopg2.connect(db_url) as conn:
|
||||
cur = conn.cursor()
|
||||
cur.execute(f"SELECT {cols} FROM {table} ORDER BY {key}")
|
||||
nkey = len(key.split(","))
|
||||
return {row[:nkey]: row for row in cur.fetchall()}
|
||||
|
||||
|
||||
def diff(py_db: str, rs_db: str) -> bool:
|
||||
identical = True
|
||||
for table, (key, cols) in TABLES.items():
|
||||
a = table_rows(py_db, table, key, cols)
|
||||
b = table_rows(rs_db, table, key, cols)
|
||||
only_py, only_rs = sorted(set(a) - set(b)), sorted(set(b) - set(a))
|
||||
differ = sorted(k for k in set(a) & set(b) if a[k] != b[k])
|
||||
same = not (only_py or only_rs or differ)
|
||||
identical &= same
|
||||
print(
|
||||
f"{table:<20} py {len(a):>6} rs {len(b):>6} only-py {len(only_py):>4} "
|
||||
f"only-rs {len(only_rs):>4} differing {len(differ):>4} "
|
||||
f"{'identical' if same else 'DIFFERENT'}"
|
||||
)
|
||||
for k in only_py[:3]:
|
||||
print(" only py:", a[k])
|
||||
for k in only_rs[:3]:
|
||||
print(" only rs:", b[k])
|
||||
for k in differ[:3]:
|
||||
print(" py:", a[k], "\n rs:", b[k])
|
||||
return identical
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.add_argument("--py-db", required=True)
|
||||
parser.add_argument("--rs-db", required=True)
|
||||
parser.add_argument(
|
||||
"--collector", default=str(REPO / "rust/target/debug/ping-collector")
|
||||
)
|
||||
parser.add_argument("--gap-days", type=int, default=2)
|
||||
parser.add_argument("--diff-only", action="store_true")
|
||||
args = parser.parse_args()
|
||||
if not args.diff_only:
|
||||
run_python(args.py_db, args.gap_days)
|
||||
run_rust(args.rs_db, args.collector, args.gap_days)
|
||||
ok = diff(args.py_db, args.rs_db)
|
||||
print("ALL IDENTICAL" if ok else "differences found")
|
||||
sys.exit(0 if ok else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+39
-7
@@ -1,9 +1,15 @@
|
||||
"""Serve the working-copy dashboard locally with API calls proxied to the
|
||||
live server, so browser-side changes can be checked against real data
|
||||
before deploy. Usage: python scripts/dev_proxy.py [port]"""
|
||||
before deploy. Usage: python scripts/dev_proxy.py [port]
|
||||
|
||||
/ is src/static/dashboard.html, /static/... any file in src/static (with HTTP
|
||||
range requests, which the map's PMTiles basemap needs), everything else the
|
||||
live server."""
|
||||
|
||||
import http.server
|
||||
import mimetypes
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
@@ -25,24 +31,50 @@ class Handler(http.server.BaseHTTPRequestHandler):
|
||||
self._send(200, "application/json", Path(file).read_bytes())
|
||||
return
|
||||
if self.path.startswith("/static/"):
|
||||
f = STATIC / self.path[len("/static/"):].split("?")[0]
|
||||
f = STATIC / self.path[len("/static/") :].split("?")[0]
|
||||
if f.is_file():
|
||||
ctype = "application/json" if f.suffix in (".json", ".geojson") else "application/octet-stream"
|
||||
self._send(200, ctype, f.read_bytes())
|
||||
ctype = (
|
||||
"application/json"
|
||||
if f.suffix in (".json", ".geojson")
|
||||
else mimetypes.guess_type(f.name)[0] or "application/octet-stream"
|
||||
)
|
||||
if ctype.startswith("text/"):
|
||||
ctype += "; charset=utf-8"
|
||||
body = f.read_bytes()
|
||||
# Byte ranges, like the server's ServeDir: PMTiles reads the
|
||||
# basemap in pieces
|
||||
m = re.fullmatch(r"bytes=(\d+)-(\d*)", self.headers.get("Range", ""))
|
||||
if m:
|
||||
start = int(m.group(1))
|
||||
end = min(int(m.group(2) or len(body) - 1), len(body) - 1)
|
||||
extra = {"Content-Range": f"bytes {start}-{end}/{len(body)}"}
|
||||
self._send(206, ctype, body[start : end + 1], extra)
|
||||
return
|
||||
self._send(200, ctype, body)
|
||||
return
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
UPSTREAM + self.path,
|
||||
headers={"User-Agent": "Mozilla/5.0 (dev_proxy; +https://buildfor.life)", "Accept": "application/json"},
|
||||
headers={
|
||||
"User-Agent": "Mozilla/5.0 (dev_proxy; +https://buildfor.life)",
|
||||
"Accept": "application/json",
|
||||
},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=60) as r:
|
||||
self._send(r.status, r.headers.get("Content-Type", "application/json"), r.read())
|
||||
self._send(
|
||||
r.status,
|
||||
r.headers.get("Content-Type", "application/json"),
|
||||
r.read(),
|
||||
)
|
||||
except urllib.error.HTTPError as e:
|
||||
self._send(e.code, "application/json", e.read())
|
||||
|
||||
def _send(self, code, ctype, body):
|
||||
def _send(self, code, ctype, body, extra=None):
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", ctype)
|
||||
self.send_header("Accept-Ranges", "bytes")
|
||||
for name, value in (extra or {}).items():
|
||||
self.send_header(name, value)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.send_header("Cache-Control", "no-store")
|
||||
self.end_headers()
|
||||
|
||||
@@ -1,132 +0,0 @@
|
||||
"""Drive the production notify path in-process: startup init -> seeded readings
|
||||
-> forecast cache -> _notify_transitions -> sqlite state -> real ntfy."""
|
||||
|
||||
import asyncio
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
import requests
|
||||
|
||||
os.environ.update(
|
||||
DB_TYPE="sqlite",
|
||||
WATER_DB_PATH=os.path.join(os.environ["LOCALAPPDATA"], "Temp", "smoke3.db"),
|
||||
NTFY_SERVER="http://127.0.0.1:2586",
|
||||
NTFY_TOKEN=os.environ.get("NTFY_TOKEN", ""),
|
||||
NTFY_TOPIC_PREFIX="ping",
|
||||
)
|
||||
for f in ("smoke3.db",):
|
||||
p = os.path.join(os.environ["LOCALAPPDATA"], "Temp", f)
|
||||
if os.path.exists(p):
|
||||
os.remove(p)
|
||||
|
||||
from src import web_api # noqa: E402
|
||||
from src.config import Config # noqa: E402
|
||||
|
||||
assert Config.NTFY_SERVER
|
||||
|
||||
|
||||
async def main():
|
||||
# what the lifespan does at startup, minus the scheduler
|
||||
from src import notify as notify_mod
|
||||
from src.forecast_history import ForecastHistoryStore
|
||||
from src.water_scraper_v3 import EnhancedWaterMonitorScraper
|
||||
|
||||
db_config = Config.get_database_config()
|
||||
web_api.app_state["scraper"] = EnhancedWaterMonitorScraper(db_config)
|
||||
store = ForecastHistoryStore(db_config["connection_string"], db_config["type"])
|
||||
store.connect()
|
||||
web_api.app_state["forecast_store"] = store
|
||||
state = notify_mod.NotificationState(store.engine, store.db_type)
|
||||
pub = notify_mod.NtfyPublisher(
|
||||
Config.NTFY_SERVER, prefix=Config.NTFY_TOPIC_PREFIX, token=Config.NTFY_TOKEN
|
||||
)
|
||||
web_api.app_state["notify"] = (pub, state)
|
||||
|
||||
scraper = web_api.app_state["scraper"]
|
||||
now = datetime.datetime.now().replace(minute=0, second=0, microsecond=0)
|
||||
|
||||
def seed(level_p1, level_p103, ts):
|
||||
rows = [
|
||||
{
|
||||
"station_code": "P.1",
|
||||
"station_id": 1,
|
||||
"timestamp": ts,
|
||||
"water_level": level_p1,
|
||||
"discharge": 400.0,
|
||||
"station_name_en": "Nawarat Bridge",
|
||||
"station_name_th": "สะพานนวรัฐ",
|
||||
"discharge_percent": 30.0,
|
||||
"status": "active",
|
||||
},
|
||||
{
|
||||
"station_code": "P.103",
|
||||
"station_id": 2,
|
||||
"timestamp": ts,
|
||||
"water_level": level_p103,
|
||||
"discharge": 300.0,
|
||||
"station_name_en": "Ring Road 3",
|
||||
"station_name_th": "วงแหวน 3",
|
||||
"discharge_percent": 20.0,
|
||||
"status": "active",
|
||||
},
|
||||
]
|
||||
scraper.db_adapter.save_measurements(rows)
|
||||
|
||||
def forecast(p):
|
||||
with web_api.FORECAST_CACHE_LOCK:
|
||||
web_api.FORECAST_CACHE["all"] = (
|
||||
0,
|
||||
[
|
||||
{
|
||||
"station_code": "P.1",
|
||||
"horizon_hours": 24,
|
||||
"p_warning": p,
|
||||
"predicted_max_level": 3.9,
|
||||
"source": "model",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
def poll(topic):
|
||||
out = []
|
||||
for line in (
|
||||
requests.get(f"{Config.NTFY_SERVER}/{topic}/json?poll=1", timeout=5)
|
||||
.text.strip()
|
||||
.splitlines()
|
||||
):
|
||||
m = json.loads(line)
|
||||
if m.get("event") == "message":
|
||||
out.append(m.get("title") or m.get("message", "")[:40])
|
||||
return out
|
||||
|
||||
# cycle 1: quiet
|
||||
seed(1.6, 3.2, now - datetime.timedelta(hours=2))
|
||||
forecast(0.02)
|
||||
await web_api._notify_transitions()
|
||||
# cycle 2: P.1 crosses warning, model outlook on
|
||||
seed(3.75, 3.3, now - datetime.timedelta(hours=1))
|
||||
forecast(0.7)
|
||||
await web_api._notify_transitions()
|
||||
# cycle 3: same state -> silence
|
||||
seed(3.80, 3.3, now)
|
||||
forecast(0.65)
|
||||
await web_api._notify_transitions()
|
||||
|
||||
print("ping-p1-warning:", poll("ping-p1-warning"))
|
||||
print("ping-warning: ", poll("ping-warning"))
|
||||
print("ping-p1-outlook:", poll("ping-p1-outlook"))
|
||||
print("ping-p103-warning:", poll("ping-p103-warning"))
|
||||
from sqlalchemy import text
|
||||
|
||||
with store.engine.connect() as c:
|
||||
print(
|
||||
"state table:",
|
||||
c.execute(
|
||||
text("SELECT key, state, value FROM notification_state ORDER BY key")
|
||||
).fetchall(),
|
||||
)
|
||||
|
||||
|
||||
asyncio.run(main())
|
||||
@@ -1,8 +1,10 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare the Python forecast with ping-collector's on the live database.
|
||||
"""Compare ping-collector's forecast with the Python reference on the live database.
|
||||
|
||||
Run before setting FORECAST_IN_COLLECTOR=true, and after any change to src/ml
|
||||
or rust/forecast. On the server, as the service user from the app dir:
|
||||
src/ml/predict.py is the reference implementation the Rust forecaster
|
||||
(rust/forecast) was ported from. Run after any change to src/ml or
|
||||
rust/forecast; scripts/install.sh runs it on every install. On the server, as
|
||||
the service user from the app dir:
|
||||
|
||||
sudo -u water-monitor .venv/bin/python scripts/forecast_parity.py
|
||||
|
||||
|
||||
+190
-97
@@ -1,127 +1,220 @@
|
||||
#!/usr/bin/env bash
|
||||
# Install or update the Ping River Monitor on a systemd host.
|
||||
#
|
||||
# Install the Thailand Water Level Monitor as a hardened systemd service.
|
||||
# sudo bash scripts/install.sh
|
||||
#
|
||||
# Creates a dedicated system user, deploys the code to /opt, builds a uv-managed
|
||||
# virtualenv, installs the systemd unit, and enables the service. Idempotent:
|
||||
# safe to re-run to update an existing install.
|
||||
#
|
||||
# Usage (as root, from a checkout of the repo):
|
||||
# sudo bash scripts/install.sh
|
||||
#
|
||||
# Override defaults via environment variables:
|
||||
# APP_DIR=/opt/thailand-water-monitor SERVICE_USER=water-monitor sudo -E bash scripts/install.sh
|
||||
# What runs:
|
||||
# ping-collector.service the service (rust/collector): RID/HII/dam/rain
|
||||
# collection, forecast, ntfy + Matrix
|
||||
# notifications, website + API on WEB_LISTEN
|
||||
# water-monitor-retrain.timer monthly model retrain (Python, scripts/retrain.sh)
|
||||
#
|
||||
# Re-run after every update (git pull + build). Steps, all idempotent:
|
||||
# 1. system user/group (water-monitor)
|
||||
# 2. code: run from a checkout other than APP_DIR, it is rsynced there (the
|
||||
# trained models, .env, stations.json and bin/ are kept) and handed to
|
||||
# the service user. Production runs in place (APP_DIR is the checkout):
|
||||
# then nothing is copied and no ownership changes.
|
||||
# 3. the binary: BIN=/path/to/ping-collector (e.g. the CI artifact
|
||||
# "ping-collector-linux-x86_64") or rust/target/release/ping-collector,
|
||||
# built beforehand as a normal user:
|
||||
# cargo build --release --locked --manifest-path rust/Cargo.toml
|
||||
# (Rust >= 1.88 from rustup, plus pkg-config and libssl-dev)
|
||||
# 4. the Python training environment (.venv, `uv sync --frozen`), as the
|
||||
# owner of .venv (in place; else the user running sudo) or as root for a
|
||||
# copy. uv is looked up on PATH and in that user's ~/.local/bin and
|
||||
# ~/.cargo/bin (sudo and root shells often lack them); UV=/path/to/uv
|
||||
# overrides, SKIP_UV=1 skips the step
|
||||
# 5. .env: created from .env.example on a fresh install (edit, then re-run);
|
||||
# WEB_LISTEN and FORECAST_IN_COLLECTOR default to the production values
|
||||
# when absent
|
||||
# 6. systemd units; the retired Python web app (water-monitor.service) and
|
||||
# the old `run.py --alert-check` Matrix cron job are removed
|
||||
# 7. checks (SKIP_CHECKS=1 skips them): a dry run of every source against
|
||||
# the database, and the forecast compared with the Python reference on
|
||||
# the live data (scripts/forecast_parity.py)
|
||||
# 8. (re)start, then wait for the website to answer
|
||||
set -euo pipefail
|
||||
|
||||
APP_DIR="${APP_DIR:-/opt/thailand-water-monitor}"
|
||||
SERVICE_USER="${SERVICE_USER:-water-monitor}"
|
||||
SERVICE_USER="${SERVICE_USER:-water-monitor}" # as in the units
|
||||
SERVICE_GROUP="${SERVICE_GROUP:-${SERVICE_USER}}"
|
||||
SERVICE_NAME="water-monitor.service"
|
||||
RETRAIN_NAME="water-monitor-retrain"
|
||||
|
||||
# Resolve the repo root (parent of this scripts/ directory).
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
ENV_FILE="${APP_DIR}/.env"
|
||||
PYTHON="${APP_DIR}/.venv/bin/python"
|
||||
REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
RETRAIN="water-monitor-retrain"
|
||||
|
||||
log() { printf '\033[1;32m==>\033[0m %s\n' "$*"; }
|
||||
warn() { printf '\033[1;33m[warn]\033[0m %s\n' "$*"; }
|
||||
warn() { printf '\033[1;33m[warn]\033[0m %s\n' "$*" >&2; }
|
||||
die() { printf '\033[1;31m[error]\033[0m %s\n' "$*" >&2; exit 1; }
|
||||
|
||||
[ "$(id -u)" -eq 0 ] || die "This script must be run as root (use sudo)."
|
||||
[ "$(id -u)" -eq 0 ] || die "run as root (sudo)"
|
||||
|
||||
# 1. Dedicated system user/group (no login, no home) --------------------------
|
||||
if ! getent group "${SERVICE_GROUP}" >/dev/null; then
|
||||
log "Creating group ${SERVICE_GROUP}"
|
||||
groupadd --system "${SERVICE_GROUP}"
|
||||
fi
|
||||
if ! id "${SERVICE_USER}" >/dev/null 2>&1; then
|
||||
log "Creating system user ${SERVICE_USER}"
|
||||
set_default() { # key value: append when the key is not set at all
|
||||
grep -q "^$1=" "$ENV_FILE" || printf '%s=%s\n' "$1" "$2" >>"$ENV_FILE"
|
||||
}
|
||||
|
||||
# A command the way the services run it: same user, working directory and
|
||||
# EnvironmentFile parsing (systemd's).
|
||||
as_service() {
|
||||
if systemd-run --version >/dev/null 2>&1; then
|
||||
systemd-run --quiet --pipe --wait --collect \
|
||||
-p User="$SERVICE_USER" -p Group="$SERVICE_GROUP" \
|
||||
-p WorkingDirectory="$APP_DIR" -p EnvironmentFile="$ENV_FILE" "$@"
|
||||
else
|
||||
(cd "$APP_DIR" && runuser -u "$SERVICE_USER" -- "$@")
|
||||
fi
|
||||
}
|
||||
|
||||
wait_http() { # url seconds
|
||||
local i
|
||||
for i in $(seq "$2"); do
|
||||
curl -fs -o /dev/null --max-time 5 "$1" 2>/dev/null && return 0
|
||||
sleep 1
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# 1. user ---------------------------------------------------------------------
|
||||
getent group "$SERVICE_GROUP" >/dev/null || groupadd --system "$SERVICE_GROUP"
|
||||
if ! id "$SERVICE_USER" >/dev/null 2>&1; then
|
||||
log "Creating system user $SERVICE_USER"
|
||||
useradd --system --no-create-home --shell /usr/sbin/nologin \
|
||||
--gid "${SERVICE_GROUP}" "${SERVICE_USER}"
|
||||
--gid "$SERVICE_GROUP" "$SERVICE_USER"
|
||||
fi
|
||||
|
||||
# 2. Deploy code to APP_DIR ----------------------------------------------------
|
||||
log "Deploying code to ${APP_DIR}"
|
||||
mkdir -p "${APP_DIR}"
|
||||
if command -v rsync >/dev/null 2>&1; then
|
||||
# 2. code ---------------------------------------------------------------------
|
||||
in_place=0
|
||||
[ -d "$APP_DIR" ] && [ "$(cd "$APP_DIR" && pwd -P)" = "$(cd "$REPO_DIR" && pwd -P)" ] && in_place=1
|
||||
if [ "$in_place" = 0 ]; then
|
||||
command -v rsync >/dev/null || die "rsync is needed to copy $REPO_DIR to $APP_DIR"
|
||||
log "Copying $REPO_DIR to $APP_DIR"
|
||||
mkdir -p "$APP_DIR"
|
||||
rsync -a --delete \
|
||||
--exclude '.git' --exclude '.venv' --exclude 'venv' \
|
||||
--exclude '__pycache__' --exclude '*.pyc' \
|
||||
--exclude '*.db' --exclude '.env' --exclude 'stations.json' \
|
||||
"${REPO_DIR}/" "${APP_DIR}/"
|
||||
else
|
||||
warn "rsync not found; falling back to cp (will not prune deleted files)"
|
||||
cp -r "${REPO_DIR}/." "${APP_DIR}/"
|
||||
--exclude '.git' --exclude '.venv' --exclude '__pycache__' --exclude '/rust/target' \
|
||||
--exclude '/.env' --exclude '/stations.json' --exclude '/bin' \
|
||||
--exclude '/collector-status.json' --exclude '/crontab-*.bak' \
|
||||
--exclude '/models/flood_*' --exclude '/models/metrics.json' \
|
||||
--exclude '/models/cache' --exclude '/models/.staging' --exclude '/models/.previous' \
|
||||
"$REPO_DIR/" "$APP_DIR/"
|
||||
fi
|
||||
|
||||
# 3. Build the uv-managed virtualenv ------------------------------------------
|
||||
# Prefer an already-installed uv. For stricter supply-chain control install uv
|
||||
# ahead of time via your distro / package manager; this script only fetches the
|
||||
# upstream installer (piped to a root shell) when AUTO_INSTALL_UV=1 is set, and
|
||||
# pins the version so the fetched script is reproducible.
|
||||
UV_VERSION="${UV_VERSION:-0.5.11}"
|
||||
if ! command -v uv >/dev/null 2>&1; then
|
||||
if [ "${AUTO_INSTALL_UV:-0}" = "1" ]; then
|
||||
warn "uv not found; installing pinned uv ${UV_VERSION} from astral.sh (runs as root)"
|
||||
curl -LsSf "https://astral.sh/uv/${UV_VERSION}/install.sh" \
|
||||
| env UV_INSTALL_DIR=/usr/local/bin sh
|
||||
else
|
||||
die "uv not found. Install it (e.g. your package manager, or 'pipx install uv'),
|
||||
or re-run with AUTO_INSTALL_UV=1 to fetch the pinned upstream installer."
|
||||
# 3. binary -------------------------------------------------------------------
|
||||
BIN="${BIN:-${REPO_DIR}/rust/target/release/ping-collector}"
|
||||
[ -x "$BIN" ] || die "no binary at $BIN: build it (see the header) or pass BIN=/path/to/ping-collector"
|
||||
log "Installing $BIN -> ${APP_DIR}/bin/ping-collector ($("$BIN" --version))"
|
||||
install -D -m 0755 -o root -g root "$BIN" "${APP_DIR}/bin/ping-collector"
|
||||
|
||||
# 4. Python training environment ------------------------------------------------
|
||||
find_uv() { # user -> path of a uv binary that user can run
|
||||
local home c
|
||||
home="$(getent passwd "$1" | cut -d: -f6)"
|
||||
for c in "${UV:-}" "$(command -v uv 2>/dev/null || true)" "${home}/.local/bin/uv" \
|
||||
"${home}/.cargo/bin/uv" /root/.local/bin/uv /usr/local/bin/uv /usr/bin/uv; do
|
||||
[ -n "$c" ] && [ -x "$c" ] && runuser -u "$1" -- test -x "$c" && {
|
||||
echo "$c"
|
||||
return 0
|
||||
}
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
if [ "${SKIP_UV:-0}" != 1 ]; then
|
||||
uv_user=root # copy mode: handed to the service user below
|
||||
if [ "$in_place" = 1 ]; then
|
||||
uv_user="${SUDO_USER:-root}"
|
||||
[ -d "${APP_DIR}/.venv" ] && uv_user="$(stat -c %U "${APP_DIR}/.venv")"
|
||||
fi
|
||||
fi
|
||||
UV="$(command -v uv)"
|
||||
|
||||
log "Syncing uv-managed virtualenv at ${APP_DIR}/.venv"
|
||||
cd "${APP_DIR}"
|
||||
# ONE environment: uv sync owns .venv/ (from pyproject.toml + uv.lock, so the
|
||||
# ML extras such as scikit-learn/joblib are present) and both systemd units
|
||||
# run its interpreter directly. Never create a second env by another name --
|
||||
# a stale 'venv/' once coexisted here and broke manual retrains with
|
||||
# ModuleNotFoundError while the service itself ran fine.
|
||||
"${UV}" sync --python 3.11 --frozen
|
||||
if [ -d "${APP_DIR}/venv" ]; then
|
||||
warn "Removing stale ${APP_DIR}/venv (superseded by .venv)"
|
||||
rm -rf "${APP_DIR}/venv"
|
||||
fi
|
||||
|
||||
# 4. Environment file ----------------------------------------------------------
|
||||
if [ ! -f "${APP_DIR}/.env" ]; then
|
||||
if [ -f "${REPO_DIR}/.env" ]; then
|
||||
log "Copying .env from checkout"
|
||||
cp "${REPO_DIR}/.env" "${APP_DIR}/.env"
|
||||
if uv_bin="$(find_uv "$uv_user")"; then
|
||||
log "Syncing the Python training environment (${APP_DIR}/.venv) as $uv_user with $uv_bin"
|
||||
(cd "$APP_DIR" && runuser -u "$uv_user" -- env HOME="$(getent passwd "$uv_user" | cut -d: -f6)" \
|
||||
"$uv_bin" sync --python 3.11 --frozen) ||
|
||||
warn "uv sync failed: the monthly retrain needs ${APP_DIR}/.venv (uv sync --python 3.11 --frozen as $uv_user)"
|
||||
else
|
||||
warn "No .env found. Copy .env.example to ${APP_DIR}/.env and fill in"
|
||||
warn "MATRIX_ACCESS_TOKEN / MATRIX_ROOM_ID and DB settings before starting."
|
||||
warn "uv not found for $uv_user (pass UV=/path/to/uv): ${APP_DIR}/.venv not synced"
|
||||
fi
|
||||
fi
|
||||
|
||||
# 5. Ownership and permissions -------------------------------------------------
|
||||
# Service user needs write access for logs / stations.json.
|
||||
log "Setting ownership to ${SERVICE_USER}:${SERVICE_GROUP}"
|
||||
chown -R "${SERVICE_USER}:${SERVICE_GROUP}" "${APP_DIR}"
|
||||
# Restrict traversal to root + the service user, and lock down the secrets file
|
||||
# (contains the Matrix token and DB credentials).
|
||||
chmod 0750 "${APP_DIR}"
|
||||
if [ -f "${APP_DIR}/.env" ]; then
|
||||
chmod 0600 "${APP_DIR}/.env"
|
||||
# 5. .env ---------------------------------------------------------------------
|
||||
fresh_env=0
|
||||
if [ ! -f "$ENV_FILE" ]; then
|
||||
cp "${APP_DIR}/.env.example" "$ENV_FILE"
|
||||
chmod 0600 "$ENV_FILE"
|
||||
fresh_env=1
|
||||
fi
|
||||
set_default WEB_LISTEN 0.0.0.0:8000
|
||||
set_default FORECAST_IN_COLLECTOR true
|
||||
|
||||
if [ "$in_place" = 0 ]; then
|
||||
# a copy belongs to the service (it writes models/, stations.json and the
|
||||
# status file); an in-place checkout keeps its owner (git pull, builds)
|
||||
chown -R "${SERVICE_USER}:${SERVICE_GROUP}" "$APP_DIR"
|
||||
chown root:root "${APP_DIR}/bin/ping-collector"
|
||||
chmod 0750 "$APP_DIR"
|
||||
fi
|
||||
|
||||
# 6. Install and enable the systemd units -------------------------------------
|
||||
# 6. units --------------------------------------------------------------------
|
||||
log "Installing systemd units"
|
||||
install -m 0644 "${SCRIPT_DIR}/${SERVICE_NAME}" "/etc/systemd/system/${SERVICE_NAME}"
|
||||
install -m 0644 "${SCRIPT_DIR}/${RETRAIN_NAME}.service" "/etc/systemd/system/${RETRAIN_NAME}.service"
|
||||
install -m 0644 "${SCRIPT_DIR}/${RETRAIN_NAME}.timer" "/etc/systemd/system/${RETRAIN_NAME}.timer"
|
||||
install -m 0644 "${APP_DIR}/scripts/ping-collector.service" /etc/systemd/system/ping-collector.service
|
||||
install -m 0644 "${APP_DIR}/scripts/${RETRAIN}.service" "/etc/systemd/system/${RETRAIN}.service"
|
||||
install -m 0644 "${APP_DIR}/scripts/${RETRAIN}.timer" "/etc/systemd/system/${RETRAIN}.timer"
|
||||
if [ -f /etc/systemd/system/water-monitor.service ]; then
|
||||
log "Removing the retired Python web app (water-monitor.service)"
|
||||
systemctl disable --now water-monitor.service 2>/dev/null || true
|
||||
rm -f /etc/systemd/system/water-monitor.service
|
||||
fi
|
||||
systemctl daemon-reload
|
||||
systemctl enable "${SERVICE_NAME}"
|
||||
systemctl enable --now "${RETRAIN_NAME}.timer"
|
||||
systemctl enable ping-collector.service
|
||||
systemctl enable --now "${RETRAIN}.timer"
|
||||
|
||||
log "Done."
|
||||
# The old Matrix cron job (`run.py --alert-check`, repeating an alert every
|
||||
# 15 minutes while a level stayed high): ping-collector sends Matrix now.
|
||||
for user in root "$SERVICE_USER"; do
|
||||
crontab -l -u "$user" 2>/dev/null | grep -v '^\s*#' | grep -q -- '--alert-check' || continue
|
||||
backup="${APP_DIR}/crontab-${user}.$(date +%Y%m%d%H%M%S).bak"
|
||||
crontab -l -u "$user" >"$backup"
|
||||
sed '/--alert-check/ s|^\([^#]\)|# retired: Matrix is sent by ping-collector # \1|' "$backup" |
|
||||
crontab -u "$user" -
|
||||
log "Commented out the old --alert-check cron job of $user (backup: $backup)"
|
||||
done
|
||||
if grep -rlqs -- '--alert-check' /etc/cron.d /etc/crontab 2>/dev/null; then
|
||||
warn "an --alert-check job is still in $(grep -rls -- '--alert-check' /etc/cron.d /etc/crontab | tr '\n' ' '): remove it"
|
||||
fi
|
||||
|
||||
if [ "$fresh_env" = 1 ]; then
|
||||
warn "created $ENV_FILE from .env.example: set the database (POSTGRES_*), NTFY_* and MATRIX_*, then re-run"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# 7. checks -------------------------------------------------------------------
|
||||
if [ "${SKIP_CHECKS:-0}" != 1 ]; then
|
||||
log "Dry run of every source against the configured database (writes nothing)"
|
||||
as_service "${APP_DIR}/bin/ping-collector" once --dry-run
|
||||
if compgen -G "${APP_DIR}/models/flood_*.joblib" >/dev/null && [ -x "$PYTHON" ]; then
|
||||
log "Exporting trained models for the forecaster (models/flood_*.json.gz)"
|
||||
as_service "$PYTHON" -m src.ml.export
|
||||
log "Forecast check: ping-collector against the Python reference, same data, moment and rain"
|
||||
as_service "$PYTHON" scripts/forecast_parity.py ||
|
||||
warn "the forecast check found differences (see above); the service runs regardless"
|
||||
else
|
||||
warn "no trained models or no ${PYTHON}: forecast check skipped (the forecast falls back to the heuristic)"
|
||||
fi
|
||||
fi
|
||||
|
||||
# 8. (re)start ----------------------------------------------------------------
|
||||
systemctl restart ping-collector.service
|
||||
listen="$(grep '^WEB_LISTEN=' "$ENV_FILE" | tail -1 | cut -d= -f2- | tr -d '"'"'"' ')"
|
||||
if [ -n "$listen" ]; then
|
||||
port="${listen##*:}"
|
||||
# readiness: a route that answers 200 whatever the data (/health may be 503)
|
||||
if wait_http "http://127.0.0.1:${port}/api/notifications" 60; then
|
||||
log "Website and API answer on :${port} ($(curl -s --max-time 5 "http://127.0.0.1:${port}/scraping/status" | grep -o '"collector":"[^"]*"'))"
|
||||
else
|
||||
warn "no answer on port ${port} yet: journalctl -u ping-collector"
|
||||
fi
|
||||
fi
|
||||
sleep 3
|
||||
systemctl --no-pager --lines=0 status ping-collector.service || true
|
||||
journalctl -u ping-collector -n 25 --no-pager || true
|
||||
echo
|
||||
echo "Next steps:"
|
||||
echo " sudo systemctl start ${SERVICE_NAME}"
|
||||
echo " systemctl status ${SERVICE_NAME}"
|
||||
echo " sudo journalctl -u ${SERVICE_NAME} -f"
|
||||
echo " systemctl list-timers ${RETRAIN_NAME}.timer # monthly flood-model retrain"
|
||||
echo " sudo systemctl start ${RETRAIN_NAME}.service # retrain now"
|
||||
log "Done. Logs: journalctl -u ping-collector -f; retrain: systemctl list-timers ${RETRAIN}.timer"
|
||||
@@ -1,286 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Install the Rust service (rust/collector, `ping-collector`) and move work
|
||||
# from the Python app to it, or back.
|
||||
#
|
||||
# sudo bash scripts/install_collector.sh # install binary + unit; checks only
|
||||
# sudo bash scripts/install_collector.sh --cutover # Rust collects (COLLECTION_ENABLED=false)
|
||||
# sudo bash scripts/install_collector.sh --cutover-forecast # Rust also forecasts and notifies:
|
||||
# # ntfy + Matrix (FORECAST_IN_COLLECTOR=true)
|
||||
# sudo bash scripts/install_collector.sh --check-api # compare the Rust website/API with the
|
||||
# # Python app's, change nothing
|
||||
# sudo bash scripts/install_collector.sh --cutover-api # Rust serves the website and API on
|
||||
# # port 8000 (WEB_LISTEN); the Python app stops
|
||||
# sudo bash scripts/install_collector.sh --rollback-api # the Python app serves the website again
|
||||
# sudo bash scripts/install_collector.sh --rollback-forecast # forecasts + notifications back to Python
|
||||
# sudo bash scripts/install_collector.sh --rollback # Python does everything again
|
||||
#
|
||||
# Every install also exports the trained models for the Rust forecaster and
|
||||
# compares its forecast with Python's on the live data
|
||||
# (scripts/forecast_parity.py); --cutover-forecast refuses to switch when they
|
||||
# differ. --cutover-api first runs `ping-collector serve` next to the Python
|
||||
# app and compares every route (scripts/api_parity.py); it refuses to switch
|
||||
# when they differ. FORCE=1 overrides either check.
|
||||
#
|
||||
# Once ping-collector sends the Matrix messages (FORECAST_IN_COLLECTOR=true
|
||||
# and MATRIX_* set), the old `run.py --alert-check` cron job is commented out
|
||||
# (backup next to the app); it repeated an alert on every run.
|
||||
#
|
||||
# The binary comes from BIN=/path/to/ping-collector (e.g. the "ping-collector-
|
||||
# linux-x86_64" CI artifact) or from rust/target/release/ping-collector, built
|
||||
# beforehand as a normal user:
|
||||
# cargo build --release --locked --manifest-path rust/Cargo.toml
|
||||
# (needs Rust >= 1.88 from rustup -- distro cargo is too old -- plus pkg-config
|
||||
# and libssl-dev; see rust/README.md).
|
||||
set -euo pipefail
|
||||
|
||||
APP_DIR="${APP_DIR:-/opt/thailand-water-monitor}"
|
||||
SERVICE_USER="${SERVICE_USER:-water-monitor}" # as in scripts/ping-collector.service
|
||||
ENV_FILE="${APP_DIR}/.env"
|
||||
STATUS_FILE="${APP_DIR}/collector-status.json"
|
||||
PYTHON="${APP_DIR}/.venv/bin/python"
|
||||
REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
WEB_ADDR="${WEB_ADDR:-0.0.0.0:8000}" # where the Python app listens today
|
||||
CHECK_PORT="${CHECK_PORT:-8001}" # the Rust side during --check-api
|
||||
MODE="${1:-install}"
|
||||
|
||||
log() { printf '\033[1;32m==>\033[0m %s\n' "$*"; }
|
||||
warn() { printf '\033[1;33m[warn]\033[0m %s\n' "$*" >&2; }
|
||||
die() { printf '\033[1;31m[error]\033[0m %s\n' "$*" >&2; exit 1; }
|
||||
|
||||
[ "$(id -u)" -eq 0 ] || die "run as root (sudo)"
|
||||
[ -f "$ENV_FILE" ] || die "$ENV_FILE not found"
|
||||
|
||||
set_env() {
|
||||
local key="$1" value="$2"
|
||||
if grep -q "^${key}=" "$ENV_FILE"; then
|
||||
sed -i "s|^${key}=.*|${key}=${value}|" "$ENV_FILE"
|
||||
else
|
||||
printf '%s=%s\n' "$key" "$value" >>"$ENV_FILE"
|
||||
fi
|
||||
}
|
||||
|
||||
# Run a command the way the services run: same user, working directory and
|
||||
# EnvironmentFile parsing (systemd's, not a .env library's).
|
||||
as_service() {
|
||||
if systemd-run --version >/dev/null 2>&1; then
|
||||
systemd-run --quiet --pipe --wait --collect \
|
||||
-p User="$SERVICE_USER" -p Group="$(id -gn "$SERVICE_USER")" \
|
||||
-p WorkingDirectory="$APP_DIR" -p EnvironmentFile="$ENV_FILE" "$@"
|
||||
else
|
||||
(cd "$APP_DIR" && runuser -u "$SERVICE_USER" -- "$@")
|
||||
fi
|
||||
}
|
||||
|
||||
show() {
|
||||
sleep "$1"
|
||||
systemctl --no-pager --lines=0 status ping-collector.service || true
|
||||
journalctl -u ping-collector -n 25 --no-pager || true
|
||||
}
|
||||
|
||||
env_is() { grep -q "^$1=$2\s*$" "$ENV_FILE"; }
|
||||
|
||||
wait_http() { # url seconds
|
||||
local i
|
||||
for i in $(seq "$2"); do
|
||||
curl -fs -o /dev/null --max-time 5 "$1" 2>/dev/null && return 0
|
||||
sleep 1
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# The Matrix messages come from ping-collector once it notifies; the old
|
||||
# cron job (`run.py --alert-check`, every 15 min, repeating while a level
|
||||
# stays high) is commented out, with a backup of the crontab.
|
||||
retire_matrix_cron() {
|
||||
env_is FORECAST_IN_COLLECTOR true || return 0
|
||||
grep -q '^MATRIX_ACCESS_TOKEN=..*' "$ENV_FILE" || return 0
|
||||
local user backup
|
||||
for user in root "$SERVICE_USER"; do
|
||||
crontab -l -u "$user" 2>/dev/null | grep -v '^\s*#' | grep -q -- '--alert-check' || continue
|
||||
backup="${APP_DIR}/crontab-${user}.$(date +%Y%m%d%H%M%S).bak"
|
||||
crontab -l -u "$user" >"$backup"
|
||||
sed '/--alert-check/ s|^\([^#]\)|# retired: Matrix is sent by ping-collector # \1|' "$backup" |
|
||||
crontab -u "$user" -
|
||||
log "Matrix: commented out the old --alert-check cron job of $user (backup: $backup)"
|
||||
done
|
||||
if grep -rlqs -- '--alert-check' /etc/cron.d /etc/crontab 2>/dev/null; then
|
||||
warn "an --alert-check job is still in $(grep -rls -- '--alert-check' /etc/cron.d /etc/crontab | tr '\n' ' '): remove it, ping-collector sends Matrix now"
|
||||
fi
|
||||
if systemctl list-timers --all --no-pager 2>/dev/null | grep -qi alert; then
|
||||
warn "a systemd timer mentions 'alert' (systemctl list-timers --all): if it runs --alert-check, disable it"
|
||||
fi
|
||||
}
|
||||
|
||||
# ping-collector serve next to the Python app; every route compared.
|
||||
api_check() {
|
||||
local unit=ping-api-check ok=0
|
||||
systemctl stop "$unit" 2>/dev/null || true
|
||||
systemd-run --quiet --unit="$unit" --collect \
|
||||
-p User="$SERVICE_USER" -p Group="$(id -gn "$SERVICE_USER")" \
|
||||
-p WorkingDirectory="$APP_DIR" -p EnvironmentFile="$ENV_FILE" \
|
||||
"${APP_DIR}/bin/ping-collector" serve --listen "127.0.0.1:${CHECK_PORT}" --no-analytics
|
||||
if wait_http "http://127.0.0.1:${CHECK_PORT}/api/notifications" 30; then
|
||||
as_service "$PYTHON" scripts/api_parity.py \
|
||||
--python "http://127.0.0.1:${WEB_ADDR##*:}" --rust "http://127.0.0.1:${CHECK_PORT}" && ok=1
|
||||
else
|
||||
warn "ping-collector serve did not answer on port ${CHECK_PORT}:"
|
||||
journalctl -u "$unit" -n 20 --no-pager || true
|
||||
fi
|
||||
systemctl stop "$unit" 2>/dev/null || true
|
||||
[ "$ok" = 1 ]
|
||||
}
|
||||
|
||||
rollback_api() {
|
||||
if env_is WEB_LISTEN '..*'; then
|
||||
log "The Python app serves the website and API again"
|
||||
set_env WEB_LISTEN ""
|
||||
systemctl restart ping-collector.service
|
||||
fi
|
||||
systemctl enable water-monitor.service
|
||||
systemctl restart water-monitor.service
|
||||
}
|
||||
|
||||
matrix_backups() {
|
||||
ls "${APP_DIR}"/crontab-*.bak 2>/dev/null | sed 's/^/ /' || true
|
||||
}
|
||||
|
||||
case "$MODE" in
|
||||
--rollback)
|
||||
log "Stopping ping-collector; the Python app collects, serves, forecasts and notifies again"
|
||||
systemctl disable --now ping-collector.service 2>/dev/null || true
|
||||
set_env WEB_LISTEN ""
|
||||
set_env FORECAST_IN_COLLECTOR false
|
||||
set_env COLLECTION_ENABLED true
|
||||
systemctl enable water-monitor.service
|
||||
systemctl restart water-monitor.service
|
||||
if [ -n "$(matrix_backups)" ]; then
|
||||
warn "Matrix alerts came from ping-collector; to restore the old cron job: crontab -u USER BACKUP"
|
||||
matrix_backups
|
||||
fi
|
||||
log "Done. Check: journalctl -u water-monitor -n 50"
|
||||
exit 0
|
||||
;;
|
||||
--rollback-api)
|
||||
rollback_api
|
||||
log "Done. Check: journalctl -u water-monitor -n 50; curl -s http://127.0.0.1:${WEB_ADDR##*:}/health"
|
||||
exit 0
|
||||
;;
|
||||
--rollback-forecast)
|
||||
# the Python app forecasts in its own process: it must serve again
|
||||
rollback_api
|
||||
log "Forecasts and notifications back to the Python app; ping-collector keeps collecting"
|
||||
set_env FORECAST_IN_COLLECTOR false
|
||||
systemctl restart ping-collector.service water-monitor.service
|
||||
if [ -n "$(matrix_backups)" ]; then
|
||||
warn "Matrix alerts stop with this (the Python app sends ntfy only); old cron job: crontab -u USER BACKUP"
|
||||
matrix_backups
|
||||
fi
|
||||
log "Done. Check: journalctl -u water-monitor -n 50"
|
||||
exit 0
|
||||
;;
|
||||
--check-api)
|
||||
[ -x "${APP_DIR}/bin/ping-collector" ] || die "install first"
|
||||
if env_is WEB_LISTEN '..*'; then
|
||||
die "ping-collector already serves the website (WEB_LISTEN is set)"
|
||||
fi
|
||||
api_check && log "The Rust API answers as the Python app" || die "the API check found differences"
|
||||
exit 0
|
||||
;;
|
||||
install | --cutover | --cutover-forecast | --cutover-api) ;;
|
||||
*) die "unknown option: $MODE" ;;
|
||||
esac
|
||||
|
||||
BIN="${BIN:-${REPO_DIR}/rust/target/release/ping-collector}"
|
||||
[ -x "$BIN" ] || die "no binary at $BIN: build it (see header) or pass BIN=/path/to/ping-collector"
|
||||
|
||||
log "Installing $BIN -> ${APP_DIR}/bin/ping-collector ($("$BIN" --version))"
|
||||
install -D -m 0755 -o root -g root "$BIN" "${APP_DIR}/bin/ping-collector"
|
||||
install -m 0644 "${REPO_DIR}/scripts/ping-collector.service" /etc/systemd/system/ping-collector.service
|
||||
systemctl daemon-reload
|
||||
|
||||
# Both services read and write /scraping/status through this file (inside
|
||||
# the app dir: the units use PrivateTmp).
|
||||
set_env SCRAPING_STATUS_FILE "$STATUS_FILE"
|
||||
touch "$STATUS_FILE"
|
||||
chown "${SERVICE_USER}:$(id -gn "$SERVICE_USER")" "$STATUS_FILE"
|
||||
|
||||
log "Dry run against the configured database (writes nothing)"
|
||||
as_service "${APP_DIR}/bin/ping-collector" once --dry-run
|
||||
|
||||
forecast_ok=1
|
||||
if compgen -G "${APP_DIR}/models/flood_*.joblib" >/dev/null; then
|
||||
log "Exporting the trained models for the Rust forecaster (models/flood_*.json.gz)"
|
||||
as_service "$PYTHON" -m src.ml.export
|
||||
log "Forecast check: Python and ping-collector on the same data, moment and rain"
|
||||
as_service "$PYTHON" scripts/forecast_parity.py || forecast_ok=0
|
||||
else
|
||||
warn "no trained models in ${APP_DIR}/models: forecast check skipped"
|
||||
forecast_ok=0
|
||||
fi
|
||||
|
||||
case "$MODE" in
|
||||
--cutover)
|
||||
log "Cutover: ping-collector collects; the Python app forecasts and notifies"
|
||||
set_env COLLECTION_ENABLED false
|
||||
systemctl enable ping-collector.service
|
||||
systemctl restart ping-collector.service water-monitor.service
|
||||
show 20
|
||||
log "Next step: --cutover-forecast. Rollback: sudo bash scripts/install_collector.sh --rollback"
|
||||
;;
|
||||
--cutover-forecast)
|
||||
grep -q '^COLLECTION_ENABLED=false' "$ENV_FILE" ||
|
||||
die "ping-collector must collect first: run --cutover"
|
||||
if [ "$forecast_ok" != 1 ] && [ "${FORCE:-0}" != 1 ]; then
|
||||
die "the forecast check did not pass; not switching (FORCE=1 overrides)"
|
||||
fi
|
||||
log "Cutover: ping-collector also forecasts and notifies; the Python app serves its forecast"
|
||||
set_env FORECAST_IN_COLLECTOR true
|
||||
retire_matrix_cron
|
||||
systemctl enable ping-collector.service
|
||||
systemctl restart ping-collector.service water-monitor.service
|
||||
show 30
|
||||
log "Next step: --cutover-api. Rollback: sudo bash scripts/install_collector.sh --rollback-forecast"
|
||||
;;
|
||||
--cutover-api)
|
||||
env_is FORECAST_IN_COLLECTOR true || die "ping-collector must forecast first: run --cutover-forecast"
|
||||
if env_is WEB_LISTEN '..*'; then
|
||||
log "ping-collector already serves the website; restarting it on the new binary"
|
||||
retire_matrix_cron
|
||||
systemctl restart ping-collector.service
|
||||
show 10
|
||||
exit 0
|
||||
fi
|
||||
api_ok=1
|
||||
if [ "$(systemctl is-active water-monitor.service 2>/dev/null)" = active ]; then
|
||||
log "API check: ping-collector serve on :${CHECK_PORT} next to the Python app, every route compared"
|
||||
api_check || api_ok=0
|
||||
else
|
||||
warn "the Python app is not running: nothing to compare the API with"
|
||||
api_ok=0
|
||||
fi
|
||||
if [ "$api_ok" != 1 ] && [ "${FORCE:-0}" != 1 ]; then
|
||||
die "the API check did not pass; not switching (FORCE=1 overrides)"
|
||||
fi
|
||||
log "Cutover: ping-collector serves the website and API on ${WEB_ADDR}; the Python app stops"
|
||||
set_env WEB_LISTEN "$WEB_ADDR"
|
||||
retire_matrix_cron
|
||||
systemctl disable --now water-monitor.service
|
||||
systemctl restart ping-collector.service
|
||||
if wait_http "http://127.0.0.1:${WEB_ADDR##*:}/health" 60; then
|
||||
log "The website answers from ping-collector: $(curl -s --max-time 5 "http://127.0.0.1:${WEB_ADDR##*:}/scraping/status" | grep -o '"collector":"[^"]*"')"
|
||||
else
|
||||
warn "no answer on port ${WEB_ADDR##*:} yet; check journalctl -u ping-collector"
|
||||
fi
|
||||
show 5
|
||||
log "Rollback if needed: sudo bash scripts/install_collector.sh --rollback-api"
|
||||
;;
|
||||
*)
|
||||
retire_matrix_cron
|
||||
if [ "$(systemctl is-active ping-collector.service 2>/dev/null)" = active ]; then
|
||||
systemctl restart ping-collector.service
|
||||
log "Installed; ping-collector restarted on the new binary"
|
||||
else
|
||||
log "Installed, not started. Switch over with: sudo bash scripts/install_collector.sh --cutover"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
@@ -92,7 +92,7 @@ if [[ -f "$env_file" ]] && ! grep -q '^NTFY_SERVER=' "$env_file"; then
|
||||
echo "NTFY_TOPIC_PREFIX=ping"
|
||||
echo "NTFY_TOKEN=${token}"
|
||||
} >> "$env_file"
|
||||
echo "wrote NTFY_* to ${env_file}; restart water-monitor to enable"
|
||||
echo "wrote NTFY_* to ${env_file}; restart ping-collector to enable"
|
||||
else
|
||||
echo "NTFY_TOKEN=${token}"
|
||||
fi
|
||||
|
||||
@@ -9,7 +9,7 @@ and errors per stage plus the slowest endpoints.
|
||||
Run against a LOCAL instance for full stress (never full-stress production —
|
||||
it hosts live flood monitoring):
|
||||
|
||||
python -m uvicorn src.web_api:app --port 8125 # separate shell
|
||||
rust/target/release/ping-collector serve --listen 127.0.0.1:8125 --no-analytics # separate shell
|
||||
python scripts/load_test.py http://localhost:8125
|
||||
|
||||
A gentle production baseline (low, fixed concurrency):
|
||||
@@ -98,9 +98,7 @@ def run_stage(base, clients, seconds):
|
||||
by_endpoint = {}
|
||||
for path, elapsed in results:
|
||||
by_endpoint.setdefault(path, []).append(elapsed)
|
||||
slowest = sorted(
|
||||
by_endpoint.items(), key=lambda kv: -statistics.median(kv[1])
|
||||
)[:4]
|
||||
slowest = sorted(by_endpoint.items(), key=lambda kv: -statistics.median(kv[1]))[:4]
|
||||
for path, values in slowest:
|
||||
print(
|
||||
f" slow: {path:45} n={len(values):5} "
|
||||
|
||||
@@ -7,7 +7,8 @@ the page makes, polling like the auto-refresh does) and API consumers
|
||||
Full stress against a LOCAL instance (never full-stress production — it hosts
|
||||
live flood monitoring):
|
||||
|
||||
# separate shell: python -m uvicorn src.web_api:app --port 8125
|
||||
# separate shell:
|
||||
# rust/target/release/ping-collector serve --listen 127.0.0.1:8125 --no-analytics
|
||||
.venv/Scripts/python.exe -m locust -f scripts/locustfile.py \
|
||||
--host http://localhost:8125 --headless \
|
||||
--users 200 --spawn-rate 20 --run-time 2m \
|
||||
@@ -77,9 +78,11 @@ class ApiConsumer(FastHttpUser):
|
||||
|
||||
@task(3)
|
||||
def hii_feeds(self):
|
||||
self.client.get(random.choice(
|
||||
["/api/hii/waterlevel/latest", "/api/hii/rainfall/latest"]
|
||||
), headers=GZIP, name="/api/hii/[feed]/latest")
|
||||
self.client.get(
|
||||
random.choice(["/api/hii/waterlevel/latest", "/api/hii/rainfall/latest"]),
|
||||
headers=GZIP,
|
||||
name="/api/hii/[feed]/latest",
|
||||
)
|
||||
|
||||
@task(2)
|
||||
def forecast(self):
|
||||
|
||||
@@ -1,619 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
SQLite to PostgreSQL Migration Tool
|
||||
Migrates all data from SQLite database to PostgreSQL
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
import sqlite3
|
||||
from datetime import datetime, timezone
|
||||
from typing import Dict, List, Optional, Tuple, Any
|
||||
from dataclasses import dataclass
|
||||
|
||||
# Add src to path for imports
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src'))
|
||||
|
||||
@dataclass
|
||||
class MigrationStats:
|
||||
stations_migrated: int = 0
|
||||
measurements_migrated: int = 0
|
||||
errors: List[str] = None
|
||||
start_time: Optional[datetime] = None
|
||||
end_time: Optional[datetime] = None
|
||||
|
||||
def __post_init__(self):
|
||||
if self.errors is None:
|
||||
self.errors = []
|
||||
|
||||
class SQLiteToPostgresMigrator:
|
||||
def __init__(self, sqlite_path: str, postgres_config: Dict[str, Any]):
|
||||
self.sqlite_path = sqlite_path
|
||||
self.postgres_config = postgres_config
|
||||
self.sqlite_conn = None
|
||||
self.postgres_adapter = None
|
||||
self.stats = MigrationStats()
|
||||
|
||||
# Setup logging with UTF-8 encoding
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(levelname)s - %(message)s',
|
||||
handlers=[
|
||||
logging.StreamHandler(),
|
||||
logging.FileHandler('migration.log', encoding='utf-8')
|
||||
]
|
||||
)
|
||||
self.logger = logging.getLogger(__name__)
|
||||
|
||||
def connect_databases(self) -> bool:
|
||||
"""Connect to both SQLite and PostgreSQL databases"""
|
||||
try:
|
||||
# Connect to SQLite
|
||||
if not os.path.exists(self.sqlite_path):
|
||||
self.logger.error(f"SQLite database not found: {self.sqlite_path}")
|
||||
return False
|
||||
|
||||
self.sqlite_conn = sqlite3.connect(self.sqlite_path)
|
||||
self.sqlite_conn.row_factory = sqlite3.Row # For dict-like access
|
||||
self.logger.info(f"Connected to SQLite database: {self.sqlite_path}")
|
||||
|
||||
# Connect to PostgreSQL
|
||||
from database_adapters import create_database_adapter
|
||||
self.postgres_adapter = create_database_adapter(
|
||||
self.postgres_config['type'],
|
||||
connection_string=self.postgres_config['connection_string']
|
||||
)
|
||||
|
||||
if not self.postgres_adapter.connect():
|
||||
self.logger.error("Failed to connect to PostgreSQL")
|
||||
return False
|
||||
|
||||
self.logger.info("Connected to PostgreSQL database")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Database connection error: {e}")
|
||||
return False
|
||||
|
||||
def analyze_sqlite_schema(self) -> Dict[str, List[str]]:
|
||||
"""Analyze SQLite database structure"""
|
||||
try:
|
||||
cursor = self.sqlite_conn.cursor()
|
||||
|
||||
# Get all tables
|
||||
cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'")
|
||||
tables = [row[0] for row in cursor.fetchall()]
|
||||
|
||||
schema_info = {}
|
||||
for table in tables:
|
||||
cursor.execute(f"PRAGMA table_info({table})")
|
||||
columns = [row[1] for row in cursor.fetchall()]
|
||||
schema_info[table] = columns
|
||||
|
||||
# Get row count
|
||||
cursor.execute(f"SELECT COUNT(*) FROM {table}")
|
||||
count = cursor.fetchone()[0]
|
||||
self.logger.info(f"Table '{table}': {len(columns)} columns, {count} rows")
|
||||
|
||||
return schema_info
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Schema analysis error: {e}")
|
||||
return {}
|
||||
|
||||
def migrate_stations(self) -> bool:
|
||||
"""Migrate station data"""
|
||||
try:
|
||||
cursor = self.sqlite_conn.cursor()
|
||||
|
||||
# Try different possible table names and structures
|
||||
station_queries = [
|
||||
# Modern structure
|
||||
"""SELECT id, station_code, station_name_th as thai_name, station_name_en as english_name,
|
||||
latitude, longitude, geohash, created_at, updated_at
|
||||
FROM stations""",
|
||||
|
||||
# Alternative structure 1
|
||||
"""SELECT id, station_code, thai_name, english_name,
|
||||
latitude, longitude, geohash, created_at, updated_at
|
||||
FROM stations""",
|
||||
|
||||
# Legacy structure
|
||||
"""SELECT station_id as id, station_code, station_name as thai_name,
|
||||
station_name as english_name, lat as latitude, lon as longitude,
|
||||
NULL as geohash, datetime('now') as created_at, datetime('now') as updated_at
|
||||
FROM water_stations""",
|
||||
|
||||
# Simple structure
|
||||
"""SELECT rowid as id, station_code, name as thai_name, name as english_name,
|
||||
NULL as latitude, NULL as longitude, NULL as geohash,
|
||||
datetime('now') as created_at, datetime('now') as updated_at
|
||||
FROM stations""",
|
||||
]
|
||||
|
||||
stations_data = []
|
||||
|
||||
for query in station_queries:
|
||||
try:
|
||||
cursor.execute(query)
|
||||
rows = cursor.fetchall()
|
||||
if rows:
|
||||
self.logger.info(f"Found {len(rows)} stations using query variant")
|
||||
|
||||
for row in rows:
|
||||
station = {
|
||||
'station_id': row[0],
|
||||
'station_code': row[1] or f"STATION_{row[0]}",
|
||||
'station_name_th': row[2] or f"Station {row[0]}",
|
||||
'station_name_en': row[3] or f"Station {row[0]}",
|
||||
'latitude': row[4],
|
||||
'longitude': row[5],
|
||||
'geohash': row[6],
|
||||
'status': 'active'
|
||||
}
|
||||
stations_data.append(station)
|
||||
break
|
||||
|
||||
except sqlite3.OperationalError as e:
|
||||
if "no such table" in str(e).lower() or "no such column" in str(e).lower():
|
||||
continue
|
||||
else:
|
||||
raise
|
||||
|
||||
if not stations_data:
|
||||
self.logger.warning("No stations found in SQLite database")
|
||||
return True
|
||||
|
||||
# Insert stations into PostgreSQL using raw SQL
|
||||
# Since the adapter is designed for measurements, we'll use direct SQL
|
||||
try:
|
||||
from sqlalchemy import create_engine, text
|
||||
engine = create_engine(self.postgres_config['connection_string'])
|
||||
|
||||
# Process stations individually to avoid transaction rollback issues
|
||||
for station in stations_data:
|
||||
try:
|
||||
with engine.begin() as conn:
|
||||
# Use PostgreSQL UPSERT syntax with correct column names
|
||||
station_sql = """
|
||||
INSERT INTO stations (id, station_code, thai_name, english_name, latitude, longitude, geohash)
|
||||
VALUES (:station_id, :station_code, :thai_name, :english_name, :latitude, :longitude, :geohash)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
thai_name = EXCLUDED.thai_name,
|
||||
english_name = EXCLUDED.english_name,
|
||||
latitude = EXCLUDED.latitude,
|
||||
longitude = EXCLUDED.longitude,
|
||||
geohash = EXCLUDED.geohash,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
"""
|
||||
|
||||
conn.execute(text(station_sql), {
|
||||
'station_id': station['station_id'],
|
||||
'station_code': station['station_code'],
|
||||
'thai_name': station['station_name_th'],
|
||||
'english_name': station['station_name_en'],
|
||||
'latitude': station.get('latitude'),
|
||||
'longitude': station.get('longitude'),
|
||||
'geohash': station.get('geohash')
|
||||
})
|
||||
|
||||
self.stats.stations_migrated += 1
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Error migrating station {station.get('station_code', 'unknown')}: {str(e)[:100]}..."
|
||||
self.logger.warning(error_msg)
|
||||
self.stats.errors.append(error_msg)
|
||||
|
||||
self.logger.info(f"Migrated {self.stats.stations_migrated} stations")
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Station migration failed: {e}")
|
||||
return False
|
||||
|
||||
self.logger.info(f"Migrated {self.stats.stations_migrated} stations")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Station migration error: {e}")
|
||||
return False
|
||||
|
||||
def migrate_measurements(self, batch_size: int = 5000) -> bool:
|
||||
"""Migrate measurement data in batches"""
|
||||
try:
|
||||
cursor = self.sqlite_conn.cursor()
|
||||
|
||||
# Try different possible measurement table structures
|
||||
measurement_queries = [
|
||||
# Modern structure
|
||||
"""SELECT w.timestamp, w.station_id, s.station_code, s.station_name_th, s.station_name_en,
|
||||
w.water_level, w.discharge, w.discharge_percent, w.status
|
||||
FROM water_measurements w
|
||||
JOIN stations s ON w.station_id = s.id
|
||||
ORDER BY w.timestamp""",
|
||||
|
||||
# Alternative with different join
|
||||
"""SELECT w.timestamp, w.station_id, s.station_code, s.thai_name, s.english_name,
|
||||
w.water_level, w.discharge, w.discharge_percent, 'active' as status
|
||||
FROM water_measurements w
|
||||
JOIN stations s ON w.station_id = s.id
|
||||
ORDER BY w.timestamp""",
|
||||
|
||||
# Legacy structure
|
||||
"""SELECT timestamp, station_id, station_code, station_name, station_name,
|
||||
water_level, discharge, discharge_percent, 'active' as status
|
||||
FROM measurements
|
||||
ORDER BY timestamp""",
|
||||
|
||||
# Simple structure without joins
|
||||
"""SELECT timestamp, station_id, 'UNKNOWN' as station_code, 'Unknown' as station_name_th, 'Unknown' as station_name_en,
|
||||
water_level, discharge, discharge_percent, 'active' as status
|
||||
FROM water_measurements
|
||||
ORDER BY timestamp""",
|
||||
]
|
||||
|
||||
measurements_processed = 0
|
||||
|
||||
for query in measurement_queries:
|
||||
try:
|
||||
# Get total count first
|
||||
count_query = query.replace("SELECT", "SELECT COUNT(*) FROM (SELECT").replace("ORDER BY w.timestamp", "") + ")"
|
||||
cursor.execute(count_query)
|
||||
total_measurements = cursor.fetchone()[0]
|
||||
|
||||
if total_measurements == 0:
|
||||
continue
|
||||
|
||||
self.logger.info(f"Found {total_measurements} measurements to migrate")
|
||||
|
||||
# Process in batches
|
||||
offset = 0
|
||||
while True:
|
||||
batch_query = f"{query} LIMIT {batch_size} OFFSET {offset}"
|
||||
cursor.execute(batch_query)
|
||||
rows = cursor.fetchall()
|
||||
|
||||
if not rows:
|
||||
break
|
||||
|
||||
# Convert to measurement format
|
||||
measurements = []
|
||||
for row in rows:
|
||||
try:
|
||||
# Parse timestamp
|
||||
timestamp_str = row[0]
|
||||
if isinstance(timestamp_str, str):
|
||||
try:
|
||||
timestamp = datetime.fromisoformat(timestamp_str.replace('Z', '+00:00'))
|
||||
except:
|
||||
# Try other common formats
|
||||
for fmt in ['%Y-%m-%d %H:%M:%S', '%Y-%m-%d %H:%M:%S.%f', '%Y-%m-%dT%H:%M:%S']:
|
||||
try:
|
||||
timestamp = datetime.strptime(timestamp_str, fmt)
|
||||
break
|
||||
except:
|
||||
continue
|
||||
else:
|
||||
timestamp = datetime.now()
|
||||
else:
|
||||
timestamp = timestamp_str
|
||||
|
||||
measurement = {
|
||||
'timestamp': timestamp,
|
||||
'station_id': row[1] or 999,
|
||||
'station_code': row[2] or 'UNKNOWN',
|
||||
'station_name_th': row[3] or 'Unknown',
|
||||
'station_name_en': row[4] or 'Unknown',
|
||||
'water_level': float(row[5]) if row[5] is not None else None,
|
||||
'discharge': float(row[6]) if row[6] is not None else None,
|
||||
'discharge_percent': float(row[7]) if row[7] is not None else None,
|
||||
'status': row[8] or 'active'
|
||||
}
|
||||
measurements.append(measurement)
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Error processing measurement row: {e}"
|
||||
self.logger.warning(error_msg)
|
||||
continue
|
||||
|
||||
# Save batch to PostgreSQL using fast bulk insert
|
||||
if measurements:
|
||||
try:
|
||||
self._fast_bulk_insert(measurements)
|
||||
measurements_processed += len(measurements)
|
||||
self.stats.measurements_migrated += len(measurements)
|
||||
self.logger.info(f"Migrated {measurements_processed}/{total_measurements} measurements")
|
||||
except Exception as e:
|
||||
error_msg = f"Error saving measurement batch: {e}"
|
||||
self.logger.error(error_msg)
|
||||
self.stats.errors.append(error_msg)
|
||||
|
||||
offset += batch_size
|
||||
|
||||
# If we processed measurements, we're done
|
||||
if measurements_processed > 0:
|
||||
break
|
||||
|
||||
except sqlite3.OperationalError as e:
|
||||
if "no such table" in str(e).lower() or "no such column" in str(e).lower():
|
||||
continue
|
||||
else:
|
||||
raise
|
||||
|
||||
if measurements_processed == 0:
|
||||
self.logger.warning("No measurements found in SQLite database")
|
||||
else:
|
||||
self.logger.info(f"Successfully migrated {measurements_processed} measurements")
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Measurement migration error: {e}")
|
||||
return False
|
||||
|
||||
def _fast_bulk_insert(self, measurements: List[Dict]) -> bool:
|
||||
"""Super fast bulk insert using PostgreSQL COPY or VALUES clause"""
|
||||
try:
|
||||
import psycopg2
|
||||
from urllib.parse import urlparse
|
||||
import io
|
||||
|
||||
# Parse connection string for direct psycopg2 connection
|
||||
parsed = urlparse(self.postgres_config['connection_string'])
|
||||
|
||||
# Try super fast COPY method first
|
||||
try:
|
||||
conn = psycopg2.connect(
|
||||
host=parsed.hostname,
|
||||
port=parsed.port or 5432,
|
||||
database=parsed.path[1:],
|
||||
user=parsed.username,
|
||||
password=parsed.password
|
||||
)
|
||||
|
||||
with conn:
|
||||
with conn.cursor() as cur:
|
||||
# Prepare data for COPY
|
||||
data_buffer = io.StringIO()
|
||||
null_val = '\\N'
|
||||
for m in measurements:
|
||||
data_buffer.write(f"{m['timestamp']}\t{m['station_id']}\t{m['water_level'] or null_val}\t{m['discharge'] or null_val}\t{m['discharge_percent'] or null_val}\t{m['status']}\n")
|
||||
|
||||
data_buffer.seek(0)
|
||||
|
||||
# Use COPY for maximum speed
|
||||
cur.copy_from(
|
||||
data_buffer,
|
||||
'water_measurements',
|
||||
columns=('timestamp', 'station_id', 'water_level', 'discharge', 'discharge_percent', 'status'),
|
||||
sep='\t'
|
||||
)
|
||||
|
||||
conn.close()
|
||||
return True
|
||||
|
||||
except Exception as copy_error:
|
||||
# Fallback to SQLAlchemy bulk insert
|
||||
self.logger.debug(f"COPY failed, using bulk VALUES: {copy_error}")
|
||||
|
||||
from sqlalchemy import create_engine, text
|
||||
engine = create_engine(self.postgres_config['connection_string'])
|
||||
|
||||
with engine.begin() as conn:
|
||||
# Use PostgreSQL's fast bulk insert with ON CONFLICT
|
||||
values_list = []
|
||||
for m in measurements:
|
||||
timestamp = m['timestamp'].isoformat() if hasattr(m['timestamp'], 'isoformat') else str(m['timestamp'])
|
||||
values_list.append(
|
||||
f"('{timestamp}', {m['station_id']}, {m['water_level'] or 'NULL'}, "
|
||||
f"{m['discharge'] or 'NULL'}, {m['discharge_percent'] or 'NULL'}, '{m['status']}')"
|
||||
)
|
||||
|
||||
# Build bulk insert query with ON CONFLICT handling
|
||||
bulk_sql = f"""
|
||||
INSERT INTO water_measurements (timestamp, station_id, water_level, discharge, discharge_percent, status)
|
||||
VALUES {','.join(values_list)}
|
||||
ON CONFLICT (timestamp, station_id) DO UPDATE SET
|
||||
water_level = EXCLUDED.water_level,
|
||||
discharge = EXCLUDED.discharge,
|
||||
discharge_percent = EXCLUDED.discharge_percent,
|
||||
status = EXCLUDED.status
|
||||
"""
|
||||
|
||||
conn.execute(text(bulk_sql))
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Fast bulk insert failed: {e}")
|
||||
# Final fallback to original method
|
||||
try:
|
||||
success = self.postgres_adapter.save_measurements(measurements)
|
||||
return success
|
||||
except Exception as fallback_e:
|
||||
self.logger.error(f"All insert methods failed: {fallback_e}")
|
||||
return False
|
||||
|
||||
def verify_migration(self) -> bool:
|
||||
"""Verify the migration by comparing counts"""
|
||||
try:
|
||||
# Get SQLite counts
|
||||
cursor = self.sqlite_conn.cursor()
|
||||
|
||||
sqlite_stations = 0
|
||||
sqlite_measurements = 0
|
||||
|
||||
# Try to get station count
|
||||
for table in ['stations', 'water_stations']:
|
||||
try:
|
||||
cursor.execute(f"SELECT COUNT(*) FROM {table}")
|
||||
sqlite_stations = cursor.fetchone()[0]
|
||||
break
|
||||
except:
|
||||
continue
|
||||
|
||||
# Try to get measurement count
|
||||
for table in ['water_measurements', 'measurements']:
|
||||
try:
|
||||
cursor.execute(f"SELECT COUNT(*) FROM {table}")
|
||||
sqlite_measurements = cursor.fetchone()[0]
|
||||
break
|
||||
except:
|
||||
continue
|
||||
|
||||
# Get PostgreSQL counts
|
||||
postgres_measurements = self.postgres_adapter.get_latest_measurements(limit=999999)
|
||||
postgres_count = len(postgres_measurements)
|
||||
|
||||
self.logger.info("Migration Verification:")
|
||||
self.logger.info(f"SQLite stations: {sqlite_stations}")
|
||||
self.logger.info(f"SQLite measurements: {sqlite_measurements}")
|
||||
self.logger.info(f"PostgreSQL measurements retrieved: {postgres_count}")
|
||||
self.logger.info(f"Migrated stations: {self.stats.stations_migrated}")
|
||||
self.logger.info(f"Migrated measurements: {self.stats.measurements_migrated}")
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Verification error: {e}")
|
||||
return False
|
||||
|
||||
def run_migration(self, sqlite_path: str = None) -> bool:
|
||||
"""Run the complete migration process"""
|
||||
self.stats.start_time = datetime.now()
|
||||
|
||||
if sqlite_path:
|
||||
self.sqlite_path = sqlite_path
|
||||
|
||||
self.logger.info("=" * 60)
|
||||
self.logger.info("SQLite to PostgreSQL Migration Tool")
|
||||
self.logger.info("=" * 60)
|
||||
self.logger.info(f"SQLite database: {self.sqlite_path}")
|
||||
self.logger.info(f"PostgreSQL: {self.postgres_config['type']}")
|
||||
|
||||
try:
|
||||
# Step 1: Connect to databases
|
||||
self.logger.info("Step 1: Connecting to databases...")
|
||||
if not self.connect_databases():
|
||||
return False
|
||||
|
||||
# Step 2: Analyze SQLite schema
|
||||
self.logger.info("Step 2: Analyzing SQLite database structure...")
|
||||
schema_info = self.analyze_sqlite_schema()
|
||||
if not schema_info:
|
||||
self.logger.error("Could not analyze SQLite database structure")
|
||||
return False
|
||||
|
||||
# Step 3: Migrate stations
|
||||
self.logger.info("Step 3: Migrating station data...")
|
||||
if not self.migrate_stations():
|
||||
self.logger.error("Station migration failed")
|
||||
return False
|
||||
|
||||
# Step 4: Migrate measurements
|
||||
self.logger.info("Step 4: Migrating measurement data...")
|
||||
if not self.migrate_measurements():
|
||||
self.logger.error("Measurement migration failed")
|
||||
return False
|
||||
|
||||
# Step 5: Verify migration
|
||||
self.logger.info("Step 5: Verifying migration...")
|
||||
self.verify_migration()
|
||||
|
||||
self.stats.end_time = datetime.now()
|
||||
duration = self.stats.end_time - self.stats.start_time
|
||||
|
||||
# Final report
|
||||
self.logger.info("=" * 60)
|
||||
self.logger.info("MIGRATION COMPLETED")
|
||||
self.logger.info("=" * 60)
|
||||
self.logger.info(f"Duration: {duration}")
|
||||
self.logger.info(f"Stations migrated: {self.stats.stations_migrated}")
|
||||
self.logger.info(f"Measurements migrated: {self.stats.measurements_migrated}")
|
||||
|
||||
if self.stats.errors:
|
||||
self.logger.warning(f"Errors encountered: {len(self.stats.errors)}")
|
||||
for error in self.stats.errors[:10]: # Show first 10 errors
|
||||
self.logger.warning(f" - {error}")
|
||||
if len(self.stats.errors) > 10:
|
||||
self.logger.warning(f" ... and {len(self.stats.errors) - 10} more errors")
|
||||
else:
|
||||
self.logger.info("No errors encountered")
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Migration failed: {e}")
|
||||
return False
|
||||
|
||||
finally:
|
||||
# Cleanup
|
||||
if self.sqlite_conn:
|
||||
self.sqlite_conn.close()
|
||||
|
||||
def main():
|
||||
"""Main entry point"""
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description="Migrate SQLite data to PostgreSQL")
|
||||
parser.add_argument("sqlite_path", nargs="?", help="Path to SQLite database file")
|
||||
parser.add_argument("--batch-size", type=int, default=5000, help="Batch size for processing measurements")
|
||||
parser.add_argument("--fast", action="store_true", help="Use maximum speed mode (batch-size 10000)")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Analyze only, don't migrate")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Set fast mode
|
||||
if args.fast:
|
||||
args.batch_size = 10000
|
||||
|
||||
# Get SQLite path
|
||||
sqlite_path = args.sqlite_path
|
||||
if not sqlite_path:
|
||||
# Try to find common SQLite database files
|
||||
possible_paths = [
|
||||
"water_levels.db",
|
||||
"water_monitoring.db",
|
||||
"database.db",
|
||||
"../water_levels.db"
|
||||
]
|
||||
|
||||
for path in possible_paths:
|
||||
if os.path.exists(path):
|
||||
sqlite_path = path
|
||||
break
|
||||
|
||||
if not sqlite_path:
|
||||
print("SQLite database file not found. Please specify the path:")
|
||||
print(" python migrate_sqlite_to_postgres.py /path/to/database.db")
|
||||
return False
|
||||
|
||||
# Get PostgreSQL configuration
|
||||
try:
|
||||
from config import Config
|
||||
postgres_config = Config.get_database_config()
|
||||
|
||||
if postgres_config['type'] != 'postgresql':
|
||||
print("Error: PostgreSQL not configured. Set DB_TYPE=postgresql in your .env file")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error loading PostgreSQL configuration: {e}")
|
||||
return False
|
||||
|
||||
# Run migration
|
||||
migrator = SQLiteToPostgresMigrator(sqlite_path, postgres_config)
|
||||
|
||||
if args.dry_run:
|
||||
print("DRY RUN MODE - Analyzing SQLite database structure only")
|
||||
if migrator.connect_databases():
|
||||
schema_info = migrator.analyze_sqlite_schema()
|
||||
print("\nSQLite database structure analysis complete.")
|
||||
print("Run without --dry-run to perform the actual migration.")
|
||||
return True
|
||||
|
||||
success = migrator.run_migration()
|
||||
return success
|
||||
|
||||
if __name__ == "__main__":
|
||||
success = main()
|
||||
sys.exit(0 if success else 1)
|
||||
@@ -14,10 +14,9 @@ Restart=always
|
||||
RestartSec=30
|
||||
TimeoutStopSec=20
|
||||
|
||||
# Same .env as water-monitor.service (POSTGRES_*, RID_*, GAP_FILL_*,
|
||||
# SCRAPING_STATUS_FILE, FORECAST_IN_COLLECTOR, NTFY_*). The Python app must run
|
||||
# with COLLECTION_ENABLED=false. Models are read from models/ (relative to
|
||||
# WorkingDirectory).
|
||||
# Every key with its default: .env.example (POSTGRES_*, WEB_LISTEN,
|
||||
# FORECAST_IN_COLLECTOR, NTFY_*, MATRIX_*, ...). Models are read from models/
|
||||
# (relative to WorkingDirectory); installed by scripts/install.sh.
|
||||
EnvironmentFile=/opt/thailand-water-monitor/.env
|
||||
|
||||
# Security settings
|
||||
|
||||
+7
-7
@@ -5,15 +5,15 @@
|
||||
#
|
||||
# Why a script rather than ExecStart=train_flood_model.py:
|
||||
# * train.py writes each station's bundle straight into models/ over ~12 min,
|
||||
# and the API's hourly precompute reloads bundles by mtime. Training into
|
||||
# a staging dir and mv-ing (atomic on one filesystem) means the API never
|
||||
# sees a half-written joblib file or a mixed old/new set.
|
||||
# and ping-collector reloads bundles by mtime on every forecast run.
|
||||
# Training into a staging dir and mv-ing (atomic on one filesystem) means
|
||||
# it never sees a half-written bundle or a mixed old/new set.
|
||||
# * A run that produced gauge-only (v2) bundles, or trained too few stations,
|
||||
# must NOT replace the deployed models. train.py already aborts on a
|
||||
# missing rain series; this script re-checks the written metrics anyway.
|
||||
# * No restart is needed: predict.py (Python) and ping-collector (Rust,
|
||||
# FORECAST_IN_COLLECTOR=true) reload changed bundles on their next
|
||||
# forecast run, so the new models are live within an hour.
|
||||
# * No restart is needed: ping-collector reloads changed bundles
|
||||
# (models/flood_*.json.gz, written by train.py next to each .joblib) on
|
||||
# its next forecast run, so the new models are live within an hour.
|
||||
#
|
||||
# Exit codes: 0 ok, 2 training refused (see log), 3 verification failed.
|
||||
set -euo pipefail
|
||||
@@ -88,4 +88,4 @@ for f in "${STAGE_DIR}"/flood_*.joblib "${STAGE_DIR}"/flood_*.json.gz "${STAGE_D
|
||||
mv -f "${f}" "${MODELS_DIR}/${name}"
|
||||
done
|
||||
rm -rf "${STAGE_DIR}"
|
||||
log "deployed ${VERSION} (${TRAINED} stations); previous generation in models/.previous. The API picks it up on its next hourly precompute."
|
||||
log "deployed ${VERSION} (${TRAINED} stations); previous generation in models/.previous. ping-collector loads it on its next forecast run."
|
||||
@@ -1,175 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
PostgreSQL setup script for Northern Thailand Ping River Monitor
|
||||
This script helps you configure and test your PostgreSQL connection
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
from typing import Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
def setup_logging():
|
||||
logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
|
||||
|
||||
def test_postgres_connection(connection_string: str) -> bool:
|
||||
"""Test connection to PostgreSQL database"""
|
||||
try:
|
||||
from sqlalchemy import create_engine, text
|
||||
|
||||
# Test connection
|
||||
engine = create_engine(connection_string, pool_pre_ping=True)
|
||||
with engine.connect() as conn:
|
||||
result = conn.execute(text("SELECT version()"))
|
||||
version = result.fetchone()[0]
|
||||
logging.info(f"✅ Connected to PostgreSQL successfully!")
|
||||
logging.info(f"Database version: {version}")
|
||||
return True
|
||||
|
||||
except ImportError:
|
||||
logging.error("❌ psycopg2-binary not installed. Run: uv add psycopg2-binary")
|
||||
return False
|
||||
except Exception as e:
|
||||
logging.error(f"❌ Connection failed: {e}")
|
||||
return False
|
||||
|
||||
def parse_connection_string(connection_string: str) -> dict:
|
||||
"""Parse PostgreSQL connection string into components"""
|
||||
try:
|
||||
parsed = urlparse(connection_string)
|
||||
return {
|
||||
'host': parsed.hostname,
|
||||
'port': parsed.port or 5432,
|
||||
'database': parsed.path[1:] if parsed.path else None,
|
||||
'username': parsed.username,
|
||||
'password': parsed.password,
|
||||
}
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to parse connection string: {e}")
|
||||
return {}
|
||||
|
||||
def create_database_if_not_exists(connection_string: str, database_name: str) -> bool:
|
||||
"""Create database if it doesn't exist"""
|
||||
try:
|
||||
from sqlalchemy import create_engine, text
|
||||
|
||||
# Connect to default postgres database to create our database
|
||||
parsed = urlparse(connection_string)
|
||||
admin_connection = connection_string.replace(f"/{parsed.path[1:]}", "/postgres")
|
||||
|
||||
engine = create_engine(admin_connection, pool_pre_ping=True)
|
||||
|
||||
with engine.connect() as conn:
|
||||
# Check if database exists
|
||||
result = conn.execute(text(
|
||||
"SELECT 1 FROM pg_database WHERE datname = :db_name"
|
||||
), {"db_name": database_name})
|
||||
|
||||
if result.fetchone():
|
||||
logging.info(f"✅ Database '{database_name}' already exists")
|
||||
return True
|
||||
else:
|
||||
# Create database
|
||||
conn.execute(text("COMMIT")) # End transaction
|
||||
conn.execute(text(f'CREATE DATABASE "{database_name}"'))
|
||||
logging.info(f"✅ Created database '{database_name}'")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"❌ Failed to create database: {e}")
|
||||
return False
|
||||
|
||||
def initialize_tables(connection_string: str) -> bool:
|
||||
"""Initialize database tables"""
|
||||
try:
|
||||
# Import the database adapter to create tables
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src'))
|
||||
from database_adapters import SQLAdapter
|
||||
|
||||
adapter = SQLAdapter(connection_string=connection_string, db_type='postgresql')
|
||||
if adapter.connect():
|
||||
logging.info("✅ Database tables initialized successfully")
|
||||
return True
|
||||
else:
|
||||
logging.error("❌ Failed to initialize tables")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"❌ Failed to initialize tables: {e}")
|
||||
return False
|
||||
|
||||
def interactive_setup():
|
||||
"""Interactive setup wizard"""
|
||||
print("🐘 PostgreSQL Setup Wizard for Ping River Monitor")
|
||||
print("=" * 50)
|
||||
|
||||
# Get connection details
|
||||
host = input("PostgreSQL host (e.g., 192.168.1.100): ").strip()
|
||||
port = input("PostgreSQL port [5432]: ").strip() or "5432"
|
||||
database = input("Database name [water_monitoring]: ").strip() or "water_monitoring"
|
||||
username = input("Username: ").strip()
|
||||
password = input("Password: ").strip()
|
||||
|
||||
# Optional SSL
|
||||
use_ssl = input("Use SSL connection? (y/N): ").strip().lower() == 'y'
|
||||
ssl_params = "?sslmode=require" if use_ssl else ""
|
||||
|
||||
connection_string = f"postgresql://{username}:{password}@{host}:{port}/{database}{ssl_params}"
|
||||
|
||||
print(f"\nGenerated connection string:")
|
||||
print(f"POSTGRES_CONNECTION_STRING={connection_string}")
|
||||
|
||||
return connection_string
|
||||
|
||||
def main():
|
||||
setup_logging()
|
||||
|
||||
print("🚀 Northern Thailand Ping River Monitor - PostgreSQL Setup")
|
||||
print("=" * 60)
|
||||
|
||||
# Check if connection string is provided via environment
|
||||
connection_string = os.getenv('POSTGRES_CONNECTION_STRING')
|
||||
|
||||
if not connection_string:
|
||||
print("No POSTGRES_CONNECTION_STRING found in environment.")
|
||||
print("Starting interactive setup...\n")
|
||||
connection_string = interactive_setup()
|
||||
|
||||
# Suggest adding to .env file
|
||||
print(f"\n💡 Add this to your .env file:")
|
||||
print(f"DB_TYPE=postgresql")
|
||||
print(f"POSTGRES_CONNECTION_STRING={connection_string}")
|
||||
|
||||
# Parse connection details
|
||||
config = parse_connection_string(connection_string)
|
||||
if not config.get('host'):
|
||||
logging.error("Invalid connection string format")
|
||||
return False
|
||||
|
||||
print(f"\n🔗 Connecting to PostgreSQL at {config['host']}:{config['port']}")
|
||||
|
||||
# Test connection
|
||||
if not test_postgres_connection(connection_string):
|
||||
return False
|
||||
|
||||
# Try to create database
|
||||
database_name = config.get('database', 'water_monitoring')
|
||||
if database_name:
|
||||
create_database_if_not_exists(connection_string, database_name)
|
||||
|
||||
# Initialize tables
|
||||
if not initialize_tables(connection_string):
|
||||
return False
|
||||
|
||||
print("\n🎉 PostgreSQL setup completed successfully!")
|
||||
print("\nNext steps:")
|
||||
print("1. Update your .env file with the connection string")
|
||||
print("2. Run: make run-test")
|
||||
print("3. Run: make run-api")
|
||||
|
||||
return True
|
||||
|
||||
if __name__ == "__main__":
|
||||
success = main()
|
||||
sys.exit(0 if success else 1)
|
||||
@@ -1,48 +0,0 @@
|
||||
@echo off
|
||||
REM Setup script for uv-based development environment on Windows
|
||||
|
||||
echo 🚀 Setting up Northern Thailand Ping River Monitor with uv...
|
||||
|
||||
REM Check if uv is installed
|
||||
uv --version >nul 2>&1
|
||||
if %errorlevel% neq 0 (
|
||||
echo ❌ uv is not installed. Please install it first:
|
||||
echo powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
echo ✅ uv found
|
||||
uv --version
|
||||
|
||||
REM Initialize uv project if not already initialized
|
||||
if not exist "uv.lock" (
|
||||
echo 🔧 Initializing uv project...
|
||||
uv sync
|
||||
) else (
|
||||
echo 📦 Syncing dependencies with uv...
|
||||
uv sync
|
||||
)
|
||||
|
||||
REM Install pre-commit hooks
|
||||
echo 🎣 Installing pre-commit hooks...
|
||||
uv run pre-commit install
|
||||
|
||||
REM Create .env file if it doesn't exist
|
||||
if not exist ".env" (
|
||||
if exist ".env.example" (
|
||||
echo 📝 Creating .env file from template...
|
||||
copy .env.example .env
|
||||
echo ⚠️ Please edit .env file with your configuration
|
||||
)
|
||||
)
|
||||
|
||||
echo ✅ Setup complete!
|
||||
echo.
|
||||
echo 📚 Quick start commands:
|
||||
echo make install-dev # Install all dependencies
|
||||
echo make run-test # Run a test cycle
|
||||
echo make run-api # Start the web API
|
||||
echo make test # Run tests
|
||||
echo make lint # Check code quality
|
||||
echo.
|
||||
echo 🎉 Happy monitoring!
|
||||
@@ -1,46 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Setup script for uv-based development environment
|
||||
|
||||
set -e
|
||||
|
||||
echo "🚀 Setting up Northern Thailand Ping River Monitor with uv..."
|
||||
|
||||
# Check if uv is installed
|
||||
if ! command -v uv &> /dev/null; then
|
||||
echo "❌ uv is not installed. Please install it first:"
|
||||
echo " curl -LsSf https://astral.sh/uv/install.sh | sh"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✅ uv found: $(uv --version)"
|
||||
|
||||
# Initialize uv project if not already initialized
|
||||
if [ ! -f "uv.lock" ]; then
|
||||
echo "🔧 Initializing uv project..."
|
||||
uv sync
|
||||
else
|
||||
echo "📦 Syncing dependencies with uv..."
|
||||
uv sync
|
||||
fi
|
||||
|
||||
# Install pre-commit hooks
|
||||
echo "🎣 Installing pre-commit hooks..."
|
||||
uv run pre-commit install
|
||||
|
||||
# Create .env file if it doesn't exist
|
||||
if [ ! -f ".env" ] && [ -f ".env.example" ]; then
|
||||
echo "📝 Creating .env file from template..."
|
||||
cp .env.example .env
|
||||
echo "⚠️ Please edit .env file with your configuration"
|
||||
fi
|
||||
|
||||
echo "✅ Setup complete!"
|
||||
echo ""
|
||||
echo "📚 Quick start commands:"
|
||||
echo " make install-dev # Install all dependencies"
|
||||
echo " make run-test # Run a test cycle"
|
||||
echo " make run-api # Start the web API"
|
||||
echo " make test # Run tests"
|
||||
echo " make lint # Check code quality"
|
||||
echo ""
|
||||
echo "🎉 Happy monitoring!"
|
||||
@@ -25,7 +25,8 @@ def summarize(paths):
|
||||
for fold in station["folds"]:
|
||||
for name, m in fold["variants"].items():
|
||||
a = agg.setdefault(
|
||||
name, {"mae": [], "mae_hi": [], "brier": [], "fa": 0, "events": []}
|
||||
name,
|
||||
{"mae": [], "mae_hi": [], "brier": [], "fa": 0, "events": []},
|
||||
)
|
||||
a["mae"].append(m["mae"])
|
||||
if m.get("mae_above_2p5") is not None:
|
||||
@@ -39,8 +40,18 @@ def summarize(paths):
|
||||
if e["peak_pred_24h_before"] is None
|
||||
else e["peak_pred_24h_before"] - e["peak_level"]
|
||||
)
|
||||
a["events"].append((fold["year"], e["crossing"][:10], e["lead_h"], e["peak_level"], err))
|
||||
print(f"{'variant':22} {'MAE':>6} {'MAE_hi':>7} {'Brier':>7} {'FA':>3} events: year crossing lead_h peak(err24h)")
|
||||
a["events"].append(
|
||||
(
|
||||
fold["year"],
|
||||
e["crossing"][:10],
|
||||
e["lead_h"],
|
||||
e["peak_level"],
|
||||
err,
|
||||
)
|
||||
)
|
||||
print(
|
||||
f"{'variant':22} {'MAE':>6} {'MAE_hi':>7} {'Brier':>7} {'FA':>3} events: year crossing lead_h peak(err24h)"
|
||||
)
|
||||
for name, a in agg.items():
|
||||
ev = " ".join(
|
||||
f"{y} {d} {'—' if l is None else format(l, '+.0f')}h {p:.2f}({'—' if err is None else format(err, '+.2f')})"
|
||||
@@ -54,8 +65,10 @@ def summarize(paths):
|
||||
f"{a['fa']:>3} {ev}"
|
||||
)
|
||||
if leads:
|
||||
print(f"{'':22} lead: mean {statistics.mean(leads):+.1f} h, min {min(leads):+.0f} h, "
|
||||
f"missed {sum(1 for *_, l, _, _ in a['events'] if l is None)}/{len(a['events'])}")
|
||||
print(
|
||||
f"{'':22} lead: mean {statistics.mean(leads):+.1f} h, min {min(leads):+.0f} h, "
|
||||
f"missed {sum(1 for *_, l, _, _ in a['events'] if l is None)}/{len(a['events'])}"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -10,11 +10,11 @@ User=water-monitor
|
||||
Group=water-monitor
|
||||
WorkingDirectory=/opt/thailand-water-monitor
|
||||
EnvironmentFile=/opt/thailand-water-monitor/.env
|
||||
# Same interpreter as water-monitor.service -- the uv-managed .venv.
|
||||
# The uv-managed .venv (Python training stack; scripts/install.sh syncs it).
|
||||
# scripts/retrain.sh trains into models/.staging, refuses to promote anything
|
||||
# that is not a rain-enabled (hgb-v3) set covering the expected stations, then
|
||||
# renames the bundles into place. The API reloads them on its next hourly
|
||||
# precompute; no restart, so a failed run leaves the old models serving.
|
||||
# renames the bundles into place. ping-collector reloads them on its next
|
||||
# forecast run; no restart, so a failed run leaves the old models serving.
|
||||
ExecStart=/bin/bash /opt/thailand-water-monitor/scripts/retrain.sh
|
||||
# HistGradientBoosting is CPU-bound; cap threads so training cannot starve
|
||||
# the API (docs/FLOOD_FORECASTING.md section 6 measured 4 as the sweet spot).
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
[Unit]
|
||||
Description=Thailand Water Level Monitor
|
||||
Documentation=https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor
|
||||
After=network.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=water-monitor
|
||||
Group=water-monitor
|
||||
WorkingDirectory=/opt/thailand-water-monitor
|
||||
# The uv-managed env (uv sync -> .venv). Same interpreter for water-monitor-retrain.service.
|
||||
ExecStart=/opt/thailand-water-monitor/.venv/bin/python run.py --web-api
|
||||
ExecReload=/bin/kill -HUP $MAINPID
|
||||
Restart=always
|
||||
RestartSec=60
|
||||
TimeoutStopSec=30
|
||||
|
||||
# DB_TYPE / POSTGRES_CONNECTION_STRING / MATRIX_* come from the .env file.
|
||||
EnvironmentFile=/opt/thailand-water-monitor/.env
|
||||
Environment=PYTHONPATH=/opt/thailand-water-monitor
|
||||
# Serving path is latency-bound; single-threaded BLAS is 2.6x faster per call
|
||||
# (docs/FLOOD_FORECASTING.md section 6). Training sets its own value.
|
||||
Environment=OMP_NUM_THREADS=1
|
||||
Environment=PYTHONUNBUFFERED=1
|
||||
|
||||
# Security settings
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
ReadWritePaths=/opt/thailand-water-monitor
|
||||
CapabilityBoundingSet=
|
||||
|
||||
# Logging
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=water-monitor
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,2 +0,0 @@
|
||||
target/
|
||||
data/
|
||||
Generated
-132
@@ -1,132 +0,0 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "1.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
||||
|
||||
[[package]]
|
||||
name = "chrono"
|
||||
version = "0.4.45"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327"
|
||||
dependencies = [
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
||||
|
||||
[[package]]
|
||||
name = "num-traits"
|
||||
version = "0.2.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ping-forecast-spike"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.107"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.47"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.229"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_core"
|
||||
version = "1.0.229"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.229"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.154"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"memchr",
|
||||
"serde",
|
||||
"serde_core",
|
||||
"zmij",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "3.0.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
|
||||
|
||||
[[package]]
|
||||
name = "zmij"
|
||||
version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
||||
@@ -1,15 +0,0 @@
|
||||
[package]
|
||||
name = "ping-forecast-spike"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
publish = false
|
||||
|
||||
[dependencies]
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
# float_roundtrip: exact float parsing. Tree thresholds often equal real data
|
||||
# values, so a 1-ulp parse error flips `x <= threshold` (default parser is not exact).
|
||||
serde_json = { version = "1", features = ["float_roundtrip"] }
|
||||
chrono = { version = "0.4", default-features = false, features = ["std"] }
|
||||
|
||||
[profile.release]
|
||||
debug = false
|
||||
@@ -1,43 +0,0 @@
|
||||
# Rust forecast spike
|
||||
|
||||
Answers one question: can the flood forecast be served from Rust without
|
||||
train/serve skew, while training stays in Python (scikit-learn)?
|
||||
Assessment and migration plan: [docs/RUST_REWRITE.md](../../docs/RUST_REWRITE.md).
|
||||
|
||||
- `export/export_fixture.py`: dumps `models/flood_P.1.joblib` (all 9 HGB heads) as
|
||||
flat tree arrays, plus two fixtures from `models/cache` (Aug-Oct 2024 record
|
||||
flood, Jun-Aug 2026): the observed hourly grid of P.1 and its 7 upstream
|
||||
gauges, rain, the pandas feature matrix and every head's sklearn prediction.
|
||||
- `src/features.rs`: port of `src/ml/features.py::build_features` (P.1 path:
|
||||
lags, rises, rolling stats, upstream leads, season, rain).
|
||||
- `src/hgb.rs`: HistGradientBoosting inference (~90 lines).
|
||||
- `src/main.rs`: compares bit-for-bit and times serving-shaped work.
|
||||
|
||||
```bash
|
||||
cd spikes/rust-forecast/export
|
||||
uv run --project ../../.. python export_fixture.py # writes ../data/*.json (gitignored)
|
||||
cd .. && cargo run --release
|
||||
uv run --project ../.. python export/time_python.py # Python timing for comparison
|
||||
```
|
||||
|
||||
Result (model hgb-v3+cbb3bf7, 3,600 hourly rows):
|
||||
|
||||
| | Result |
|
||||
| --- | --- |
|
||||
| Features, 68 columns x 3,600 rows | 244,800 / 244,800 cells bit-identical |
|
||||
| Regression heads (max_6/12/24) | bit-identical on every row |
|
||||
| Classifier heads (warn/danger) | equal to the last ulp (<= 2.2e-16, `exp` vs scipy `expit`) |
|
||||
| One station: 336 h window -> features + 9 heads | Rust 0.25 ms, Python 158 ms |
|
||||
| Model load | Rust 16 ms (JSON), Python 199 ms (joblib) |
|
||||
|
||||
Two traps the spike had to get past, both of which flip tree branches because
|
||||
HGB thresholds often equal real data values exactly:
|
||||
|
||||
1. `serde_json` parses floats best-effort (can be 1 ulp off) unless the
|
||||
`float_roundtrip` feature is on.
|
||||
2. pandas `rolling().sum()/.mean()` are incremental with Kahan compensation,
|
||||
a "run of identical values" shortcut and a sign clamp. A plain window sum
|
||||
differs in the last bits; `rolling_kahan()` replicates pandas exactly.
|
||||
|
||||
Not covered yet: dam features (Mae Ngat), HII gap fill, Open-Meteo catchment
|
||||
mean, the heuristic fallback, stage probabilities, the other 14 stations.
|
||||
@@ -1,132 +0,0 @@
|
||||
"""Export a trained flood bundle + a feature/prediction fixture for the Rust spike.
|
||||
|
||||
Run from the monitor repo's environment:
|
||||
uv run --project ../../.. python export_fixture.py (from this directory)
|
||||
|
||||
Writes ../data/model_<station>.json (all heads as flat tree arrays) and
|
||||
../data/fixture_<name>.json (observed hourly grid + rain, the Python feature
|
||||
matrix, and every head's Python prediction for every row).
|
||||
"""
|
||||
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import joblib
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
REPO = Path(__file__).resolve().parents[3] # repo root
|
||||
sys.path.insert(0, str(REPO))
|
||||
|
||||
from src.ml import features # noqa: E402
|
||||
|
||||
OUT = Path(__file__).resolve().parent.parent / "data"
|
||||
OUT.mkdir(exist_ok=True)
|
||||
STATION = "P.1"
|
||||
WINDOWS = {
|
||||
"flood2024": ("2024-08-15", "2024-10-31"), # record flood, high-level paths
|
||||
"recent2026": ("2026-06-01", "2026-08-13"), # latest cached data
|
||||
}
|
||||
|
||||
|
||||
def clean(values):
|
||||
return [None if (v is None or (isinstance(v, float) and math.isnan(v))) else float(v) for v in values]
|
||||
|
||||
|
||||
def export_head(est):
|
||||
preds = est._predictors
|
||||
assert all(len(it) == 1 for it in preds), "multiclass not expected"
|
||||
trees = []
|
||||
for (tp,) in preds:
|
||||
n = tp.nodes
|
||||
assert not n["is_categorical"].any()
|
||||
assert np.isfinite(n["value"]).all() and not np.isneginf(n["num_threshold"]).any()
|
||||
assert not np.isnan(n["num_threshold"]).any()
|
||||
trees.append(
|
||||
{
|
||||
"feature": n["feature_idx"].astype(int).tolist(),
|
||||
# +inf thresholds (a "missing vs everything" split) are not
|
||||
# valid JSON: written as null, read back as +inf in Rust
|
||||
"threshold": [
|
||||
None if np.isposinf(t) else float(t) for t in n["num_threshold"]
|
||||
],
|
||||
"missing_left": n["missing_go_to_left"].astype(bool).tolist(),
|
||||
"left": n["left"].astype(int).tolist(),
|
||||
"right": n["right"].astype(int).tolist(),
|
||||
"leaf": n["is_leaf"].astype(bool).tolist(),
|
||||
"value": n["value"].astype(float).tolist(),
|
||||
}
|
||||
)
|
||||
kind = "classifier" if hasattr(est, "predict_proba") else "regressor"
|
||||
return {
|
||||
"kind": kind,
|
||||
"baseline": float(np.ravel(est._baseline_prediction)[0]),
|
||||
"trees": trees,
|
||||
}
|
||||
|
||||
|
||||
bundle = joblib.load(REPO / "models" / f"flood_{STATION}.joblib")
|
||||
model = {
|
||||
"station_code": STATION,
|
||||
"model_version": bundle["model_version"],
|
||||
"regression_target": bundle.get("regression_target"),
|
||||
"feature_names": bundle["feature_names"],
|
||||
"horizons": bundle["horizons"],
|
||||
"sigma": {str(k): float(v) for k, v in bundle["sigma"].items()},
|
||||
"thresholds": bundle["thresholds"],
|
||||
"heads": {name: export_head(est) for name, est in bundle["heads"].items()},
|
||||
}
|
||||
(OUT / f"model_{STATION}.json").write_text(json.dumps(model))
|
||||
n_nodes = sum(len(t["value"]) for h in model["heads"].values() for t in h["trees"])
|
||||
print(f"model {model['model_version']}: {len(model['heads'])} heads, {n_nodes} nodes")
|
||||
|
||||
stations = [STATION] + [u for u, _ in features.UPSTREAM_LEADS[STATION]]
|
||||
long = pd.concat(
|
||||
pd.read_csv(REPO / "models" / "cache" / f"{s}.csv.gz", parse_dates=["timestamp"])
|
||||
for s in stations
|
||||
)
|
||||
rain_all = pd.read_csv(REPO / "models" / "cache" / "rain_openmeteo.csv.gz")
|
||||
print("rain columns:", list(rain_all.columns))
|
||||
ts_col = rain_all.columns[0]
|
||||
rain_all[ts_col] = pd.to_datetime(rain_all[ts_col])
|
||||
rain_all = rain_all.set_index(ts_col)[rain_all.columns[1]].astype(float)
|
||||
|
||||
for name, (start, end) in WINDOWS.items():
|
||||
df = long[(long.timestamp >= start) & (long.timestamp < end)]
|
||||
grid = features.make_hourly_grid(df)
|
||||
idx = grid.observed.index
|
||||
rain = rain_all.reindex(idx)
|
||||
# punch deterministic holes so NaN handling of the rolling sums is exercised
|
||||
rain.iloc[::97] = np.nan
|
||||
feats = features.build_features(grid, STATION, rain=rain)[model["feature_names"]]
|
||||
|
||||
preds = {}
|
||||
for head_name, est in bundle["heads"].items():
|
||||
if hasattr(est, "predict_proba"):
|
||||
preds[head_name] = est.predict_proba(feats)[:, 1].tolist()
|
||||
else:
|
||||
preds[head_name] = est.predict(feats).tolist()
|
||||
|
||||
fixture = {
|
||||
"station": STATION,
|
||||
"timestamps": [t.strftime("%Y-%m-%dT%H:%M:%S") for t in idx],
|
||||
"observed": {
|
||||
s: {
|
||||
"water_level": clean(grid.observed[(s, "water_level")].tolist()),
|
||||
"discharge": clean(grid.observed[(s, "discharge")].tolist()),
|
||||
}
|
||||
for s in stations
|
||||
if (s, "water_level") in grid.observed.columns
|
||||
},
|
||||
"rain": clean(rain.tolist()),
|
||||
"feature_names": model["feature_names"],
|
||||
"features": [clean(row) for row in feats.to_numpy().tolist()],
|
||||
"predictions": preds,
|
||||
}
|
||||
(OUT / f"fixture_{name}.json").write_text(json.dumps(fixture))
|
||||
print(
|
||||
f"{name}: {len(idx)} hours, {len(fixture['observed'])} stations, "
|
||||
f"P.1 max {np.nanmax(grid.observed[(STATION, 'water_level')]):.2f} m"
|
||||
)
|
||||
@@ -1,38 +0,0 @@
|
||||
"""Python timing for the same serving-shaped work as the Rust spike."""
|
||||
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import joblib
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
REPO = Path(__file__).resolve().parents[3] # repo root
|
||||
sys.path.insert(0, str(REPO))
|
||||
from src.ml import features # noqa: E402
|
||||
|
||||
bundle = joblib.load(REPO / "models" / "flood_P.1.joblib")
|
||||
stations = ["P.1"] + [u for u, _ in features.UPSTREAM_LEADS["P.1"]]
|
||||
long = pd.concat(
|
||||
pd.read_csv(REPO / "models" / "cache" / f"{s}.csv.gz", parse_dates=["timestamp"])
|
||||
for s in stations
|
||||
)
|
||||
df = long[(long.timestamp >= "2026-07-30") & (long.timestamp < "2026-08-13")]
|
||||
rain_all = pd.read_csv(REPO / "models" / "cache" / "rain_openmeteo.csv.gz", parse_dates=["timestamp"])
|
||||
rain_all = rain_all.set_index("timestamp")["chiang_dao"].astype(float)
|
||||
|
||||
reps = 20
|
||||
t = time.perf_counter()
|
||||
for _ in range(reps):
|
||||
grid = features.make_hourly_grid(df)
|
||||
rain = rain_all.reindex(grid.observed.index)
|
||||
row = features.build_features(grid, "P.1", rain=rain)[bundle["feature_names"]].iloc[[-1]]
|
||||
out = {
|
||||
k: (h.predict_proba(row)[0, 1] if hasattr(h, "predict_proba") else h.predict(row)[0])
|
||||
for k, h in bundle["heads"].items()
|
||||
}
|
||||
print(f"python: grid + features + 9 heads for the last hour: {(time.perf_counter() - t) / reps * 1000:.1f} ms per station")
|
||||
t = time.perf_counter()
|
||||
joblib.load(REPO / "models" / "flood_P.1.joblib")
|
||||
print(f"python: bundle load {(time.perf_counter() - t) * 1000:.0f} ms")
|
||||
@@ -1,318 +0,0 @@
|
||||
//! Port of `src/ml/features.py::build_features` (pandas) to plain Rust.
|
||||
//!
|
||||
//! Pandas semantics reproduced: `ffill(limit)`, `shift(k)` (NaN-padded),
|
||||
//! `rolling(w, min_periods=1)` aggregations that skip NaN and yield NaN for an
|
||||
//! all-NaN window, NaN propagation through subtraction.
|
||||
|
||||
use chrono::{Datelike, NaiveDateTime};
|
||||
use std::collections::HashMap;
|
||||
use std::f64::consts::PI;
|
||||
|
||||
pub const FFILL_LIMIT_H: usize = 3;
|
||||
pub const BASIN_ANCHOR: &str = "P.1";
|
||||
const MONSOON_MONTHS: [u32; 5] = [6, 7, 8, 9, 10];
|
||||
|
||||
pub fn upstream_leads(station: &str) -> &'static [(&'static str, i64)] {
|
||||
match station {
|
||||
"P.1" => &[
|
||||
("P.103", 1),
|
||||
("P.67", 7),
|
||||
("P.21", 9),
|
||||
("P.75", 12),
|
||||
("P.4A", 12),
|
||||
("P.92", 15),
|
||||
("P.20", 17),
|
||||
],
|
||||
"P.21" => &[
|
||||
("P.67", 1),
|
||||
("P.75", 3),
|
||||
("P.4A", 3),
|
||||
("P.92", 6),
|
||||
("P.20", 8),
|
||||
],
|
||||
"P.67" => &[("P.75", 5), ("P.4A", 5), ("P.92", 8), ("P.20", 10)],
|
||||
"P.75" | "P.4A" => &[("P.92", 3), ("P.20", 5)],
|
||||
"P.92" => &[("P.20", 2)],
|
||||
"P.5" => &[("P.1", 12), ("P.103", 13)],
|
||||
"P.81" => &[("P.1", 4), ("P.103", 5)],
|
||||
_ => &[], // spike: P.103's table omitted
|
||||
}
|
||||
}
|
||||
|
||||
pub type Series = Vec<f64>;
|
||||
|
||||
pub struct Grid {
|
||||
pub times: Vec<NaiveDateTime>,
|
||||
/// station -> observed (unfilled) hourly series, NaN where missing
|
||||
pub level: HashMap<String, Series>,
|
||||
pub discharge: HashMap<String, Series>,
|
||||
}
|
||||
|
||||
fn nan_series(n: usize) -> Series {
|
||||
vec![f64::NAN; n]
|
||||
}
|
||||
|
||||
pub fn ffill(v: &[f64], limit: usize) -> Series {
|
||||
let mut out = v.to_vec();
|
||||
let mut last = f64::NAN;
|
||||
let mut run = 0;
|
||||
for x in out.iter_mut() {
|
||||
if x.is_nan() {
|
||||
run += 1;
|
||||
if !last.is_nan() && run <= limit {
|
||||
*x = last;
|
||||
}
|
||||
} else {
|
||||
last = *x;
|
||||
run = 0;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// pandas `shift(k)`: positive k looks back, negative k looks ahead.
|
||||
pub fn shift(v: &[f64], k: i64) -> Series {
|
||||
let n = v.len() as i64;
|
||||
(0..n)
|
||||
.map(|i| {
|
||||
let j = i - k;
|
||||
if (0..n).contains(&j) {
|
||||
v[j as usize]
|
||||
} else {
|
||||
f64::NAN
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn sub(a: &[f64], b: &[f64]) -> Series {
|
||||
a.iter().zip(b).map(|(x, y)| x - y).collect()
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, PartialEq)]
|
||||
enum Agg {
|
||||
Mean,
|
||||
Sum,
|
||||
Max,
|
||||
Min,
|
||||
}
|
||||
|
||||
/// pandas `rolling(w, min_periods=1).<agg>()` over the trailing window [i-w+1, i].
|
||||
///
|
||||
/// Sum/Mean replicate pandas' `roll_sum`/`roll_mean` (window/aggregations.pyx)
|
||||
/// operation for operation: an incremental window with separate Kahan
|
||||
/// compensations for adds and removes, the "run of identical values" shortcut
|
||||
/// and the sign clamp of the mean. A plain per-window sum differs in the last
|
||||
/// bits, and HGB thresholds often sit exactly on real data values, so last-bit
|
||||
/// differences flip tree branches. Max/Min involve no arithmetic: exact as is.
|
||||
fn rolling(v: &[f64], w: usize, agg: Agg) -> Series {
|
||||
match agg {
|
||||
Agg::Sum | Agg::Mean => rolling_kahan(v, w, agg == Agg::Mean),
|
||||
Agg::Max | Agg::Min => (0..v.len())
|
||||
.map(|i| {
|
||||
let lo = (i + 1).saturating_sub(w);
|
||||
v[lo..=i]
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|x| !x.is_nan())
|
||||
.fold(f64::NAN, |acc, x| {
|
||||
if acc.is_nan() {
|
||||
x
|
||||
} else if agg == Agg::Max {
|
||||
acc.max(x)
|
||||
} else {
|
||||
acc.min(x)
|
||||
}
|
||||
})
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn rolling_kahan(v: &[f64], w: usize, mean: bool) -> Series {
|
||||
let (mut sum, mut comp_add, mut comp_remove) = (0.0f64, 0.0f64, 0.0f64);
|
||||
let (mut nobs, mut neg_ct, mut same_run) = (0i64, 0i64, 0i64);
|
||||
let mut prev = if v.is_empty() { f64::NAN } else { v[0] };
|
||||
let mut out = Vec::with_capacity(v.len());
|
||||
for i in 0..v.len() {
|
||||
// window [s, e) for row i; previous window was [i-w, i) clipped
|
||||
if i >= w {
|
||||
let val = v[i - w];
|
||||
if !val.is_nan() {
|
||||
nobs -= 1;
|
||||
let y = -val - comp_remove;
|
||||
let t = sum + y;
|
||||
comp_remove = t - sum - y;
|
||||
sum = t;
|
||||
if val.is_sign_negative() {
|
||||
neg_ct -= 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
let val = v[i];
|
||||
if !val.is_nan() {
|
||||
nobs += 1;
|
||||
let y = val - comp_add;
|
||||
let t = sum + y;
|
||||
comp_add = t - sum - y;
|
||||
sum = t;
|
||||
if val.is_sign_negative() {
|
||||
neg_ct += 1;
|
||||
}
|
||||
same_run = if val == prev { same_run + 1 } else { 1 };
|
||||
prev = val;
|
||||
}
|
||||
out.push(if nobs < 1 {
|
||||
f64::NAN
|
||||
} else if mean {
|
||||
if same_run >= nobs {
|
||||
prev
|
||||
} else {
|
||||
let r = sum / nobs as f64;
|
||||
if neg_ct == 0 && r < 0.0 || neg_ct == nobs && r > 0.0 {
|
||||
0.0
|
||||
} else {
|
||||
r
|
||||
}
|
||||
}
|
||||
} else if same_run >= nobs {
|
||||
prev * nobs as f64
|
||||
} else {
|
||||
sum
|
||||
});
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Hours since the last observed value (0 when observed; NaN before the first).
|
||||
fn hours_since_observed(mask: &[bool]) -> Series {
|
||||
let mut last: Option<usize> = None;
|
||||
mask.iter()
|
||||
.enumerate()
|
||||
.map(|(i, &m)| {
|
||||
if m {
|
||||
last = Some(i);
|
||||
}
|
||||
last.map_or(f64::NAN, |l| (i - l) as f64)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Feature columns by name, in pandas' construction order.
|
||||
pub fn build_features(grid: &Grid, station: &str, rain: Option<&[f64]>) -> Vec<(String, Series)> {
|
||||
let n = grid.times.len();
|
||||
let filled = |s: &str| -> Series {
|
||||
grid.level
|
||||
.get(s)
|
||||
.map_or_else(|| nan_series(n), |v| ffill(v, FFILL_LIMIT_H))
|
||||
};
|
||||
let mut cols: Vec<(String, Series)> = Vec::new();
|
||||
let mut put = |name: String, s: Series| cols.push((name, s));
|
||||
|
||||
let level = filled(station);
|
||||
let discharge = grid
|
||||
.discharge
|
||||
.get(station)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| nan_series(n));
|
||||
let mask: Vec<bool> = grid.level.get(station).map_or_else(
|
||||
|| vec![false; n],
|
||||
|v| v.iter().map(|x| !x.is_nan()).collect(),
|
||||
);
|
||||
|
||||
put("level".into(), level.clone());
|
||||
for k in [1, 2, 3, 6, 12, 24, 48, 72] {
|
||||
put(format!("level_lag_{k}"), shift(&level, k));
|
||||
}
|
||||
for k in [1, 3, 6, 12, 24] {
|
||||
put(format!("rise_{k}"), sub(&level, &shift(&level, k)));
|
||||
}
|
||||
put("roll_mean_6".into(), rolling(&level, 6, Agg::Mean));
|
||||
put("roll_mean_24".into(), rolling(&level, 24, Agg::Mean));
|
||||
put("roll_max_6".into(), rolling(&level, 6, Agg::Max));
|
||||
put("roll_max_24".into(), rolling(&level, 24, Agg::Max));
|
||||
put("roll_max_72".into(), rolling(&level, 72, Agg::Max));
|
||||
put("roll_min_24".into(), rolling(&level, 24, Agg::Min));
|
||||
|
||||
put("discharge".into(), discharge.clone());
|
||||
put("discharge_lag_6".into(), shift(&discharge, 6));
|
||||
put("discharge_lag_24".into(), shift(&discharge, 24));
|
||||
put(
|
||||
"discharge_rise_6".into(),
|
||||
sub(&discharge, &shift(&discharge, 6)),
|
||||
);
|
||||
|
||||
let age = hours_since_observed(&mask);
|
||||
put(
|
||||
"obs_age_h".into(),
|
||||
age.iter()
|
||||
.map(|&a| {
|
||||
if a <= FFILL_LIMIT_H as f64 {
|
||||
a
|
||||
} else {
|
||||
f64::NAN
|
||||
}
|
||||
})
|
||||
.collect(),
|
||||
);
|
||||
let mask_f: Series = mask.iter().map(|&m| if m { 1.0 } else { 0.0 }).collect();
|
||||
put("cov_24h".into(), rolling(&mask_f, 24, Agg::Mean));
|
||||
|
||||
for &(up, lead) in upstream_leads(station) {
|
||||
let u = filled(up);
|
||||
let u_rise_6 = sub(&u, &shift(&u, 6));
|
||||
let u_rollmax_24 = rolling(&u, 24, Agg::Max);
|
||||
let near = (lead - 3).max(0);
|
||||
put(format!("{up}_level_lag_{near}"), shift(&u, near));
|
||||
put(format!("{up}_level_lag_{lead}"), shift(&u, lead));
|
||||
put(format!("{up}_level_lag_{}", lead + 3), shift(&u, lead + 3));
|
||||
put(format!("{up}_rise_6_lag_{lead}"), shift(&u_rise_6, lead));
|
||||
put(
|
||||
format!("{up}_rollmax_24_lag_{near}"),
|
||||
shift(&u_rollmax_24, near),
|
||||
);
|
||||
}
|
||||
|
||||
if station != BASIN_ANCHOR {
|
||||
let p1 = filled(BASIN_ANCHOR);
|
||||
put("P1_level".into(), p1.clone());
|
||||
put("P1_rollmax_24".into(), rolling(&p1, 24, Agg::Max));
|
||||
put("P1_rise_24".into(), sub(&p1, &shift(&p1, 24)));
|
||||
}
|
||||
|
||||
let doy: Vec<f64> = grid.times.iter().map(|t| t.ordinal() as f64).collect();
|
||||
put(
|
||||
"doy_sin".into(),
|
||||
doy.iter().map(|d| (2.0 * PI * d / 365.25).sin()).collect(),
|
||||
);
|
||||
put(
|
||||
"doy_cos".into(),
|
||||
doy.iter().map(|d| (2.0 * PI * d / 365.25).cos()).collect(),
|
||||
);
|
||||
put(
|
||||
"is_monsoon".into(),
|
||||
grid.times
|
||||
.iter()
|
||||
.map(|t| {
|
||||
if MONSOON_MONTHS.contains(&t.month()) {
|
||||
1.0
|
||||
} else {
|
||||
0.0
|
||||
}
|
||||
})
|
||||
.collect(),
|
||||
);
|
||||
|
||||
if let Some(r) = rain {
|
||||
put("rain_6h".into(), rolling(r, 6, Agg::Sum));
|
||||
put("rain_24h".into(), rolling(r, 24, Agg::Sum));
|
||||
put("rain_72h".into(), rolling(r, 72, Agg::Sum));
|
||||
// forward sum over (t, t+24]
|
||||
let mut ahead = shift(r, -1);
|
||||
ahead.reverse();
|
||||
let mut fc = rolling(&ahead, 24, Agg::Sum);
|
||||
fc.reverse();
|
||||
put("rain_fc24".into(), fc);
|
||||
}
|
||||
cols
|
||||
}
|
||||
@@ -1,85 +0,0 @@
|
||||
//! Inference for scikit-learn HistGradientBoosting models exported as flat
|
||||
//! tree arrays (see export/export_fixture.py). Mirrors sklearn's
|
||||
//! `_predictor.pyx`: NaN follows `missing_go_to_left`, otherwise
|
||||
//! `x <= threshold` goes left; raw = baseline + sum of leaf values, summed in
|
||||
//! tree order (same order as sklearn, so results are bit-identical).
|
||||
|
||||
use serde::Deserialize;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
fn thresholds<'de, D: serde::Deserializer<'de>>(d: D) -> Result<Vec<f64>, D::Error> {
|
||||
let raw: Vec<Option<f64>> = Deserialize::deserialize(d)?;
|
||||
Ok(raw
|
||||
.into_iter()
|
||||
.map(|t| t.unwrap_or(f64::INFINITY))
|
||||
.collect())
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct Tree {
|
||||
feature: Vec<usize>,
|
||||
#[serde(deserialize_with = "thresholds")]
|
||||
threshold: Vec<f64>,
|
||||
missing_left: Vec<bool>,
|
||||
left: Vec<usize>,
|
||||
right: Vec<usize>,
|
||||
leaf: Vec<bool>,
|
||||
value: Vec<f64>,
|
||||
}
|
||||
|
||||
impl Tree {
|
||||
#[inline]
|
||||
fn predict(&self, x: &[f64]) -> f64 {
|
||||
let mut n = 0;
|
||||
while !self.leaf[n] {
|
||||
let v = x[self.feature[n]];
|
||||
let go_left = if v.is_nan() {
|
||||
self.missing_left[n]
|
||||
} else {
|
||||
v <= self.threshold[n]
|
||||
};
|
||||
n = if go_left { self.left[n] } else { self.right[n] };
|
||||
}
|
||||
self.value[n]
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize, PartialEq, Clone, Copy)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum Kind {
|
||||
Regressor,
|
||||
Classifier,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct Head {
|
||||
pub kind: Kind,
|
||||
baseline: f64,
|
||||
trees: Vec<Tree>,
|
||||
}
|
||||
|
||||
impl Head {
|
||||
/// Regressor: the prediction. Classifier: P(class 1), like predict_proba[:, 1].
|
||||
pub fn predict(&self, x: &[f64]) -> f64 {
|
||||
let mut raw = self.baseline;
|
||||
for t in &self.trees {
|
||||
raw += t.predict(x);
|
||||
}
|
||||
match self.kind {
|
||||
Kind::Regressor => raw,
|
||||
Kind::Classifier => 1.0 / (1.0 + (-raw).exp()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct Bundle {
|
||||
pub station_code: String,
|
||||
pub model_version: String,
|
||||
pub regression_target: Option<String>,
|
||||
pub feature_names: Vec<String>,
|
||||
pub horizons: Vec<u32>,
|
||||
pub sigma: BTreeMap<String, f64>,
|
||||
pub thresholds: BTreeMap<String, f64>,
|
||||
pub heads: BTreeMap<String, Head>,
|
||||
}
|
||||
@@ -1,211 +0,0 @@
|
||||
//! Parity + timing check: Rust features and HGB inference vs the Python output
|
||||
//! recorded in data/fixture_*.json.
|
||||
|
||||
mod features;
|
||||
mod hgb;
|
||||
|
||||
use chrono::NaiveDateTime;
|
||||
use serde::Deserialize;
|
||||
use std::collections::HashMap;
|
||||
use std::time::Instant;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct StationObs {
|
||||
water_level: Vec<Option<f64>>,
|
||||
discharge: Vec<Option<f64>>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Fixture {
|
||||
station: String,
|
||||
timestamps: Vec<String>,
|
||||
observed: HashMap<String, StationObs>,
|
||||
rain: Vec<Option<f64>>,
|
||||
feature_names: Vec<String>,
|
||||
features: Vec<Vec<Option<f64>>>,
|
||||
predictions: HashMap<String, Vec<f64>>,
|
||||
}
|
||||
|
||||
fn nanify(v: &[Option<f64>]) -> Vec<f64> {
|
||||
v.iter().map(|x| x.unwrap_or(f64::NAN)).collect()
|
||||
}
|
||||
|
||||
fn same(a: f64, b: f64, tol: f64) -> bool {
|
||||
(a.is_nan() && b.is_nan()) || (a - b).abs() <= tol * (1.0 + b.abs())
|
||||
}
|
||||
|
||||
fn matrix(cols: &[(String, Vec<f64>)], names: &[String]) -> Vec<Vec<f64>> {
|
||||
let by_name: HashMap<&str, &Vec<f64>> = cols.iter().map(|(k, v)| (k.as_str(), v)).collect();
|
||||
let picked: Vec<&Vec<f64>> = names
|
||||
.iter()
|
||||
.map(|n| {
|
||||
*by_name
|
||||
.get(n.as_str())
|
||||
.unwrap_or_else(|| panic!("missing feature {n}"))
|
||||
})
|
||||
.collect();
|
||||
let rows = picked[0].len();
|
||||
(0..rows)
|
||||
.map(|i| picked.iter().map(|c| c[i]).collect())
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let data = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("data");
|
||||
let t = Instant::now();
|
||||
let bundle: hgb::Bundle =
|
||||
serde_json::from_slice(&std::fs::read(data.join("model_P.1.json")).unwrap()).unwrap();
|
||||
println!(
|
||||
"model {} ({}): {} heads, {} features, target={:?}, horizons {:?}, sigma {:?}, thresholds {:?}; loaded in {:?}",
|
||||
bundle.station_code,
|
||||
bundle.model_version,
|
||||
bundle.heads.len(),
|
||||
bundle.feature_names.len(),
|
||||
bundle.regression_target,
|
||||
bundle.horizons,
|
||||
bundle.sigma,
|
||||
bundle.thresholds,
|
||||
t.elapsed()
|
||||
);
|
||||
|
||||
for name in ["flood2024", "recent2026"] {
|
||||
let fx: Fixture = serde_json::from_slice(
|
||||
&std::fs::read(data.join(format!("fixture_{name}.json"))).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(fx.feature_names, bundle.feature_names);
|
||||
let grid = features::Grid {
|
||||
times: fx
|
||||
.timestamps
|
||||
.iter()
|
||||
.map(|s| NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S").unwrap())
|
||||
.collect(),
|
||||
level: fx
|
||||
.observed
|
||||
.iter()
|
||||
.map(|(k, v)| (k.clone(), nanify(&v.water_level)))
|
||||
.collect(),
|
||||
discharge: fx
|
||||
.observed
|
||||
.iter()
|
||||
.map(|(k, v)| (k.clone(), nanify(&v.discharge)))
|
||||
.collect(),
|
||||
};
|
||||
let rain = nanify(&fx.rain);
|
||||
|
||||
// 1. feature parity
|
||||
let t = Instant::now();
|
||||
let cols = features::build_features(&grid, &fx.station, Some(&rain));
|
||||
let feat_time = t.elapsed();
|
||||
let rust_x = matrix(&cols, &bundle.feature_names);
|
||||
let py_x: Vec<Vec<f64>> = fx.features.iter().map(|r| nanify(r)).collect();
|
||||
let (mut cells, mut exact, mut close, mut worst) =
|
||||
(0usize, 0usize, 0usize, (0.0f64, String::new()));
|
||||
for (r, p) in rust_x.iter().zip(&py_x) {
|
||||
for (j, (a, b)) in r.iter().zip(p).enumerate() {
|
||||
cells += 1;
|
||||
if a.to_bits() == b.to_bits() || (a.is_nan() && b.is_nan()) {
|
||||
exact += 1;
|
||||
} else if same(*a, *b, 1e-9) {
|
||||
close += 1;
|
||||
let d = (a - b).abs();
|
||||
if d > worst.0 {
|
||||
worst = (d, bundle.feature_names[j].clone());
|
||||
}
|
||||
} else {
|
||||
panic!(
|
||||
"{name}: feature {} differs: rust {a} vs python {b}",
|
||||
bundle.feature_names[j]
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
println!(
|
||||
"\n[{name}] {} rows x {} features: {exact} bit-identical, {close} within 1e-9 (max |diff| {:.1e} in {}), 0 mismatches; Rust feature build {:?}",
|
||||
rust_x.len(),
|
||||
bundle.feature_names.len(),
|
||||
worst.0,
|
||||
if worst.1.is_empty() { "-" } else { &worst.1 },
|
||||
feat_time
|
||||
);
|
||||
let _ = cells;
|
||||
|
||||
// 2. inference parity, on Python's features and on Rust's own features
|
||||
let t = Instant::now();
|
||||
let mut report = Vec::new();
|
||||
for (head_name, head) in &bundle.heads {
|
||||
let want = &fx.predictions[head_name];
|
||||
let (mut max_py, mut max_rs, mut ident) = (0.0f64, 0.0f64, 0usize);
|
||||
for i in 0..rust_x.len() {
|
||||
let on_py = head.predict(&py_x[i]);
|
||||
let on_rs = head.predict(&rust_x[i]);
|
||||
if on_py.to_bits() == want[i].to_bits() {
|
||||
ident += 1;
|
||||
}
|
||||
max_py = max_py.max((on_py - want[i]).abs());
|
||||
max_rs = max_rs.max((on_rs - want[i]).abs());
|
||||
}
|
||||
report.push(format!(
|
||||
" {head_name:<10} bit-identical {ident}/{} max|diff| on py-features {max_py:.1e}, on rust-features {max_rs:.1e}",
|
||||
rust_x.len()
|
||||
));
|
||||
}
|
||||
let infer_time = t.elapsed();
|
||||
println!("{}", report.join("\n"));
|
||||
println!(
|
||||
" {} heads x {} rows x 2 inference passes in {:?}",
|
||||
bundle.heads.len(),
|
||||
rust_x.len(),
|
||||
infer_time
|
||||
);
|
||||
|
||||
// 3. serving-shaped work: 14-day window, forecast for the last hour
|
||||
let tail = 336.min(grid.times.len());
|
||||
let start = grid.times.len() - tail;
|
||||
let window = features::Grid {
|
||||
times: grid.times[start..].to_vec(),
|
||||
level: grid
|
||||
.level
|
||||
.iter()
|
||||
.map(|(k, v)| (k.clone(), v[start..].to_vec()))
|
||||
.collect(),
|
||||
discharge: grid
|
||||
.discharge
|
||||
.iter()
|
||||
.map(|(k, v)| (k.clone(), v[start..].to_vec()))
|
||||
.collect(),
|
||||
};
|
||||
let w_rain = rain[start..].to_vec();
|
||||
let reps = 200;
|
||||
let t = Instant::now();
|
||||
let mut last = Vec::new();
|
||||
for _ in 0..reps {
|
||||
let cols = features::build_features(&window, &fx.station, Some(&w_rain));
|
||||
let x = matrix(&cols, &bundle.feature_names);
|
||||
let row = x.last().unwrap();
|
||||
last = bundle
|
||||
.heads
|
||||
.iter()
|
||||
.map(|(k, h)| (k.clone(), h.predict(row)))
|
||||
.collect();
|
||||
}
|
||||
let per = t.elapsed() / reps;
|
||||
let level_now = window.level[&fx.station]
|
||||
.iter()
|
||||
.rev()
|
||||
.find(|x| !x.is_nan())
|
||||
.copied()
|
||||
.unwrap();
|
||||
let rise24 = last.iter().find(|(k, _)| k == "max_24").unwrap().1;
|
||||
let warn24 = last.iter().find(|(k, _)| k == "warn_24").unwrap().1;
|
||||
println!(
|
||||
" serving: 336 h window -> features + {} heads for the last hour: {:?} per station; at {} level {:.2} m, predicted 24 h max {:.2} m, warn_24 head p={:.3}",
|
||||
bundle.heads.len(),
|
||||
per,
|
||||
window.times.last().unwrap(),
|
||||
level_now,
|
||||
(rise24 + level_now).max(level_now),
|
||||
warn24
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,162 +0,0 @@
|
||||
-- Northern Thailand Ping River Monitor - PostgreSQL Database Schema
|
||||
-- This script initializes the database tables for water monitoring data
|
||||
|
||||
-- Enable required extensions
|
||||
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
|
||||
|
||||
-- Create schema for better organization
|
||||
CREATE SCHEMA IF NOT EXISTS water_monitor;
|
||||
SET search_path TO water_monitor, public;
|
||||
|
||||
-- Stations table - stores monitoring station information
|
||||
CREATE TABLE IF NOT EXISTS stations (
|
||||
id SERIAL PRIMARY KEY,
|
||||
station_code VARCHAR(10) UNIQUE NOT NULL,
|
||||
thai_name VARCHAR(255) NOT NULL,
|
||||
english_name VARCHAR(255) NOT NULL,
|
||||
latitude DECIMAL(10,8),
|
||||
longitude DECIMAL(11,8),
|
||||
geohash VARCHAR(20),
|
||||
elevation DECIMAL(8,2), -- meters above sea level
|
||||
river_basin VARCHAR(100),
|
||||
province VARCHAR(100),
|
||||
district VARCHAR(100),
|
||||
is_active BOOLEAN DEFAULT true,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
-- Water measurements table - stores time series data
|
||||
CREATE TABLE IF NOT EXISTS water_measurements (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
timestamp TIMESTAMP NOT NULL,
|
||||
station_id INTEGER NOT NULL,
|
||||
water_level NUMERIC(10,3), -- meters
|
||||
discharge NUMERIC(10,2), -- cubic meters per second
|
||||
discharge_percent NUMERIC(5,2), -- percentage of normal discharge
|
||||
status VARCHAR(20) DEFAULT 'active',
|
||||
data_quality VARCHAR(20) DEFAULT 'good', -- good, fair, poor, missing
|
||||
remarks TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (station_id) REFERENCES stations(id) ON DELETE CASCADE,
|
||||
UNIQUE(timestamp, station_id)
|
||||
);
|
||||
|
||||
-- Alert thresholds table - stores warning/danger levels for each station
|
||||
CREATE TABLE IF NOT EXISTS alert_thresholds (
|
||||
id SERIAL PRIMARY KEY,
|
||||
station_id INTEGER NOT NULL,
|
||||
threshold_type VARCHAR(20) NOT NULL, -- 'warning', 'danger', 'critical'
|
||||
water_level_min NUMERIC(10,3),
|
||||
water_level_max NUMERIC(10,3),
|
||||
discharge_min NUMERIC(10,2),
|
||||
discharge_max NUMERIC(10,2),
|
||||
is_active BOOLEAN DEFAULT true,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (station_id) REFERENCES stations(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
-- Data quality log - tracks data collection issues
|
||||
CREATE TABLE IF NOT EXISTS data_quality_log (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
timestamp TIMESTAMP NOT NULL,
|
||||
station_id INTEGER,
|
||||
issue_type VARCHAR(50) NOT NULL, -- 'connection_failed', 'invalid_data', 'missing_data'
|
||||
description TEXT,
|
||||
severity VARCHAR(20) DEFAULT 'info', -- 'info', 'warning', 'error', 'critical'
|
||||
resolved_at TIMESTAMP,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (station_id) REFERENCES stations(id) ON DELETE SET NULL
|
||||
);
|
||||
|
||||
-- Create indexes for better query performance
|
||||
CREATE INDEX IF NOT EXISTS idx_water_measurements_timestamp ON water_measurements(timestamp DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_water_measurements_station_id ON water_measurements(station_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_water_measurements_station_timestamp ON water_measurements(station_id, timestamp DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_water_measurements_status ON water_measurements(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_stations_code ON stations(station_code);
|
||||
CREATE INDEX IF NOT EXISTS idx_stations_active ON stations(is_active);
|
||||
CREATE INDEX IF NOT EXISTS idx_data_quality_timestamp ON data_quality_log(timestamp DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_data_quality_station ON data_quality_log(station_id);
|
||||
|
||||
-- Create a view for latest measurements per station
|
||||
CREATE OR REPLACE VIEW latest_measurements AS
|
||||
SELECT
|
||||
s.id as station_id,
|
||||
s.station_code,
|
||||
s.english_name,
|
||||
s.thai_name,
|
||||
s.latitude,
|
||||
s.longitude,
|
||||
s.province,
|
||||
s.river_basin,
|
||||
m.timestamp,
|
||||
m.water_level,
|
||||
m.discharge,
|
||||
m.discharge_percent,
|
||||
m.status,
|
||||
m.data_quality,
|
||||
CASE
|
||||
WHEN m.timestamp > CURRENT_TIMESTAMP - INTERVAL '2 hours' THEN 'online'
|
||||
WHEN m.timestamp > CURRENT_TIMESTAMP - INTERVAL '24 hours' THEN 'delayed'
|
||||
ELSE 'offline'
|
||||
END as station_status
|
||||
FROM stations s
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT * FROM water_measurements
|
||||
WHERE station_id = s.id
|
||||
ORDER BY timestamp DESC
|
||||
LIMIT 1
|
||||
) m ON true
|
||||
WHERE s.is_active = true
|
||||
ORDER BY s.station_code;
|
||||
|
||||
-- Create a function to update the updated_at timestamp
|
||||
CREATE OR REPLACE FUNCTION update_modified_column()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
NEW.updated_at = CURRENT_TIMESTAMP;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ language 'plpgsql';
|
||||
|
||||
-- Create triggers to automatically update updated_at
|
||||
DROP TRIGGER IF EXISTS update_stations_modtime ON stations;
|
||||
CREATE TRIGGER update_stations_modtime
|
||||
BEFORE UPDATE ON stations
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_modified_column();
|
||||
|
||||
-- Insert sample stations (Northern Thailand Ping River stations)
|
||||
INSERT INTO stations (id, station_code, thai_name, english_name, latitude, longitude, province, river_basin) VALUES
|
||||
(1, 'P.1', 'เชียงใหม่', 'Chiang Mai', 18.7883, 98.9853, 'Chiang Mai', 'Ping River'),
|
||||
(2, 'P.4A', 'ท่าแพ', 'Tha Phae', 18.7875, 99.0045, 'Chiang Mai', 'Ping River'),
|
||||
(3, 'P.12', 'สันป่าตอง', 'San Pa Tong', 18.6167, 98.9500, 'Chiang Mai', 'Ping River'),
|
||||
(4, 'P.20', 'ลำพูน', 'Lamphun', 18.5737, 99.0081, 'Lamphun', 'Ping River'),
|
||||
(5, 'P.30', 'ลี้', 'Li', 17.4833, 99.3000, 'Lamphun', 'Ping River'),
|
||||
(6, 'P.35', 'ป่าซาง', 'Pa Sang', 18.5444, 98.9397, 'Lamphun', 'Ping River'),
|
||||
(7, 'P.67', 'ตาก', 'Tak', 16.8839, 99.1267, 'Tak', 'Ping River'),
|
||||
(8, 'P.75', 'สามเงา', 'Sam Ngao', 17.1019, 99.4644, 'Tak', 'Ping River')
|
||||
ON CONFLICT (id) DO NOTHING;
|
||||
|
||||
-- Insert sample alert thresholds
|
||||
INSERT INTO alert_thresholds (station_id, threshold_type, water_level_min, water_level_max) VALUES
|
||||
(1, 'warning', 4.5, NULL),
|
||||
(1, 'danger', 6.0, NULL),
|
||||
(1, 'critical', 7.5, NULL),
|
||||
(2, 'warning', 4.0, NULL),
|
||||
(2, 'danger', 5.5, NULL),
|
||||
(2, 'critical', 7.0, NULL)
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
-- Grant permissions (adjust as needed for your setup)
|
||||
GRANT USAGE ON SCHEMA water_monitor TO postgres;
|
||||
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA water_monitor TO postgres;
|
||||
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA water_monitor TO postgres;
|
||||
|
||||
-- Optional: Create a read-only user for reporting
|
||||
-- CREATE USER water_monitor_readonly WITH PASSWORD 'readonly_password';
|
||||
-- GRANT USAGE ON SCHEMA water_monitor TO water_monitor_readonly;
|
||||
-- GRANT SELECT ON ALL TABLES IN SCHEMA water_monitor TO water_monitor_readonly;
|
||||
|
||||
COMMIT;
|
||||
+3
-36
@@ -1,38 +1,5 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Northern Thailand Ping River Monitor Package
|
||||
|
||||
A comprehensive real-time water level monitoring system for the Ping River Basin
|
||||
in Northern Thailand, covering Royal Irrigation Department (RID) stations.
|
||||
"""
|
||||
"""Ping River Monitor: the Python side, i.e. the flood-model training and
|
||||
evaluation stack (src/ml) and the archive backfills. The service (collection,
|
||||
forecast, notifications, website/API) is ping-collector in rust/."""
|
||||
|
||||
__version__ = "3.1.3"
|
||||
__author__ = "Ping River Monitor Team"
|
||||
__description__ = "Northern Thailand Ping River Monitoring System"
|
||||
|
||||
from .config import Config
|
||||
from .database_adapters import DatabaseAdapter, create_database_adapter
|
||||
from .exceptions import (
|
||||
APIConnectionError,
|
||||
ConfigurationError,
|
||||
DatabaseConnectionError,
|
||||
DataValidationError,
|
||||
WaterMonitorException,
|
||||
)
|
||||
from .models import DatabaseConfig, StationInfo, WaterMeasurement
|
||||
from .water_scraper_v3 import EnhancedWaterMonitorScraper
|
||||
|
||||
__all__ = [
|
||||
"EnhancedWaterMonitorScraper",
|
||||
"create_database_adapter",
|
||||
"DatabaseAdapter",
|
||||
"Config",
|
||||
"WaterMeasurement",
|
||||
"StationInfo",
|
||||
"DatabaseConfig",
|
||||
"WaterMonitorException",
|
||||
"DatabaseConnectionError",
|
||||
"APIConnectionError",
|
||||
"DataValidationError",
|
||||
"ConfigurationError",
|
||||
]
|
||||
-602
@@ -1,602 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Water Level Alerting System with Matrix Integration
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import html
|
||||
import os
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
import requests
|
||||
|
||||
try:
|
||||
from .config import Config
|
||||
from .database_adapters import create_database_adapter
|
||||
from .logging_config import get_logger
|
||||
except ImportError:
|
||||
import logging
|
||||
|
||||
from config import Config
|
||||
from database_adapters import create_database_adapter
|
||||
|
||||
def get_logger(name):
|
||||
return logging.getLogger(name)
|
||||
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
_BOLD_RE = re.compile(r"\*\*(.+?)\*\*")
|
||||
_URL_RE = re.compile(r"(https?://[^\s<]+)")
|
||||
|
||||
|
||||
def markdown_to_matrix_html(text: str) -> str:
|
||||
"""Convert the small Markdown subset we emit into Matrix-compatible HTML.
|
||||
|
||||
Matrix clients do NOT render Markdown in the plain ``body`` field; formatting
|
||||
only shows when an HTML ``formatted_body`` is sent alongside it. We only use
|
||||
``**bold**``, bare URLs and newlines, so a minimal converter is sufficient and
|
||||
avoids adding a Markdown dependency.
|
||||
"""
|
||||
# Escape HTML special chars first so station/message data can't inject markup.
|
||||
result = html.escape(text, quote=False)
|
||||
result = _BOLD_RE.sub(r"<strong>\1</strong>", result)
|
||||
result = _URL_RE.sub(r'<a href="\1">\1</a>', result)
|
||||
result = result.replace("\n", "<br/>")
|
||||
return result
|
||||
|
||||
|
||||
def strip_markdown(text: str) -> str:
|
||||
"""Produce a clean plain-text fallback for the Matrix ``body`` field."""
|
||||
return _BOLD_RE.sub(r"\1", text)
|
||||
|
||||
|
||||
class AlertLevel(Enum):
|
||||
INFO = "info"
|
||||
WARNING = "warning"
|
||||
CRITICAL = "critical"
|
||||
EMERGENCY = "emergency"
|
||||
|
||||
|
||||
@dataclass
|
||||
class WaterAlert:
|
||||
station_code: str
|
||||
station_name: str
|
||||
alert_type: str
|
||||
level: AlertLevel
|
||||
water_level: float
|
||||
threshold: float
|
||||
discharge: Optional[float] = None
|
||||
timestamp: Optional[datetime.datetime] = None
|
||||
message: Optional[str] = None
|
||||
|
||||
|
||||
class MatrixNotifier:
|
||||
def __init__(self, homeserver: str, access_token: str, room_id: str):
|
||||
self.homeserver = homeserver.rstrip("/")
|
||||
self.access_token = access_token
|
||||
self.room_id = room_id
|
||||
self.session = requests.Session()
|
||||
|
||||
def send_message(
|
||||
self, message: str, msgtype: str = "m.text", markdown: bool = True
|
||||
) -> bool:
|
||||
"""Send a message to the Matrix room.
|
||||
|
||||
When ``markdown`` is True (default) the ``message`` is treated as Markdown:
|
||||
a rendered HTML ``formatted_body`` is sent so clients show real formatting,
|
||||
with a plain-text ``body`` fallback for clients that ignore HTML.
|
||||
"""
|
||||
try:
|
||||
# Add transaction ID to prevent duplicates
|
||||
txn_id = datetime.datetime.now().strftime("%Y%m%d_%H%M%S_%f")
|
||||
url = f"{self.homeserver}/_matrix/client/v3/rooms/{self.room_id}/send/m.room.message/{txn_id}"
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.access_token}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
if markdown:
|
||||
data = {
|
||||
"msgtype": msgtype,
|
||||
"body": strip_markdown(message),
|
||||
"format": "org.matrix.custom.html",
|
||||
"formatted_body": markdown_to_matrix_html(message),
|
||||
}
|
||||
else:
|
||||
data = {"msgtype": msgtype, "body": message}
|
||||
|
||||
# Matrix API requires PUT when transaction ID is in the URL path
|
||||
response = self.session.put(url, headers=headers, json=data, timeout=10)
|
||||
response.raise_for_status()
|
||||
|
||||
logger.info(
|
||||
f"Matrix message sent successfully: {response.json().get('event_id')}"
|
||||
)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to send Matrix message: {e}")
|
||||
return False
|
||||
|
||||
def send_alert(self, alert: WaterAlert) -> bool:
|
||||
"""Send formatted water alert to Matrix"""
|
||||
emoji_map = {
|
||||
AlertLevel.INFO: "ℹ️",
|
||||
AlertLevel.WARNING: "⚠️",
|
||||
AlertLevel.CRITICAL: "🚨",
|
||||
AlertLevel.EMERGENCY: "🆘",
|
||||
}
|
||||
|
||||
emoji = emoji_map.get(alert.level, "📊")
|
||||
|
||||
message = f"""{emoji} **WATER LEVEL ALERT**
|
||||
|
||||
**Station:** {alert.station_code} ({alert.station_name})
|
||||
**Alert Type:** {alert.alert_type}
|
||||
**Severity:** {alert.level.value.upper()}
|
||||
|
||||
**Current Level:** {alert.water_level:.2f}m
|
||||
**Threshold:** {alert.threshold:.2f}m
|
||||
**Difference:** {(alert.water_level - alert.threshold):+.2f}m
|
||||
"""
|
||||
|
||||
if alert.discharge:
|
||||
message += f"**Discharge:** {alert.discharge:.1f} cms\n"
|
||||
|
||||
if alert.timestamp:
|
||||
message += f"**Time:** {alert.timestamp.strftime('%Y-%m-%d %H:%M:%S')}\n"
|
||||
|
||||
if alert.message:
|
||||
message += f"\n**Details:** {alert.message}\n"
|
||||
|
||||
# Add live dashboard link
|
||||
dashboard_url = os.getenv("ALERT_DASHBOARD_URL", "https://water.buildfor.life/")
|
||||
message += f"\n📈 **View Dashboard:** {dashboard_url}"
|
||||
|
||||
return self.send_message(message)
|
||||
|
||||
|
||||
class WaterLevelAlertSystem:
|
||||
# Stations upstream of Chiang Mai (and CNX itself) to monitor
|
||||
UPSTREAM_STATIONS = {
|
||||
"P.20", # Ban Chiang Dao
|
||||
"P.75", # Ban Chai Lat
|
||||
"P.92", # Ban Muang Aut
|
||||
"P.4A", # Ban Mae Taeng
|
||||
"P.67", # Ban Tae
|
||||
"P.21", # Ban Rim Tai
|
||||
"P.103", # Ring Bridge 3
|
||||
"P.1", # Nawarat Bridge (Chiang Mai)
|
||||
}
|
||||
|
||||
def __init__(self):
|
||||
self.db_adapter = None
|
||||
self.matrix_notifier = None
|
||||
self.thresholds = self._load_thresholds()
|
||||
|
||||
# Matrix configuration from environment
|
||||
matrix_homeserver = os.getenv("MATRIX_HOMESERVER", "https://matrix.org")
|
||||
matrix_token = os.getenv("MATRIX_ACCESS_TOKEN")
|
||||
matrix_room = os.getenv("MATRIX_ROOM_ID")
|
||||
|
||||
if matrix_token and matrix_room:
|
||||
self.matrix_notifier = MatrixNotifier(
|
||||
matrix_homeserver, matrix_token, matrix_room
|
||||
)
|
||||
logger.info("Matrix notifications enabled")
|
||||
else:
|
||||
logger.warning("Matrix configuration missing - notifications disabled")
|
||||
|
||||
def _load_thresholds(self) -> Dict[str, Dict[str, float]]:
|
||||
"""Load alert thresholds from config or database"""
|
||||
# Default thresholds for Northern Thailand stations
|
||||
return {
|
||||
"P.1": {
|
||||
# Zone-based thresholds for Nawarat Bridge (P.1)
|
||||
"zone_1": 3.7,
|
||||
"zone_2": 3.9,
|
||||
"zone_3": 4.0,
|
||||
"zone_4": 4.1,
|
||||
"zone_5": 4.2,
|
||||
"zone_6": 4.3,
|
||||
"zone_7": 4.6,
|
||||
"zone_8": 4.8,
|
||||
"newedge": 4.8, # Same as zone 8 or adjust as needed
|
||||
# Keep legacy thresholds for compatibility
|
||||
"warning": 3.7,
|
||||
"critical": 4.3,
|
||||
"emergency": 4.8,
|
||||
},
|
||||
"P.4A": {"warning": 4.5, "critical": 6.0, "emergency": 7.5},
|
||||
"P.20": {"warning": 3.0, "critical": 4.5, "emergency": 6.0},
|
||||
"P.21": {"warning": 4.0, "critical": 5.5, "emergency": 7.0},
|
||||
"P.67": {"warning": 6.0, "critical": 8.0, "emergency": 10.0},
|
||||
"P.75": {"warning": 5.5, "critical": 7.5, "emergency": 9.5},
|
||||
"P.103": {"warning": 7.0, "critical": 9.0, "emergency": 11.0},
|
||||
# Default for unknown stations
|
||||
"default": {"warning": 4.0, "critical": 6.0, "emergency": 8.0},
|
||||
}
|
||||
|
||||
def connect_database(self):
|
||||
"""Initialize database connection"""
|
||||
try:
|
||||
db_config = Config.get_database_config()
|
||||
self.db_adapter = create_database_adapter(
|
||||
db_config["type"], connection_string=db_config["connection_string"]
|
||||
)
|
||||
|
||||
if self.db_adapter.connect():
|
||||
logger.info("Database connection established for alerting")
|
||||
return True
|
||||
else:
|
||||
logger.error("Failed to connect to database")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Database connection error: {e}")
|
||||
return False
|
||||
|
||||
def check_water_levels(self) -> List[WaterAlert]:
|
||||
"""Check current water levels against thresholds"""
|
||||
alerts = []
|
||||
|
||||
if not self.db_adapter:
|
||||
logger.error("Database not connected")
|
||||
return alerts
|
||||
|
||||
try:
|
||||
# Get latest measurements
|
||||
measurements = self.db_adapter.get_latest_measurements(limit=50)
|
||||
|
||||
for measurement in measurements:
|
||||
station_code = measurement.get("station_code", "UNKNOWN")
|
||||
water_level = measurement.get("water_level")
|
||||
|
||||
if not water_level:
|
||||
continue
|
||||
|
||||
# Only alert for upstream stations and Chiang Mai
|
||||
if station_code not in self.UPSTREAM_STATIONS:
|
||||
continue
|
||||
|
||||
# Get thresholds for this station
|
||||
station_thresholds = self.thresholds.get(
|
||||
station_code, self.thresholds["default"]
|
||||
)
|
||||
|
||||
# Check each threshold level
|
||||
alert_level = None
|
||||
threshold_value = None
|
||||
alert_type = None
|
||||
|
||||
# Special handling for P.1 with zone-based thresholds
|
||||
if station_code == "P.1" and "zone_1" in station_thresholds:
|
||||
# Check all zones in reverse order (highest to lowest)
|
||||
zones = [
|
||||
("zone_8", 4.8, AlertLevel.EMERGENCY, "Zone 8 - Emergency"),
|
||||
("newedge", 4.8, AlertLevel.EMERGENCY, "NewEdge Alert Level"),
|
||||
("zone_7", 4.6, AlertLevel.CRITICAL, "Zone 7 - Critical"),
|
||||
("zone_6", 4.3, AlertLevel.CRITICAL, "Zone 6 - Critical"),
|
||||
("zone_5", 4.2, AlertLevel.WARNING, "Zone 5 - Warning"),
|
||||
("zone_4", 4.1, AlertLevel.WARNING, "Zone 4 - Warning"),
|
||||
("zone_3", 4.0, AlertLevel.WARNING, "Zone 3 - Warning"),
|
||||
("zone_2", 3.9, AlertLevel.INFO, "Zone 2 - Info"),
|
||||
("zone_1", 3.7, AlertLevel.INFO, "Zone 1 - Info"),
|
||||
]
|
||||
|
||||
for (
|
||||
zone_name,
|
||||
zone_threshold,
|
||||
zone_alert_level,
|
||||
zone_description,
|
||||
) in zones:
|
||||
if water_level >= zone_threshold:
|
||||
alert_level = zone_alert_level
|
||||
threshold_value = zone_threshold
|
||||
alert_type = zone_description
|
||||
break
|
||||
|
||||
else:
|
||||
# Standard threshold checking for other stations
|
||||
if water_level >= station_thresholds.get("emergency", float("inf")):
|
||||
alert_level = AlertLevel.EMERGENCY
|
||||
threshold_value = station_thresholds["emergency"]
|
||||
alert_type = "Emergency Water Level"
|
||||
elif water_level >= station_thresholds.get(
|
||||
"critical", float("inf")
|
||||
):
|
||||
alert_level = AlertLevel.CRITICAL
|
||||
threshold_value = station_thresholds["critical"]
|
||||
alert_type = "Critical Water Level"
|
||||
elif water_level >= station_thresholds.get("warning", float("inf")):
|
||||
alert_level = AlertLevel.WARNING
|
||||
threshold_value = station_thresholds["warning"]
|
||||
alert_type = "High Water Level"
|
||||
|
||||
if alert_level:
|
||||
alert = WaterAlert(
|
||||
station_code=station_code,
|
||||
station_name=measurement.get(
|
||||
"station_name_th", f"Station {station_code}"
|
||||
),
|
||||
alert_type=alert_type,
|
||||
level=alert_level,
|
||||
water_level=water_level,
|
||||
threshold=threshold_value,
|
||||
discharge=measurement.get("discharge"),
|
||||
timestamp=measurement.get("timestamp"),
|
||||
)
|
||||
alerts.append(alert)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error checking water levels: {e}")
|
||||
|
||||
return alerts
|
||||
|
||||
def check_data_freshness(self, max_age_hours: int = 12) -> List[WaterAlert]:
|
||||
"""Check if data is fresh enough"""
|
||||
alerts = []
|
||||
|
||||
if not self.db_adapter:
|
||||
return alerts
|
||||
|
||||
try:
|
||||
measurements = self.db_adapter.get_latest_measurements(limit=20)
|
||||
cutoff_time = datetime.datetime.now() - datetime.timedelta(
|
||||
hours=max_age_hours
|
||||
)
|
||||
|
||||
for measurement in measurements:
|
||||
timestamp = measurement.get("timestamp")
|
||||
if timestamp and timestamp < cutoff_time:
|
||||
station_code = measurement.get("station_code", "UNKNOWN")
|
||||
|
||||
age_hours = (
|
||||
datetime.datetime.now() - timestamp
|
||||
).total_seconds() / 3600
|
||||
|
||||
alert = WaterAlert(
|
||||
station_code=station_code,
|
||||
station_name=measurement.get(
|
||||
"station_name_th", f"Station {station_code}"
|
||||
),
|
||||
alert_type="Stale Data",
|
||||
level=AlertLevel.WARNING,
|
||||
water_level=measurement.get("water_level", 0),
|
||||
threshold=max_age_hours,
|
||||
timestamp=timestamp,
|
||||
message=f"No fresh data for {age_hours:.1f} hours",
|
||||
)
|
||||
alerts.append(alert)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error checking data freshness: {e}")
|
||||
|
||||
return alerts
|
||||
|
||||
def check_rate_of_change(self, lookback_hours: int = 3) -> List[WaterAlert]:
|
||||
"""Check for rapid water level changes over recent hours"""
|
||||
alerts = []
|
||||
|
||||
if not self.db_adapter:
|
||||
return alerts
|
||||
|
||||
try:
|
||||
# Define rate-of-change thresholds (meters per hour)
|
||||
rate_thresholds = {
|
||||
"P.1": {
|
||||
"warning": 0.15, # 15cm/hour - moderate rise
|
||||
"critical": 0.25, # 25cm/hour - rapid rise
|
||||
"emergency": 0.40, # 40cm/hour - very rapid rise
|
||||
},
|
||||
"default": {"warning": 0.20, "critical": 0.35, "emergency": 0.50},
|
||||
}
|
||||
|
||||
# Get recent measurements for each station
|
||||
cutoff_time = datetime.datetime.now() - datetime.timedelta(
|
||||
hours=lookback_hours
|
||||
)
|
||||
|
||||
# Get unique stations from latest data
|
||||
latest = self.db_adapter.get_latest_measurements(limit=20)
|
||||
station_codes = set(
|
||||
m.get("station_code") for m in latest if m.get("station_code")
|
||||
)
|
||||
|
||||
for station_code in station_codes:
|
||||
try:
|
||||
# Only alert for upstream stations and Chiang Mai
|
||||
if station_code not in self.UPSTREAM_STATIONS:
|
||||
continue
|
||||
|
||||
# Get measurements for this station in the time window
|
||||
current_time = datetime.datetime.now()
|
||||
measurements = self.db_adapter.get_measurements_by_timerange(
|
||||
start_time=cutoff_time,
|
||||
end_time=current_time,
|
||||
station_codes=[station_code],
|
||||
)
|
||||
|
||||
if len(measurements) < 2:
|
||||
continue # Need at least 2 points to calculate rate
|
||||
|
||||
# Sort by timestamp
|
||||
measurements = sorted(
|
||||
measurements, key=lambda m: m.get("timestamp")
|
||||
)
|
||||
|
||||
# Get oldest and newest measurements
|
||||
oldest = measurements[0]
|
||||
newest = measurements[-1]
|
||||
|
||||
oldest_time = oldest.get("timestamp")
|
||||
oldest_level = oldest.get("water_level")
|
||||
newest_time = newest.get("timestamp")
|
||||
newest_level = newest.get("water_level")
|
||||
|
||||
# Convert timestamp strings to datetime if needed
|
||||
if isinstance(oldest_time, str):
|
||||
oldest_time = datetime.datetime.fromisoformat(oldest_time)
|
||||
if isinstance(newest_time, str):
|
||||
newest_time = datetime.datetime.fromisoformat(newest_time)
|
||||
|
||||
# Calculate rate of change
|
||||
time_diff_hours = (newest_time - oldest_time).total_seconds() / 3600
|
||||
if time_diff_hours == 0:
|
||||
continue
|
||||
|
||||
level_change = newest_level - oldest_level
|
||||
rate_per_hour = level_change / time_diff_hours
|
||||
|
||||
# Only alert on rising water (positive rate)
|
||||
if rate_per_hour <= 0:
|
||||
continue
|
||||
|
||||
# Get station info from latest data
|
||||
station_info = next(
|
||||
(m for m in latest if m.get("station_code") == station_code), {}
|
||||
)
|
||||
station_name = station_info.get("station_name_th", station_code)
|
||||
|
||||
# Get thresholds for this station
|
||||
station_rate_threshold = rate_thresholds.get(
|
||||
station_code, rate_thresholds["default"]
|
||||
)
|
||||
|
||||
alert_level = None
|
||||
threshold_value = None
|
||||
alert_type = None
|
||||
|
||||
if rate_per_hour >= station_rate_threshold["emergency"]:
|
||||
alert_level = AlertLevel.EMERGENCY
|
||||
threshold_value = station_rate_threshold["emergency"]
|
||||
alert_type = "Very Rapid Water Level Rise"
|
||||
elif rate_per_hour >= station_rate_threshold["critical"]:
|
||||
alert_level = AlertLevel.CRITICAL
|
||||
threshold_value = station_rate_threshold["critical"]
|
||||
alert_type = "Rapid Water Level Rise"
|
||||
elif rate_per_hour >= station_rate_threshold["warning"]:
|
||||
alert_level = AlertLevel.WARNING
|
||||
threshold_value = station_rate_threshold["warning"]
|
||||
alert_type = "Moderate Water Level Rise"
|
||||
|
||||
if alert_level:
|
||||
message = (
|
||||
f"Rising at {rate_per_hour:.2f}m/h over last {time_diff_hours:.1f}h "
|
||||
f"(change: {level_change:+.2f}m)"
|
||||
)
|
||||
|
||||
alert = WaterAlert(
|
||||
station_code=station_code,
|
||||
station_name=station_name or f"Station {station_code}",
|
||||
alert_type=alert_type,
|
||||
level=alert_level,
|
||||
water_level=newest_level,
|
||||
threshold=threshold_value,
|
||||
timestamp=newest_time,
|
||||
message=message,
|
||||
)
|
||||
alerts.append(alert)
|
||||
|
||||
except Exception as station_error:
|
||||
logger.debug(
|
||||
f"Error checking rate of change for station {station_code}: {station_error}"
|
||||
)
|
||||
continue
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error checking rate of change: {e}")
|
||||
|
||||
return alerts
|
||||
|
||||
def send_alerts(self, alerts: List[WaterAlert]) -> int:
|
||||
"""Send alerts via configured channels"""
|
||||
sent_count = 0
|
||||
|
||||
if not alerts:
|
||||
return sent_count
|
||||
|
||||
if self.matrix_notifier:
|
||||
for alert in alerts:
|
||||
if self.matrix_notifier.send_alert(alert):
|
||||
sent_count += 1
|
||||
|
||||
# Could add other notification channels here:
|
||||
# - Email
|
||||
# - Discord
|
||||
# - Telegram
|
||||
# - SMS
|
||||
|
||||
return sent_count
|
||||
|
||||
def run_alert_check(self) -> Dict[str, int]:
|
||||
"""Run complete alert check cycle"""
|
||||
if not self.connect_database():
|
||||
return {"error": 1}
|
||||
|
||||
# Check water levels
|
||||
water_alerts = self.check_water_levels()
|
||||
|
||||
# Check data freshness
|
||||
data_alerts = self.check_data_freshness()
|
||||
|
||||
# Check rate of change (rapid rises)
|
||||
rate_alerts = self.check_rate_of_change()
|
||||
|
||||
# Combine alerts
|
||||
all_alerts = water_alerts + rate_alerts
|
||||
|
||||
# Send alerts
|
||||
sent_count = self.send_alerts(all_alerts)
|
||||
|
||||
logger.info(
|
||||
f"Alert check complete: {len(all_alerts)} alerts, {sent_count} sent"
|
||||
)
|
||||
|
||||
return {
|
||||
"water_alerts": len(water_alerts),
|
||||
"data_alerts": len(data_alerts),
|
||||
"rate_alerts": len(rate_alerts),
|
||||
"total_alerts": len(all_alerts),
|
||||
"sent": sent_count,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
"""Standalone alerting check"""
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description="Water Level Alert System")
|
||||
parser.add_argument("--check", action="store_true", help="Run alert check")
|
||||
parser.add_argument("--test", action="store_true", help="Send test message")
|
||||
args = parser.parse_args()
|
||||
|
||||
alerting = WaterLevelAlertSystem()
|
||||
|
||||
if args.test:
|
||||
if alerting.matrix_notifier:
|
||||
test_message = (
|
||||
"🧪 **Test Alert**\n\nThis is a test message from the Water Level Alert System.\n\n"
|
||||
"If you received this, Matrix notifications are working correctly!"
|
||||
)
|
||||
success = alerting.matrix_notifier.send_message(test_message)
|
||||
print(f"Test message sent: {success}")
|
||||
else:
|
||||
print("Matrix notifier not configured")
|
||||
|
||||
elif args.check:
|
||||
results = alerting.run_alert_check()
|
||||
print(f"Alert check results: {results}")
|
||||
|
||||
else:
|
||||
print("Use --check or --test")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+31
-342
@@ -1,354 +1,43 @@
|
||||
import os
|
||||
from typing import Any, Dict
|
||||
"""Database settings for the Python tools (training, evaluation, backfills).
|
||||
|
||||
The service itself is ping-collector (rust/collector); it reads the same
|
||||
environment / `.env` file (rust/collector/src/config.rs). PostgreSQL only:
|
||||
POSTGRES_CONNECTION_STRING, or POSTGRES_HOST / POSTGRES_PORT / POSTGRES_DB /
|
||||
POSTGRES_USER / POSTGRES_PASSWORD with the same defaults as ping-collector.
|
||||
"""
|
||||
|
||||
import os
|
||||
import urllib.parse
|
||||
from typing import Optional
|
||||
|
||||
# Load environment variables from .env file
|
||||
try:
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
except ImportError:
|
||||
# python-dotenv not installed, continue without it
|
||||
except ImportError: # the systemd units pass the variables in directly
|
||||
pass
|
||||
|
||||
try:
|
||||
from .exceptions import ConfigurationError
|
||||
from .models import DatabaseType
|
||||
except ImportError:
|
||||
# Handle case when running as standalone script
|
||||
class ConfigurationError(Exception):
|
||||
pass
|
||||
|
||||
from enum import Enum
|
||||
|
||||
class DatabaseType(Enum):
|
||||
SQLITE = "sqlite"
|
||||
MYSQL = "mysql"
|
||||
POSTGRESQL = "postgresql"
|
||||
INFLUXDB = "influxdb"
|
||||
VICTORIAMETRICS = "victoriametrics"
|
||||
|
||||
|
||||
class Config:
|
||||
"""Configuration class for the Water Level Monitor"""
|
||||
|
||||
# Database settings
|
||||
DATABASE_PATH = os.getenv("WATER_DB_PATH", "water_levels.db")
|
||||
|
||||
# Website settings
|
||||
TARGET_URL = "https://hyd-app-db.rid.go.th/hydro1h.html"
|
||||
API_URL = "https://hyd-app-db.rid.go.th/webservice/getGroupHourlyWaterLevelReportAllHL.ashx"
|
||||
THAIWATER_API_KEY = os.getenv("THAIWATER_API_KEY")
|
||||
|
||||
# Public flood notifications (ntfy). Off unless NTFY_SERVER is set.
|
||||
# NTFY_SERVER is what subscribers use (public https URL, shown on the
|
||||
# dashboard). NTFY_PUBLISH_URL is where the monitor POSTs; defaults to
|
||||
# NTFY_SERVER, set it to http://127.0.0.1:2586 when ntfy runs on the same
|
||||
# host so publishing never depends on DNS/proxy/tunnel being up.
|
||||
NTFY_SERVER = os.getenv("NTFY_SERVER", "").strip()
|
||||
NTFY_PUBLISH_URL = os.getenv("NTFY_PUBLISH_URL", "").strip() or NTFY_SERVER
|
||||
NTFY_TOPIC_PREFIX = os.getenv("NTFY_TOPIC_PREFIX", "ping").strip()
|
||||
NTFY_TOKEN = os.getenv("NTFY_TOKEN", "").strip() # publish token if ACL enabled
|
||||
PUBLIC_URL = os.getenv("PUBLIC_URL", "https://water.buildfor.life/").strip()
|
||||
REQUEST_TIMEOUT = int(os.getenv("REQUEST_TIMEOUT", "30"))
|
||||
USER_AGENT = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
|
||||
def database_url() -> Optional[str]:
|
||||
"""The PostgreSQL connection string, or None when none is configured."""
|
||||
url = os.getenv("POSTGRES_CONNECTION_STRING", "").strip()
|
||||
if url:
|
||||
return url
|
||||
password = os.getenv("POSTGRES_PASSWORD")
|
||||
if not password:
|
||||
return None
|
||||
user = urllib.parse.quote(os.getenv("POSTGRES_USER", "postgres"), safe="")
|
||||
host = os.getenv("POSTGRES_HOST", "localhost")
|
||||
port = os.getenv("POSTGRES_PORT", "5432")
|
||||
name = os.getenv("POSTGRES_DB", "water_monitoring")
|
||||
return (
|
||||
f"postgresql://{user}:{urllib.parse.quote(password, safe='')}"
|
||||
f"@{host}:{port}/{name}"
|
||||
)
|
||||
|
||||
# Database configuration
|
||||
# When DB_TYPE is not set explicitly, a configured Postgres connection wins over the sqlite default
|
||||
DB_TYPE = os.getenv(
|
||||
"DB_TYPE",
|
||||
"postgresql" if os.getenv("POSTGRES_CONNECTION_STRING") else "sqlite",
|
||||
).lower()
|
||||
|
||||
# VictoriaMetrics settings
|
||||
# Default to localhost; set VM_HOST in the environment for real deployments
|
||||
# (avoids committing infrastructure hostnames to the repo).
|
||||
VM_HOST = os.getenv("VM_HOST", "localhost")
|
||||
VM_PORT = int(os.getenv("VM_PORT", "443"))
|
||||
|
||||
# Support for HTTPS URLs (e.g., behind reverse proxy)
|
||||
VM_URL = os.getenv("VM_URL") # Full URL override (e.g., https://vm.example.com)
|
||||
|
||||
# InfluxDB settings
|
||||
INFLUX_HOST = os.getenv("INFLUX_HOST", "localhost")
|
||||
INFLUX_PORT = int(os.getenv("INFLUX_PORT", "8086"))
|
||||
INFLUX_DATABASE = os.getenv("INFLUX_DATABASE", "water_monitoring")
|
||||
INFLUX_USERNAME = os.getenv("INFLUX_USERNAME")
|
||||
INFLUX_PASSWORD = os.getenv("INFLUX_PASSWORD")
|
||||
|
||||
# PostgreSQL settings
|
||||
POSTGRES_CONNECTION_STRING = os.getenv("POSTGRES_CONNECTION_STRING")
|
||||
POSTGRES_HOST = os.getenv("POSTGRES_HOST", "localhost")
|
||||
POSTGRES_PORT = int(os.getenv("POSTGRES_PORT", "5432"))
|
||||
POSTGRES_DB = os.getenv("POSTGRES_DB", "water_monitoring")
|
||||
POSTGRES_USER = os.getenv("POSTGRES_USER", "postgres")
|
||||
POSTGRES_PASSWORD = os.getenv("POSTGRES_PASSWORD")
|
||||
|
||||
# MySQL settings
|
||||
MYSQL_CONNECTION_STRING = os.getenv("MYSQL_CONNECTION_STRING")
|
||||
|
||||
# HII/ThaiWater open api-v3 collection (rainfall + backup water level)
|
||||
# See docs/DATA_SOURCES.md. Requires a SQL DB_TYPE (sqlite/postgresql/mysql).
|
||||
ENABLE_HII_COLLECTION = os.getenv("ENABLE_HII_COLLECTION", "true").lower() in (
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
)
|
||||
HII_BASIN_CODE = int(os.getenv("HII_BASIN_CODE", "6")) # 6 = Ping Basin
|
||||
|
||||
# RID large-dam daily status (app.rid.go.th/reservoir) — Mae Ngat et al.
|
||||
ENABLE_RESERVOIR_COLLECTION = os.getenv(
|
||||
"ENABLE_RESERVOIR_COLLECTION", "true"
|
||||
).lower() in ("1", "true", "yes")
|
||||
# TTL for the /api/hii/*/latest response cache; source data changes hourly
|
||||
HII_CACHE_TTL_SECONDS = int(os.getenv("HII_CACHE_TTL_SECONDS", "120"))
|
||||
# TTL for the /measurements/latest response cache (hottest endpoint)
|
||||
LATEST_CACHE_TTL_SECONDS = int(os.getenv("LATEST_CACHE_TTL_SECONDS", "45"))
|
||||
|
||||
# TTL for /health check results (includes an external RID-API probe)
|
||||
HEALTH_CACHE_TTL_SECONDS = int(os.getenv("HEALTH_CACHE_TTL_SECONDS", "30"))
|
||||
|
||||
# Thread-pool size for blocking work in the web process (DB queries,
|
||||
# inference, health probes). Waiting threads are cheap; starving the pool
|
||||
# stalls every endpoint that needs a thread.
|
||||
EXECUTOR_THREADS = int(os.getenv("EXECUTOR_THREADS", "48"))
|
||||
|
||||
# Web server worker processes. Above 1, uvicorn forks workers and a
|
||||
# localhost lock port elects a single background-collection leader.
|
||||
WEB_WORKERS = int(os.getenv("WEB_WORKERS", "2"))
|
||||
COLLECTION_LEADER_PORT = int(os.getenv("COLLECTION_LEADER_PORT", "8901"))
|
||||
|
||||
# Umami analytics (self-hosted). The website id is public (it ships in the
|
||||
# dashboard <script> tag); server-side API tracking posts to /api/send.
|
||||
UMAMI_API_URL = os.getenv("UMAMI_API_URL", "https://stats.buildfor.life/api/send")
|
||||
UMAMI_WEBSITE_ID = os.getenv(
|
||||
"UMAMI_WEBSITE_ID", "00b2be73-8f5f-4400-9029-3be852eb08f7"
|
||||
)
|
||||
UMAMI_TRACK_API = os.getenv("UMAMI_TRACK_API", "true").lower() in (
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
)
|
||||
|
||||
# Scheduler settings
|
||||
SCRAPING_INTERVAL_HOURS = int(os.getenv("SCRAPING_INTERVAL_HOURS", "1"))
|
||||
# RID is polled this many minutes past each hour; while the current hour is
|
||||
# not published yet (the evening report often lags 1-2 h) it is retried
|
||||
# every RID_RETRY_MINUTES. Late readings are stored either way.
|
||||
RID_POLL_OFFSET_MINUTES = int(os.getenv("RID_POLL_OFFSET_MINUTES", "5"))
|
||||
RID_RETRY_MINUTES = int(os.getenv("RID_RETRY_MINUTES", "10"))
|
||||
# On startup the collection leader re-fetches RID reports with holes from
|
||||
# the last N days (0 = off). Older history: `run.py --fill-gaps all`.
|
||||
GAP_FILL_DAYS_ON_START = int(os.getenv("GAP_FILL_DAYS_ON_START", "14"))
|
||||
# Leader -> other workers hand-off for /scraping/status (default: temp dir).
|
||||
# With ping-collector, point both services at the same file inside the app
|
||||
# dir (the systemd units use PrivateTmp).
|
||||
SCRAPING_STATUS_FILE = os.getenv("SCRAPING_STATUS_FILE", "")
|
||||
# false: data collection runs in the Rust ping-collector service
|
||||
# (rust/collector); this app then only serves, forecasts and notifies,
|
||||
# re-forecasting when the newest RID reading advances (polled every
|
||||
# FORECAST_POLL_SECONDS) and at least once per clock hour.
|
||||
COLLECTION_ENABLED = os.getenv("COLLECTION_ENABLED", "true").lower() in (
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
)
|
||||
FORECAST_POLL_SECONDS = int(os.getenv("FORECAST_POLL_SECONDS", "60"))
|
||||
# true: ping-collector also forecasts (rust/forecast) and sends the ntfy
|
||||
# notifications; this app serves its stored forecast (forecast_current)
|
||||
# and does neither. Read by both services from the shared .env.
|
||||
FORECAST_IN_COLLECTOR = os.getenv("FORECAST_IN_COLLECTOR", "false").lower() in (
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
)
|
||||
# A stored forecast older than this is not served; /forecast then computes
|
||||
# in-process as before (the collector is down or has no models).
|
||||
FORECAST_MAX_AGE_SECONDS = int(os.getenv("FORECAST_MAX_AGE_SECONDS", "7200"))
|
||||
|
||||
# Logging settings
|
||||
LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO")
|
||||
LOG_FILE = os.getenv("LOG_FILE", "water_monitor.log")
|
||||
LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s"
|
||||
|
||||
# Data retention
|
||||
DATA_RETENTION_DAYS = int(os.getenv("DATA_RETENTION_DAYS", "365"))
|
||||
|
||||
# Retry settings
|
||||
MAX_RETRIES = int(os.getenv("MAX_RETRIES", "3"))
|
||||
RETRY_DELAY_SECONDS = int(os.getenv("RETRY_DELAY_SECONDS", "60"))
|
||||
|
||||
# Station configuration
|
||||
# Runtime-writable JSON file that persists the station mapping across restarts
|
||||
# (station CRUD via the API writes here). If it does not exist, the bundled
|
||||
# defaults in src/data/stations.json are used to seed it.
|
||||
STATION_CONFIG_PATH = os.getenv("STATION_CONFIG_PATH", "stations.json")
|
||||
|
||||
# Web API / CORS settings
|
||||
# Comma-separated list of allowed origins. Defaults to none (same-origin only);
|
||||
# set CORS_ALLOW_ORIGINS to a specific list of front-end origins in production.
|
||||
# Credentials are only enabled when explicit (non-wildcard) origins are set,
|
||||
# because "*" + credentials is rejected by browsers and unsafe.
|
||||
CORS_ALLOW_ORIGINS = [
|
||||
origin.strip()
|
||||
for origin in os.getenv("CORS_ALLOW_ORIGINS", "").split(",")
|
||||
if origin.strip()
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def validate_config(cls) -> bool:
|
||||
"""Validate configuration settings"""
|
||||
errors = []
|
||||
|
||||
# Validate database type
|
||||
try:
|
||||
DatabaseType(cls.DB_TYPE)
|
||||
except ValueError:
|
||||
errors.append(f"Invalid DB_TYPE: {cls.DB_TYPE}")
|
||||
|
||||
# Validate database-specific settings
|
||||
if cls.DB_TYPE == "victoriametrics":
|
||||
if not cls.VM_HOST:
|
||||
errors.append("VM_HOST is required for VictoriaMetrics")
|
||||
if not isinstance(cls.VM_PORT, int) or cls.VM_PORT <= 0:
|
||||
errors.append("VM_PORT must be a positive integer")
|
||||
|
||||
elif cls.DB_TYPE == "influxdb":
|
||||
if not cls.INFLUX_HOST:
|
||||
errors.append("INFLUX_HOST is required for InfluxDB")
|
||||
if not cls.INFLUX_DATABASE:
|
||||
errors.append("INFLUX_DATABASE is required for InfluxDB")
|
||||
|
||||
elif cls.DB_TYPE in ["postgresql", "mysql"]:
|
||||
if cls.DB_TYPE == "postgresql":
|
||||
# Check if either connection string or individual components are provided
|
||||
if not cls.POSTGRES_CONNECTION_STRING:
|
||||
# If no connection string, check individual components
|
||||
if not cls.POSTGRES_HOST:
|
||||
errors.append("POSTGRES_HOST is required for PostgreSQL")
|
||||
if not cls.POSTGRES_USER:
|
||||
errors.append("POSTGRES_USER is required for PostgreSQL")
|
||||
if not cls.POSTGRES_PASSWORD:
|
||||
errors.append("POSTGRES_PASSWORD is required for PostgreSQL")
|
||||
if not cls.POSTGRES_DB:
|
||||
errors.append("POSTGRES_DB is required for PostgreSQL")
|
||||
else: # mysql
|
||||
if not cls.MYSQL_CONNECTION_STRING:
|
||||
errors.append("MYSQL_CONNECTION_STRING is required for MySQL")
|
||||
|
||||
# Validate numeric settings
|
||||
if cls.SCRAPING_INTERVAL_HOURS <= 0:
|
||||
errors.append("SCRAPING_INTERVAL_HOURS must be positive")
|
||||
|
||||
if not 0 <= cls.RID_POLL_OFFSET_MINUTES < 60:
|
||||
errors.append("RID_POLL_OFFSET_MINUTES must be between 0 and 59")
|
||||
|
||||
if cls.RID_RETRY_MINUTES <= 0:
|
||||
errors.append("RID_RETRY_MINUTES must be positive")
|
||||
|
||||
if cls.DATA_RETENTION_DAYS <= 0:
|
||||
errors.append("DATA_RETENTION_DAYS must be positive")
|
||||
|
||||
if errors:
|
||||
raise ConfigurationError(f"Configuration errors: {'; '.join(errors)}")
|
||||
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def get_database_config(cls) -> Dict[str, Any]:
|
||||
"""Returns database configuration based on DB_TYPE"""
|
||||
if cls.DB_TYPE == "victoriametrics":
|
||||
return {"type": "victoriametrics", "host": cls.VM_HOST, "port": cls.VM_PORT}
|
||||
elif cls.DB_TYPE == "influxdb":
|
||||
return {
|
||||
"type": "influxdb",
|
||||
"host": cls.INFLUX_HOST,
|
||||
"port": cls.INFLUX_PORT,
|
||||
"database": cls.INFLUX_DATABASE,
|
||||
"username": cls.INFLUX_USERNAME,
|
||||
"password": cls.INFLUX_PASSWORD,
|
||||
}
|
||||
elif cls.DB_TYPE == "postgresql":
|
||||
# Use individual components if POSTGRES_CONNECTION_STRING is not provided
|
||||
if cls.POSTGRES_CONNECTION_STRING:
|
||||
return {
|
||||
"type": "postgresql",
|
||||
"connection_string": cls.POSTGRES_CONNECTION_STRING,
|
||||
}
|
||||
else:
|
||||
# Build connection string from components (automatically URL-encodes password)
|
||||
import urllib.parse
|
||||
|
||||
if not cls.POSTGRES_PASSWORD:
|
||||
raise ConfigurationError(
|
||||
"POSTGRES_PASSWORD is required for PostgreSQL (no default is provided)"
|
||||
)
|
||||
password = urllib.parse.quote(cls.POSTGRES_PASSWORD, safe="")
|
||||
connection_string = (
|
||||
f"postgresql://{cls.POSTGRES_USER}:{password}"
|
||||
f"@{cls.POSTGRES_HOST}:{cls.POSTGRES_PORT}/{cls.POSTGRES_DB}"
|
||||
)
|
||||
return {"type": "postgresql", "connection_string": connection_string}
|
||||
elif cls.DB_TYPE == "mysql":
|
||||
if not cls.MYSQL_CONNECTION_STRING:
|
||||
raise ConfigurationError(
|
||||
"MYSQL_CONNECTION_STRING is required for MySQL (no default is provided)"
|
||||
)
|
||||
return {"type": "mysql", "connection_string": cls.MYSQL_CONNECTION_STRING}
|
||||
else: # sqlite
|
||||
return {
|
||||
"type": "sqlite",
|
||||
"connection_string": f"sqlite:///{cls.DATABASE_PATH}",
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def get_all_settings(cls) -> Dict[str, Any]:
|
||||
"""Returns all configuration settings"""
|
||||
return {
|
||||
"DB_TYPE": cls.DB_TYPE,
|
||||
"DATABASE_PATH": cls.DATABASE_PATH,
|
||||
"TARGET_URL": cls.TARGET_URL,
|
||||
"API_URL": cls.API_URL,
|
||||
"REQUEST_TIMEOUT": cls.REQUEST_TIMEOUT,
|
||||
"SCRAPING_INTERVAL_HOURS": cls.SCRAPING_INTERVAL_HOURS,
|
||||
"RID_POLL_OFFSET_MINUTES": cls.RID_POLL_OFFSET_MINUTES,
|
||||
"RID_RETRY_MINUTES": cls.RID_RETRY_MINUTES,
|
||||
"GAP_FILL_DAYS_ON_START": cls.GAP_FILL_DAYS_ON_START,
|
||||
"COLLECTION_ENABLED": cls.COLLECTION_ENABLED,
|
||||
"FORECAST_IN_COLLECTOR": cls.FORECAST_IN_COLLECTOR,
|
||||
"LOG_LEVEL": cls.LOG_LEVEL,
|
||||
"LOG_FILE": cls.LOG_FILE,
|
||||
"DATA_RETENTION_DAYS": cls.DATA_RETENTION_DAYS,
|
||||
"MAX_RETRIES": cls.MAX_RETRIES,
|
||||
"RETRY_DELAY_SECONDS": cls.RETRY_DELAY_SECONDS,
|
||||
"VM_HOST": cls.VM_HOST,
|
||||
"VM_PORT": cls.VM_PORT,
|
||||
"INFLUX_HOST": cls.INFLUX_HOST,
|
||||
"INFLUX_PORT": cls.INFLUX_PORT,
|
||||
"INFLUX_DATABASE": cls.INFLUX_DATABASE,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def print_settings(cls):
|
||||
"""Prints all current settings"""
|
||||
print("=== Water Level Monitor Configuration ===")
|
||||
for key, value in cls.get_all_settings().items():
|
||||
# Hide sensitive information
|
||||
if "PASSWORD" in key and value:
|
||||
value = "*" * len(str(value))
|
||||
print(f"{key}: {value}")
|
||||
print("=" * 45)
|
||||
|
||||
print("\nDatabase Configuration:")
|
||||
db_config = cls.get_database_config()
|
||||
for key, value in db_config.items():
|
||||
if "password" in key and value:
|
||||
value = "*" * len(str(value))
|
||||
print(f" {key}: {value}")
|
||||
print("=" * 45)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
Config.print_settings()
|
||||
url = database_url()
|
||||
if url and "@" in url:
|
||||
url = url.split("://", 1)[0] + "://***@" + url.rsplit("@", 1)[1]
|
||||
print(url or "no database configured")
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,352 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Demo script showing different database backend options for water monitoring
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import os
|
||||
import sys
|
||||
|
||||
from water_scraper_v3 import EnhancedWaterMonitorScraper
|
||||
|
||||
|
||||
def demo_sqlite():
|
||||
"""Demo with SQLite (local development)"""
|
||||
print("=" * 60)
|
||||
print("🗄️ SQLite Demo (Local Development)")
|
||||
print("=" * 60)
|
||||
|
||||
config = {"type": "sqlite", "connection_string": "sqlite:///demo_water_sqlite.db"}
|
||||
|
||||
try:
|
||||
scraper = EnhancedWaterMonitorScraper(config)
|
||||
|
||||
# Fetch and save data
|
||||
print("Fetching data from API...")
|
||||
data = scraper.fetch_water_data()
|
||||
|
||||
if data:
|
||||
print(f"✓ Fetched {len(data)} data points")
|
||||
success = scraper.save_to_database(data)
|
||||
|
||||
if success:
|
||||
print("✓ Data saved to SQLite database")
|
||||
|
||||
# Show latest data
|
||||
latest = scraper.get_latest_data(5)
|
||||
print(f"\nLatest 5 measurements:")
|
||||
for measurement in latest:
|
||||
print(
|
||||
f" • {measurement['station_code']} ({measurement['station_name_en']}): "
|
||||
f"{measurement['water_level']:.2f}m, {measurement['discharge']:.1f} cms"
|
||||
)
|
||||
else:
|
||||
print("✗ Failed to save data")
|
||||
else:
|
||||
print("✗ No data fetched")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
|
||||
|
||||
def demo_influxdb():
|
||||
"""Demo with InfluxDB (requires InfluxDB running)"""
|
||||
print("\n" + "=" * 60)
|
||||
print("📊 InfluxDB Demo (Time-Series Database)")
|
||||
print("=" * 60)
|
||||
|
||||
config = {
|
||||
"type": "influxdb",
|
||||
"host": "localhost",
|
||||
"port": 8086,
|
||||
"database": "water_monitoring_demo",
|
||||
"username": None, # Set if authentication is enabled
|
||||
"password": None,
|
||||
}
|
||||
|
||||
try:
|
||||
scraper = EnhancedWaterMonitorScraper(config)
|
||||
|
||||
if scraper.db_adapter and scraper.db_adapter.client:
|
||||
print("✓ Connected to InfluxDB")
|
||||
|
||||
# Fetch and save data
|
||||
print("Fetching data from API...")
|
||||
data = scraper.fetch_water_data()
|
||||
|
||||
if data:
|
||||
print(f"✓ Fetched {len(data)} data points")
|
||||
success = scraper.save_to_database(data)
|
||||
|
||||
if success:
|
||||
print("✓ Data saved to InfluxDB")
|
||||
print("💡 You can now query this data in Grafana or InfluxDB CLI")
|
||||
print(
|
||||
" Example query: SELECT * FROM water_data ORDER BY time DESC LIMIT 10"
|
||||
)
|
||||
else:
|
||||
print("✗ Failed to save data")
|
||||
else:
|
||||
print("✗ No data fetched")
|
||||
else:
|
||||
print("✗ Could not connect to InfluxDB")
|
||||
print(
|
||||
"💡 Make sure InfluxDB is running: docker run -p 8086:8086 influxdb:1.8"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
print("💡 InfluxDB might not be running or accessible")
|
||||
|
||||
|
||||
def demo_postgresql():
|
||||
"""Demo with PostgreSQL (requires PostgreSQL running)"""
|
||||
print("\n" + "=" * 60)
|
||||
print("🐘 PostgreSQL Demo (Relational Database)")
|
||||
print("=" * 60)
|
||||
|
||||
config = {
|
||||
"type": "postgresql",
|
||||
"connection_string": "postgresql://postgres:password@localhost:5432/water_monitoring",
|
||||
}
|
||||
|
||||
try:
|
||||
scraper = EnhancedWaterMonitorScraper(config)
|
||||
|
||||
if scraper.db_adapter and scraper.db_adapter.engine:
|
||||
print("✓ Connected to PostgreSQL")
|
||||
|
||||
# Fetch and save data
|
||||
print("Fetching data from API...")
|
||||
data = scraper.fetch_water_data()
|
||||
|
||||
if data:
|
||||
print(f"✓ Fetched {len(data)} data points")
|
||||
success = scraper.save_to_database(data)
|
||||
|
||||
if success:
|
||||
print("✓ Data saved to PostgreSQL")
|
||||
print("💡 You can now query this data with SQL")
|
||||
print(
|
||||
" Example: SELECT * FROM water_measurements ORDER BY timestamp DESC LIMIT 10;"
|
||||
)
|
||||
else:
|
||||
print("✗ Failed to save data")
|
||||
else:
|
||||
print("✗ No data fetched")
|
||||
else:
|
||||
print("✗ Could not connect to PostgreSQL")
|
||||
print("💡 Make sure PostgreSQL is running with correct credentials")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
print("💡 PostgreSQL might not be running or credentials might be wrong")
|
||||
|
||||
|
||||
def demo_mysql():
|
||||
"""Demo with MySQL (requires MySQL running)"""
|
||||
print("\n" + "=" * 60)
|
||||
print("🐬 MySQL Demo (Relational Database)")
|
||||
print("=" * 60)
|
||||
|
||||
config = {
|
||||
"type": "mysql",
|
||||
"connection_string": "mysql://root:password@localhost:3306/water_monitoring",
|
||||
}
|
||||
|
||||
try:
|
||||
scraper = EnhancedWaterMonitorScraper(config)
|
||||
|
||||
if scraper.db_adapter and scraper.db_adapter.engine:
|
||||
print("✓ Connected to MySQL")
|
||||
|
||||
# Fetch and save data
|
||||
print("Fetching data from API...")
|
||||
data = scraper.fetch_water_data()
|
||||
|
||||
if data:
|
||||
print(f"✓ Fetched {len(data)} data points")
|
||||
success = scraper.save_to_database(data)
|
||||
|
||||
if success:
|
||||
print("✓ Data saved to MySQL")
|
||||
print("💡 You can now query this data with SQL")
|
||||
print(
|
||||
" Example: SELECT * FROM water_measurements ORDER BY timestamp DESC LIMIT 10;"
|
||||
)
|
||||
else:
|
||||
print("✗ Failed to save data")
|
||||
else:
|
||||
print("✗ No data fetched")
|
||||
else:
|
||||
print("✗ Could not connect to MySQL")
|
||||
print("💡 Make sure MySQL is running with correct credentials")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
print("💡 MySQL might not be running or credentials might be wrong")
|
||||
|
||||
|
||||
def demo_victoriametrics():
|
||||
"""Demo with VictoriaMetrics (supports both local and HTTPS configurations)"""
|
||||
print("\n" + "=" * 60)
|
||||
print("⚡ VictoriaMetrics Demo (High-Performance Metrics)")
|
||||
print("=" * 60)
|
||||
|
||||
# Use configuration from environment or config.py
|
||||
from config import Config
|
||||
|
||||
db_config = Config.get_database_config()
|
||||
|
||||
if db_config["type"] != "victoriametrics":
|
||||
# Fallback to default local configuration
|
||||
config = {"type": "victoriametrics", "host": "vm.newedge.house", "port": 443}
|
||||
else:
|
||||
config = db_config
|
||||
|
||||
print(f"Connecting to: {config['host']}:{config['port']}")
|
||||
|
||||
try:
|
||||
scraper = EnhancedWaterMonitorScraper(config)
|
||||
|
||||
if scraper.db_adapter:
|
||||
# Test connection using the adapter's connect method
|
||||
if scraper.db_adapter.connect():
|
||||
print("✓ Connected to VictoriaMetrics")
|
||||
|
||||
# Fetch and save data
|
||||
print("Fetching data from API...")
|
||||
data = scraper.fetch_water_data()
|
||||
|
||||
if data:
|
||||
print(f"✓ Fetched {len(data)} data points")
|
||||
success = scraper.save_to_database(data)
|
||||
|
||||
if success:
|
||||
print("✓ Data saved to VictoriaMetrics")
|
||||
print("💡 You can now query this data via Prometheus API")
|
||||
|
||||
# Show appropriate query URL based on configuration
|
||||
base_url = scraper.db_adapter.base_url
|
||||
print(f" Example: {base_url}/api/v1/query?query=water_level")
|
||||
print(f" Health check: {base_url}/health")
|
||||
else:
|
||||
print("✗ Failed to save data")
|
||||
else:
|
||||
print("✗ No data fetched")
|
||||
else:
|
||||
print("✗ Could not connect to VictoriaMetrics")
|
||||
if config["host"] == "localhost":
|
||||
print("💡 Make sure VictoriaMetrics is running locally:")
|
||||
print(" docker run -p 8428:8428 victoriametrics/victoria-metrics")
|
||||
else:
|
||||
print(
|
||||
f"💡 Check if VictoriaMetrics is accessible at {config['host']}:{config['port']}"
|
||||
)
|
||||
print("💡 Verify HTTPS configuration and network connectivity")
|
||||
else:
|
||||
print("✗ Failed to initialize VictoriaMetrics adapter")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
print("💡 Check your VictoriaMetrics configuration and network connectivity")
|
||||
|
||||
|
||||
def show_recommendations():
|
||||
"""Show database recommendations"""
|
||||
print("\n" + "=" * 60)
|
||||
print("🏆 Database Recommendations")
|
||||
print("=" * 60)
|
||||
|
||||
recommendations = [
|
||||
{
|
||||
"name": "InfluxDB",
|
||||
"best_for": "Time-series data, Grafana dashboards",
|
||||
"pros": [
|
||||
"Purpose-built for time-series",
|
||||
"Great compression",
|
||||
"Built-in retention",
|
||||
],
|
||||
"cons": ["Learning curve", "Less flexible for complex queries"],
|
||||
"use_case": "Recommended for most water monitoring deployments",
|
||||
},
|
||||
{
|
||||
"name": "PostgreSQL + TimescaleDB",
|
||||
"best_for": "Complex queries, existing PostgreSQL infrastructure",
|
||||
"pros": ["Mature ecosystem", "SQL compatibility", "ACID compliance"],
|
||||
"cons": ["More complex setup", "Higher resource usage"],
|
||||
"use_case": "Best for organizations already using PostgreSQL",
|
||||
},
|
||||
{
|
||||
"name": "VictoriaMetrics",
|
||||
"best_for": "High-performance metrics, Prometheus compatibility",
|
||||
"pros": ["Extremely fast", "Low resource usage", "Better compression"],
|
||||
"cons": ["Newer ecosystem", "Less tooling"],
|
||||
"use_case": "Best for high-volume, performance-critical deployments",
|
||||
},
|
||||
{
|
||||
"name": "MySQL",
|
||||
"best_for": "Existing MySQL infrastructure, familiar SQL",
|
||||
"pros": ["Familiar", "Mature", "Wide support"],
|
||||
"cons": ["Not optimized for time-series", "Manual optimization needed"],
|
||||
"use_case": "Good for organizations with existing MySQL expertise",
|
||||
},
|
||||
]
|
||||
|
||||
for rec in recommendations:
|
||||
print(f"\n📊 {rec['name']}")
|
||||
print(f" Best for: {rec['best_for']}")
|
||||
print(f" Pros: {', '.join(rec['pros'])}")
|
||||
print(f" Cons: {', '.join(rec['cons'])}")
|
||||
print(f" 💡 {rec['use_case']}")
|
||||
|
||||
|
||||
def main():
|
||||
"""Main demo function"""
|
||||
print("🌊 Thailand Water Monitor - Database Backend Demo")
|
||||
print("This demo shows how to use different database backends")
|
||||
|
||||
# Always run SQLite demo (no external dependencies)
|
||||
demo_sqlite()
|
||||
|
||||
# Check for command line arguments to run specific demos
|
||||
if len(sys.argv) > 1:
|
||||
db_type = sys.argv[1].lower()
|
||||
|
||||
if db_type == "influxdb":
|
||||
demo_influxdb()
|
||||
elif db_type == "postgresql":
|
||||
demo_postgresql()
|
||||
elif db_type == "mysql":
|
||||
demo_mysql()
|
||||
elif db_type == "victoriametrics":
|
||||
demo_victoriametrics()
|
||||
elif db_type == "all":
|
||||
demo_influxdb()
|
||||
demo_postgresql()
|
||||
demo_mysql()
|
||||
demo_victoriametrics()
|
||||
else:
|
||||
print(f"\nUnknown database type: {db_type}")
|
||||
print(
|
||||
"Available options: influxdb, postgresql, mysql, victoriametrics, all"
|
||||
)
|
||||
else:
|
||||
print("\n💡 To test other databases, run:")
|
||||
print(" python demo_databases.py influxdb")
|
||||
print(" python demo_databases.py postgresql")
|
||||
print(" python demo_databases.py mysql")
|
||||
print(" python demo_databases.py victoriametrics")
|
||||
print(" python demo_databases.py all")
|
||||
|
||||
# Show recommendations
|
||||
show_recommendations()
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("✅ Demo completed!")
|
||||
print("📖 See DATABASE_DEPLOYMENT_GUIDE.md for production setup instructions")
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,46 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Custom exceptions for water monitoring system
|
||||
"""
|
||||
|
||||
|
||||
class WaterMonitorException(Exception):
|
||||
"""Base exception for water monitoring system"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class DatabaseConnectionError(WaterMonitorException):
|
||||
"""Raised when database connection fails"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class APIConnectionError(WaterMonitorException):
|
||||
"""Raised when API connection fails"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class DataValidationError(WaterMonitorException):
|
||||
"""Raised when data validation fails"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class ConfigurationError(WaterMonitorException):
|
||||
"""Raised when configuration is invalid"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class DataParsingError(WaterMonitorException):
|
||||
"""Raised when data parsing fails"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class RetryExhaustedError(WaterMonitorException):
|
||||
"""Raised when all retry attempts are exhausted"""
|
||||
|
||||
pass
|
||||
@@ -1,172 +0,0 @@
|
||||
"""Persistence for issued flood forecasts.
|
||||
|
||||
Every background precompute stores what the deployed model predicted at that
|
||||
moment — predicted 24/12/6 h peak, warning/danger probabilities, model
|
||||
version. Keyed by (as_of, station, horizon), so hourly data yields one row
|
||||
per station-horizon per hour regardless of how often the precompute runs.
|
||||
This is the operational record that lets "predicted vs actual" be graphed
|
||||
later without retraining historical models.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import logging
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ForecastHistoryStore:
|
||||
"""SQL store (sqlite / postgresql / mysql), same pattern as HiiStore."""
|
||||
|
||||
def __init__(self, connection_string: str, db_type: str):
|
||||
self.db_type = db_type.lower()
|
||||
if self.db_type not in ("sqlite", "postgresql", "mysql"):
|
||||
raise ValueError(
|
||||
f"Forecast history requires a SQL database, got '{db_type}'"
|
||||
)
|
||||
self.connection_string = connection_string
|
||||
self.engine = None
|
||||
|
||||
def connect(self) -> bool:
|
||||
try:
|
||||
from sqlalchemy import create_engine, text
|
||||
|
||||
self.engine = create_engine(self.connection_string, pool_pre_ping=True)
|
||||
ddl = """
|
||||
CREATE TABLE IF NOT EXISTS forecast_history (
|
||||
as_of TIMESTAMP NOT NULL,
|
||||
station_code VARCHAR(10) NOT NULL,
|
||||
horizon_hours INTEGER NOT NULL,
|
||||
predicted_max_level NUMERIC(8,3),
|
||||
p_warning NUMERIC(7,5),
|
||||
p_danger NUMERIC(7,5),
|
||||
current_level NUMERIC(8,3),
|
||||
model_version VARCHAR(64),
|
||||
source VARCHAR(16),
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (as_of, station_code, horizon_hours)
|
||||
)
|
||||
"""
|
||||
index = (
|
||||
"CREATE INDEX IF NOT EXISTS idx_forecast_history_station "
|
||||
"ON forecast_history(station_code, as_of)"
|
||||
)
|
||||
with self.engine.begin() as conn:
|
||||
conn.execute(text(ddl))
|
||||
if self.db_type != "mysql": # MySQL lacks IF NOT EXISTS for indexes
|
||||
conn.execute(text(index))
|
||||
return True
|
||||
except Exception as error:
|
||||
logger.error(f"ForecastHistoryStore failed to connect: {error}")
|
||||
self.engine = None
|
||||
return False
|
||||
|
||||
def save_rows(self, rows: List[Dict]) -> int:
|
||||
"""Upsert forecast rows as returned by ml.predict (idempotent)."""
|
||||
if not rows:
|
||||
return 0
|
||||
if not self.engine and not self.connect():
|
||||
return 0
|
||||
from sqlalchemy import text
|
||||
|
||||
cols = (
|
||||
"(as_of, station_code, horizon_hours, predicted_max_level, "
|
||||
"p_warning, p_danger, current_level, model_version, source)"
|
||||
)
|
||||
values = (
|
||||
"(:as_of, :station_code, :horizon_hours, :predicted_max_level, "
|
||||
":p_warning, :p_danger, :current_level, :model_version, :source)"
|
||||
)
|
||||
update_cols = (
|
||||
"predicted_max_level",
|
||||
"p_warning",
|
||||
"p_danger",
|
||||
"current_level",
|
||||
"model_version",
|
||||
"source",
|
||||
)
|
||||
if self.db_type == "mysql":
|
||||
updates = ", ".join(f"{c} = VALUES({c})" for c in update_cols)
|
||||
sql = (
|
||||
f"INSERT INTO forecast_history {cols} VALUES {values} "
|
||||
f"ON DUPLICATE KEY UPDATE {updates}"
|
||||
)
|
||||
else:
|
||||
updates = ", ".join(f"{c} = EXCLUDED.{c}" for c in update_cols)
|
||||
sql = (
|
||||
f"INSERT INTO forecast_history {cols} VALUES {values} "
|
||||
f"ON CONFLICT (as_of, station_code, horizon_hours) "
|
||||
f"DO UPDATE SET {updates}"
|
||||
)
|
||||
|
||||
params = []
|
||||
for row in rows:
|
||||
as_of = row.get("as_of")
|
||||
if isinstance(as_of, str):
|
||||
as_of = datetime.datetime.fromisoformat(as_of)
|
||||
if as_of is None or row.get("station_code") is None:
|
||||
continue
|
||||
params.append(
|
||||
{
|
||||
"as_of": as_of,
|
||||
"station_code": row["station_code"],
|
||||
"horizon_hours": row.get("horizon_hours"),
|
||||
"predicted_max_level": row.get("predicted_max_level"),
|
||||
"p_warning": row.get("p_warning"),
|
||||
"p_danger": row.get("p_danger"),
|
||||
"current_level": row.get("current_level"),
|
||||
"model_version": row.get("model_version"),
|
||||
"source": row.get("source"),
|
||||
}
|
||||
)
|
||||
if not params:
|
||||
return 0
|
||||
try:
|
||||
with self.engine.begin() as conn:
|
||||
conn.execute(text(sql), params)
|
||||
return len(params)
|
||||
except Exception as error:
|
||||
logger.error(f"ForecastHistoryStore save failed: {error}")
|
||||
return 0
|
||||
|
||||
def fetch(
|
||||
self,
|
||||
station_code: str,
|
||||
start: Optional[datetime.datetime] = None,
|
||||
end: Optional[datetime.datetime] = None,
|
||||
horizon_hours: Optional[int] = None,
|
||||
limit: int = 5000,
|
||||
) -> List[Dict]:
|
||||
"""Issued forecasts for one station, ascending by as_of."""
|
||||
if not self.engine and not self.connect():
|
||||
return []
|
||||
from sqlalchemy import text
|
||||
|
||||
clauses = ["station_code = :code"]
|
||||
params: Dict = {"code": station_code, "limit": limit}
|
||||
if start is not None:
|
||||
clauses.append("as_of >= :start")
|
||||
params["start"] = start
|
||||
if end is not None:
|
||||
clauses.append("as_of <= :end")
|
||||
params["end"] = end
|
||||
if horizon_hours is not None:
|
||||
clauses.append("horizon_hours = :horizon")
|
||||
params["horizon"] = horizon_hours
|
||||
sql = (
|
||||
"SELECT as_of, station_code, horizon_hours, predicted_max_level, "
|
||||
"p_warning, p_danger, current_level, model_version, source "
|
||||
f"FROM forecast_history WHERE {' AND '.join(clauses)} "
|
||||
"ORDER BY as_of ASC, horizon_hours ASC LIMIT :limit"
|
||||
)
|
||||
try:
|
||||
with self.engine.connect() as conn:
|
||||
rows = [dict(r._mapping) for r in conn.execute(text(sql), params)]
|
||||
for row in rows:
|
||||
for key, value in row.items():
|
||||
if hasattr(value, "is_finite"): # Decimal -> float
|
||||
row[key] = float(value)
|
||||
return rows
|
||||
except Exception as error:
|
||||
logger.error(f"ForecastHistoryStore fetch failed: {error}")
|
||||
return []
|
||||
@@ -1,290 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Health check system for water monitoring application
|
||||
"""
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta
|
||||
from enum import Enum
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HealthStatus(Enum):
|
||||
HEALTHY = "healthy"
|
||||
DEGRADED = "degraded"
|
||||
UNHEALTHY = "unhealthy"
|
||||
|
||||
|
||||
@dataclass
|
||||
class HealthCheckResult:
|
||||
"""Result of a health check"""
|
||||
|
||||
name: str
|
||||
status: HealthStatus
|
||||
message: str
|
||||
timestamp: datetime
|
||||
response_time_ms: Optional[float] = None
|
||||
details: Optional[Dict[str, Any]] = None
|
||||
|
||||
|
||||
class HealthCheck:
|
||||
"""Base health check class"""
|
||||
|
||||
def __init__(self, name: str, timeout_seconds: int = 30):
|
||||
self.name = name
|
||||
self.timeout_seconds = timeout_seconds
|
||||
|
||||
def check(self) -> HealthCheckResult:
|
||||
"""Perform the health check"""
|
||||
start_time = time.time()
|
||||
|
||||
try:
|
||||
result = self._perform_check()
|
||||
response_time = (time.time() - start_time) * 1000
|
||||
|
||||
return HealthCheckResult(
|
||||
name=self.name,
|
||||
status=result.get("status", HealthStatus.HEALTHY),
|
||||
message=result.get("message", "OK"),
|
||||
timestamp=datetime.now(),
|
||||
response_time_ms=response_time,
|
||||
details=result.get("details"),
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
response_time = (time.time() - start_time) * 1000
|
||||
logger.error(f"Health check {self.name} failed: {e}")
|
||||
|
||||
return HealthCheckResult(
|
||||
name=self.name,
|
||||
status=HealthStatus.UNHEALTHY,
|
||||
message=f"Check failed: {str(e)}",
|
||||
timestamp=datetime.now(),
|
||||
response_time_ms=response_time,
|
||||
)
|
||||
|
||||
def _perform_check(self) -> Dict[str, Any]:
|
||||
"""Override this method to implement the actual check"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class DatabaseHealthCheck(HealthCheck):
|
||||
"""Health check for database connectivity"""
|
||||
|
||||
def __init__(self, db_adapter, name: str = "database"):
|
||||
super().__init__(name)
|
||||
self.db_adapter = db_adapter
|
||||
|
||||
def _perform_check(self) -> Dict[str, Any]:
|
||||
if not self.db_adapter:
|
||||
return {
|
||||
"status": HealthStatus.UNHEALTHY,
|
||||
"message": "Database adapter not initialized",
|
||||
}
|
||||
|
||||
try:
|
||||
# Connect only when there is no live engine yet: connect() re-runs
|
||||
# the CREATE TABLE DDL suite, which is far too heavy per probe.
|
||||
if getattr(self.db_adapter, "engine", None) is None and hasattr(
|
||||
self.db_adapter, "connect"
|
||||
):
|
||||
connected = self.db_adapter.connect()
|
||||
if not connected:
|
||||
return {
|
||||
"status": HealthStatus.UNHEALTHY,
|
||||
"message": "Database connection failed",
|
||||
}
|
||||
|
||||
# Try to get latest data
|
||||
latest_data = self.db_adapter.get_latest_measurements(limit=1)
|
||||
|
||||
if latest_data:
|
||||
latest_timestamp = latest_data[0].get("timestamp")
|
||||
if isinstance(latest_timestamp, str):
|
||||
latest_timestamp = datetime.fromisoformat(
|
||||
latest_timestamp.replace("Z", "+00:00")
|
||||
)
|
||||
|
||||
# Check if data is recent (within last 2 hours)
|
||||
if datetime.now() - latest_timestamp.replace(tzinfo=None) > timedelta(
|
||||
hours=2
|
||||
):
|
||||
return {
|
||||
"status": HealthStatus.DEGRADED,
|
||||
"message": f"Latest data is old: {latest_timestamp}",
|
||||
"details": {"latest_data_timestamp": str(latest_timestamp)},
|
||||
}
|
||||
|
||||
return {
|
||||
"status": HealthStatus.HEALTHY,
|
||||
"message": "Database connection OK",
|
||||
"details": {
|
||||
"latest_data_count": len(latest_data),
|
||||
"latest_timestamp": (
|
||||
str(latest_data[0].get("timestamp")) if latest_data else None
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
return {
|
||||
"status": HealthStatus.UNHEALTHY,
|
||||
"message": f"Database check failed: {str(e)}",
|
||||
}
|
||||
|
||||
|
||||
class APIHealthCheck(HealthCheck):
|
||||
"""Health check for external API connectivity"""
|
||||
|
||||
def __init__(self, api_url: str, session, name: str = "api"):
|
||||
super().__init__(name)
|
||||
# A liveness probe should fail fast: the default 30s timeout meant a
|
||||
# slow upstream pinned executor threads for longer than the /health
|
||||
# cache TTL, so the pool never drained under load.
|
||||
self.timeout_seconds = 5
|
||||
self.api_url = api_url
|
||||
self.session = session
|
||||
|
||||
def _perform_check(self) -> Dict[str, Any]:
|
||||
try:
|
||||
# Simple GET request to check API availability
|
||||
response = self.session.get(self.api_url, timeout=self.timeout_seconds)
|
||||
|
||||
if response.status_code == 200:
|
||||
return {
|
||||
"status": HealthStatus.HEALTHY,
|
||||
"message": "API connection OK",
|
||||
"details": {
|
||||
"status_code": response.status_code,
|
||||
"response_size": len(response.content),
|
||||
},
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"status": HealthStatus.DEGRADED,
|
||||
"message": f"API returned status {response.status_code}",
|
||||
"details": {"status_code": response.status_code},
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
return {
|
||||
"status": HealthStatus.UNHEALTHY,
|
||||
"message": f"API check failed: {str(e)}",
|
||||
}
|
||||
|
||||
|
||||
class MemoryHealthCheck(HealthCheck):
|
||||
"""Health check for memory usage"""
|
||||
|
||||
def __init__(self, max_memory_mb: int = 1000, name: str = "memory"):
|
||||
super().__init__(name)
|
||||
self.max_memory_mb = max_memory_mb
|
||||
|
||||
def _perform_check(self) -> Dict[str, Any]:
|
||||
try:
|
||||
import psutil
|
||||
|
||||
process = psutil.Process()
|
||||
memory_info = process.memory_info()
|
||||
memory_mb = memory_info.rss / 1024 / 1024
|
||||
|
||||
if memory_mb > self.max_memory_mb:
|
||||
return {
|
||||
"status": HealthStatus.DEGRADED,
|
||||
"message": f"High memory usage: {memory_mb:.1f}MB",
|
||||
"details": {
|
||||
"memory_mb": memory_mb,
|
||||
"max_memory_mb": self.max_memory_mb,
|
||||
},
|
||||
}
|
||||
|
||||
return {
|
||||
"status": HealthStatus.HEALTHY,
|
||||
"message": f"Memory usage OK: {memory_mb:.1f}MB",
|
||||
"details": {"memory_mb": memory_mb},
|
||||
}
|
||||
|
||||
except ImportError:
|
||||
return {
|
||||
"status": HealthStatus.HEALTHY,
|
||||
"message": "Memory check skipped (psutil not available)",
|
||||
}
|
||||
except Exception as e:
|
||||
return {
|
||||
"status": HealthStatus.UNHEALTHY,
|
||||
"message": f"Memory check failed: {str(e)}",
|
||||
}
|
||||
|
||||
|
||||
class HealthCheckManager:
|
||||
"""Manages multiple health checks"""
|
||||
|
||||
def __init__(self):
|
||||
self.checks: List[HealthCheck] = []
|
||||
self.last_results: Dict[str, HealthCheckResult] = {}
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def add_check(self, health_check: HealthCheck):
|
||||
"""Add a health check"""
|
||||
with self._lock:
|
||||
self.checks.append(health_check)
|
||||
|
||||
def run_all_checks(self) -> Dict[str, HealthCheckResult]:
|
||||
"""Run all health checks"""
|
||||
results = {}
|
||||
|
||||
for check in self.checks:
|
||||
try:
|
||||
result = check.check()
|
||||
results[check.name] = result
|
||||
|
||||
with self._lock:
|
||||
self.last_results[check.name] = result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error running health check {check.name}: {e}")
|
||||
results[check.name] = HealthCheckResult(
|
||||
name=check.name,
|
||||
status=HealthStatus.UNHEALTHY,
|
||||
message=f"Check execution failed: {str(e)}",
|
||||
timestamp=datetime.now(),
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
def get_overall_status(self) -> HealthStatus:
|
||||
"""Get overall system health status"""
|
||||
if not self.last_results:
|
||||
return HealthStatus.UNHEALTHY
|
||||
|
||||
statuses = [result.status for result in self.last_results.values()]
|
||||
|
||||
if any(status == HealthStatus.UNHEALTHY for status in statuses):
|
||||
return HealthStatus.UNHEALTHY
|
||||
elif any(status == HealthStatus.DEGRADED for status in statuses):
|
||||
return HealthStatus.DEGRADED
|
||||
else:
|
||||
return HealthStatus.HEALTHY
|
||||
|
||||
def get_health_summary(self) -> Dict[str, Any]:
|
||||
"""Get a summary of system health"""
|
||||
overall_status = self.get_overall_status()
|
||||
|
||||
return {
|
||||
"overall_status": overall_status.value,
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"checks": {
|
||||
name: {
|
||||
"status": result.status.value,
|
||||
"message": result.message,
|
||||
"response_time_ms": result.response_time_ms,
|
||||
"timestamp": result.timestamp.isoformat(),
|
||||
}
|
||||
for name, result in self.last_results.items()
|
||||
},
|
||||
}
|
||||
+5
-5
@@ -198,13 +198,13 @@ def main(argv: Optional[List[str]] = None) -> bool:
|
||||
level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
|
||||
)
|
||||
|
||||
from .config import Config
|
||||
from .config import database_url
|
||||
|
||||
db_config = Config.get_database_config()
|
||||
if db_config["type"] not in ("sqlite", "postgresql", "mysql"):
|
||||
logger.error(f"Backfill requires a SQL DB_TYPE, got '{db_config['type']}'")
|
||||
url = database_url()
|
||||
if not url:
|
||||
logger.error("no database: set POSTGRES_CONNECTION_STRING or POSTGRES_*")
|
||||
return False
|
||||
store = HiiStore(db_config["connection_string"], db_config["type"])
|
||||
store = HiiStore(url)
|
||||
if not store.connect():
|
||||
return False
|
||||
|
||||
|
||||
+19
-95
@@ -1,7 +1,10 @@
|
||||
"""Collector for HII/ThaiWater open api-v3 feeds (rainfall + water level).
|
||||
"""HII/ThaiWater open api-v3 client and PostgreSQL store (rainfall + water level).
|
||||
|
||||
Polls the unauthenticated api-v3.thaiwater.net public endpoints, filters to the
|
||||
Ping basin, and persists to dedicated tables alongside the RID data:
|
||||
The hourly collection runs in ping-collector (rust/collector/src/hii.rs, same
|
||||
parsing and tables); this module is used by the history backfill
|
||||
(src/hii_backfill.py, scripts/backfill_hii_waterlevel.py) and the tests.
|
||||
Endpoints: the unauthenticated api-v3.thaiwater.net public API, filtered to the
|
||||
Ping basin. Tables:
|
||||
|
||||
- hii_rain_stations / hii_rainfall (rain_1h / rain_24h gauge telemetry)
|
||||
- hii_wl_stations / hii_waterlevel (independent water-level source, m MSL)
|
||||
@@ -175,16 +178,10 @@ class HiiClient:
|
||||
|
||||
|
||||
class HiiStore:
|
||||
"""SQL persistence for HII feeds (sqlite / postgresql / mysql).
|
||||
"""PostgreSQL persistence for the HII feeds: the hii_* tables next to the
|
||||
RID data in the same database (the DDL matches rust/collector/src/db.rs)."""
|
||||
|
||||
Reuses the app's main relational database (same connection string as
|
||||
the RID tables) but writes to its own hii_* tables.
|
||||
"""
|
||||
|
||||
def __init__(self, connection_string: str, db_type: str):
|
||||
self.db_type = db_type.lower()
|
||||
if self.db_type not in ("sqlite", "postgresql", "mysql"):
|
||||
raise ValueError(f"HII collection requires a SQL database, got '{db_type}'")
|
||||
def __init__(self, connection_string: str):
|
||||
self.connection_string = connection_string
|
||||
self.engine = None
|
||||
|
||||
@@ -203,7 +200,6 @@ class HiiStore:
|
||||
def _create_tables(self):
|
||||
from sqlalchemy import text
|
||||
|
||||
bool_type = "BOOLEAN" if self.db_type != "mysql" else "TINYINT(1)"
|
||||
ddl = [
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS hii_rain_stations (
|
||||
@@ -230,7 +226,7 @@ class HiiStore:
|
||||
PRIMARY KEY (station_id, timestamp)
|
||||
)
|
||||
""",
|
||||
f"""
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS hii_wl_stations (
|
||||
id INTEGER PRIMARY KEY,
|
||||
oldcode VARCHAR(60),
|
||||
@@ -247,7 +243,7 @@ class HiiStore:
|
||||
critical_level_msl NUMERIC(8,3),
|
||||
critical_level_m NUMERIC(8,3),
|
||||
qmax NUMERIC(10,2),
|
||||
is_key_station {bool_type},
|
||||
is_key_station BOOLEAN,
|
||||
updated_at TIMESTAMP
|
||||
)
|
||||
""",
|
||||
@@ -269,10 +265,6 @@ class HiiStore:
|
||||
"CREATE INDEX IF NOT EXISTS idx_hii_rainfall_ts ON hii_rainfall(timestamp)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_hii_waterlevel_ts ON hii_waterlevel(timestamp)",
|
||||
]
|
||||
# MySQL (<8.0.13 semantics) lacks CREATE INDEX IF NOT EXISTS; the unique
|
||||
# constraints already cover the hot (station_id, timestamp) lookups there.
|
||||
if self.db_type == "mysql":
|
||||
ddl = ddl[:4]
|
||||
with self.engine.begin() as conn:
|
||||
for statement in ddl:
|
||||
conn.execute(text(statement))
|
||||
@@ -281,19 +273,11 @@ class HiiStore:
|
||||
cols = key_cols + value_cols
|
||||
col_list = ", ".join(cols)
|
||||
params = ", ".join(f":{c}" for c in cols)
|
||||
if self.db_type == "sqlite":
|
||||
return f"INSERT OR REPLACE INTO {table} ({col_list}) VALUES ({params})"
|
||||
if self.db_type == "postgresql":
|
||||
updates = ", ".join(f"{c} = EXCLUDED.{c}" for c in value_cols)
|
||||
conflict = ", ".join(key_cols)
|
||||
return (
|
||||
f"INSERT INTO {table} ({col_list}) VALUES ({params}) "
|
||||
f"ON CONFLICT ({conflict}) DO UPDATE SET {updates}"
|
||||
)
|
||||
updates = ", ".join(f"{c} = VALUES({c})" for c in value_cols)
|
||||
updates = ", ".join(f"{c} = EXCLUDED.{c}" for c in value_cols)
|
||||
conflict = ", ".join(key_cols)
|
||||
return (
|
||||
f"INSERT INTO {table} ({col_list}) VALUES ({params}) "
|
||||
f"ON DUPLICATE KEY UPDATE {updates}"
|
||||
f"ON CONFLICT ({conflict}) DO UPDATE SET {updates}"
|
||||
)
|
||||
|
||||
def save_rain(self, records: List[Dict]) -> int:
|
||||
@@ -360,18 +344,11 @@ class HiiStore:
|
||||
|
||||
cols = "(station_id, timestamp, wl_msl, discharge)"
|
||||
values = "(:station_id, :timestamp, :wl_msl, :discharge)"
|
||||
if self.db_type == "mysql":
|
||||
sql = (
|
||||
f"INSERT INTO hii_waterlevel {cols} VALUES {values} "
|
||||
"ON DUPLICATE KEY UPDATE wl_msl = VALUES(wl_msl), "
|
||||
"discharge = VALUES(discharge)"
|
||||
)
|
||||
else: # sqlite (>=3.24) and postgresql share upsert syntax
|
||||
sql = (
|
||||
f"INSERT INTO hii_waterlevel {cols} VALUES {values} "
|
||||
"ON CONFLICT (station_id, timestamp) DO UPDATE SET "
|
||||
"wl_msl = EXCLUDED.wl_msl, discharge = EXCLUDED.discharge"
|
||||
)
|
||||
sql = (
|
||||
f"INSERT INTO hii_waterlevel {cols} VALUES {values} "
|
||||
"ON CONFLICT (station_id, timestamp) DO UPDATE SET "
|
||||
"wl_msl = EXCLUDED.wl_msl, discharge = EXCLUDED.discharge"
|
||||
)
|
||||
params = [{**row, "station_id": station_id} for row in rows]
|
||||
try:
|
||||
with self.engine.begin() as conn:
|
||||
@@ -423,56 +400,3 @@ class HiiStore:
|
||||
except Exception as e:
|
||||
logger.error(f"HiiStore save to {measurement_table} failed: {e}")
|
||||
return 0
|
||||
|
||||
|
||||
class HiiCollector:
|
||||
"""Fetch + persist one snapshot of both HII feeds."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
db_config: Dict,
|
||||
basin_code: int = PING_BASIN_CODE,
|
||||
client: Optional[HiiClient] = None,
|
||||
):
|
||||
self.client = client or HiiClient()
|
||||
self.basin_code = basin_code
|
||||
self.store = HiiStore(
|
||||
connection_string=db_config["connection_string"],
|
||||
db_type=db_config["type"],
|
||||
)
|
||||
|
||||
def run_cycle(self) -> Dict[str, int]:
|
||||
"""Collect both feeds; each is independent and failure-isolated."""
|
||||
counts = {"rainfall": 0, "waterlevel": 0}
|
||||
try:
|
||||
counts["rainfall"] = self.store.save_rain(
|
||||
self.client.fetch_rain(self.basin_code)
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"HII rainfall collection failed: {e}")
|
||||
try:
|
||||
counts["waterlevel"] = self.store.save_waterlevel(
|
||||
self.client.fetch_waterlevel(self.basin_code)
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"HII waterlevel collection failed: {e}")
|
||||
logger.info(
|
||||
f"HII collection: {counts['rainfall']} rainfall, "
|
||||
f"{counts['waterlevel']} waterlevel rows saved"
|
||||
)
|
||||
return counts
|
||||
|
||||
|
||||
def create_collector_from_config() -> Optional[HiiCollector]:
|
||||
"""Build a collector from app Config; None when disabled or non-SQL DB."""
|
||||
from .config import Config
|
||||
|
||||
if not Config.ENABLE_HII_COLLECTION:
|
||||
return None
|
||||
db_config = Config.get_database_config()
|
||||
if db_config["type"] not in ("sqlite", "postgresql", "mysql"):
|
||||
logger.warning(
|
||||
f"HII collection skipped: DB_TYPE '{db_config['type']}' is not SQL"
|
||||
)
|
||||
return None
|
||||
return HiiCollector(db_config, basin_code=Config.HII_BASIN_CODE)
|
||||
@@ -1,135 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Centralized logging configuration for water monitoring system
|
||||
"""
|
||||
|
||||
import logging
|
||||
import logging.handlers
|
||||
import os
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class ColoredFormatter(logging.Formatter):
|
||||
"""Colored console formatter"""
|
||||
|
||||
COLORS = {
|
||||
"DEBUG": "\033[36m", # Cyan
|
||||
"INFO": "\033[32m", # Green
|
||||
"WARNING": "\033[33m", # Yellow
|
||||
"ERROR": "\033[31m", # Red
|
||||
"CRITICAL": "\033[35m", # Magenta
|
||||
"RESET": "\033[0m", # Reset
|
||||
}
|
||||
|
||||
def format(self, record):
|
||||
if hasattr(record, "levelname"):
|
||||
color = self.COLORS.get(record.levelname, self.COLORS["RESET"])
|
||||
record.levelname = f"{color}{record.levelname}{self.COLORS['RESET']}"
|
||||
return super().format(record)
|
||||
|
||||
|
||||
def setup_logging(
|
||||
log_level: str = "INFO",
|
||||
log_file: Optional[str] = None,
|
||||
max_file_size: int = 10 * 1024 * 1024, # 10MB
|
||||
backup_count: int = 5,
|
||||
enable_console: bool = True,
|
||||
enable_colors: bool = True,
|
||||
) -> logging.Logger:
|
||||
"""
|
||||
Setup comprehensive logging configuration
|
||||
|
||||
Args:
|
||||
log_level: Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
|
||||
log_file: Path to log file (optional)
|
||||
max_file_size: Maximum size of log file before rotation
|
||||
backup_count: Number of backup files to keep
|
||||
enable_console: Whether to enable console logging
|
||||
enable_colors: Whether to enable colored console output
|
||||
|
||||
Returns:
|
||||
Configured logger instance
|
||||
"""
|
||||
|
||||
# Create logs directory if it doesn't exist
|
||||
if log_file:
|
||||
log_dir = os.path.dirname(log_file)
|
||||
if log_dir and not os.path.exists(log_dir):
|
||||
os.makedirs(log_dir)
|
||||
|
||||
# Configure root logger
|
||||
logger = logging.getLogger()
|
||||
logger.setLevel(getattr(logging, log_level.upper()))
|
||||
|
||||
# Clear existing handlers
|
||||
logger.handlers.clear()
|
||||
|
||||
# Create formatters
|
||||
detailed_formatter = logging.Formatter(
|
||||
"%(asctime)s - %(name)s - %(levelname)s - %(filename)s:%(lineno)d - %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
|
||||
simple_formatter = logging.Formatter(
|
||||
"%(asctime)s - %(levelname)s - %(message)s", datefmt="%H:%M:%S"
|
||||
)
|
||||
|
||||
# Console handler
|
||||
if enable_console:
|
||||
console_handler = logging.StreamHandler()
|
||||
if enable_colors and os.name != "nt": # Don't use colors on Windows
|
||||
console_formatter = ColoredFormatter(
|
||||
"%(asctime)s - %(levelname)s - %(message)s", datefmt="%H:%M:%S"
|
||||
)
|
||||
else:
|
||||
console_formatter = simple_formatter
|
||||
|
||||
console_handler.setFormatter(console_formatter)
|
||||
console_handler.setLevel(getattr(logging, log_level.upper()))
|
||||
logger.addHandler(console_handler)
|
||||
|
||||
# File handler with rotation
|
||||
if log_file:
|
||||
file_handler = logging.handlers.RotatingFileHandler(
|
||||
log_file, maxBytes=max_file_size, backupCount=backup_count, encoding="utf-8"
|
||||
)
|
||||
file_handler.setFormatter(detailed_formatter)
|
||||
file_handler.setLevel(logging.DEBUG) # Always log everything to file
|
||||
logger.addHandler(file_handler)
|
||||
|
||||
# Add performance logger for metrics
|
||||
perf_logger = logging.getLogger("performance")
|
||||
if log_file:
|
||||
perf_file = log_file.replace(".log", "_performance.log")
|
||||
perf_handler = logging.handlers.RotatingFileHandler(
|
||||
perf_file,
|
||||
maxBytes=max_file_size,
|
||||
backupCount=backup_count,
|
||||
encoding="utf-8",
|
||||
)
|
||||
perf_formatter = logging.Formatter(
|
||||
"%(asctime)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S"
|
||||
)
|
||||
perf_handler.setFormatter(perf_formatter)
|
||||
perf_logger.addHandler(perf_handler)
|
||||
perf_logger.setLevel(logging.INFO)
|
||||
perf_logger.propagate = False
|
||||
|
||||
return logger
|
||||
|
||||
|
||||
def log_performance_metric(
|
||||
operation: str, duration: float, additional_info: Optional[str] = None
|
||||
):
|
||||
"""Log performance metrics"""
|
||||
perf_logger = logging.getLogger("performance")
|
||||
message = f"PERF: {operation} took {duration:.3f}s"
|
||||
if additional_info:
|
||||
message += f" - {additional_info}"
|
||||
perf_logger.info(message)
|
||||
|
||||
|
||||
def get_logger(name: str) -> logging.Logger:
|
||||
"""Get a logger with the specified name"""
|
||||
return logging.getLogger(name)
|
||||
Loaded 100 of 151 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user