Compare commits
2
Commits
330fdf7804
...
6fe2e287e5
| Author | SHA1 | Date | |
|---|---|---|---|
 grabowski
|
6fe2e287e5 | ||
|
grabowski
|
7e642555b4 |
No files matched your search
+63
-119
@@ -1,141 +1,85 @@
|
||||
# Northern Thailand Ping River Monitor Configuration
|
||||
# Copy this file to .env and customize for your environment
|
||||
# Ping River Monitor configuration: copy to .env and fill in.
|
||||
#
|
||||
# Read by ping-collector (the service; rust/collector/src/config.rs) and by the
|
||||
# Python training tools (src/config.py). Under systemd the variables come from
|
||||
# EnvironmentFile= (systemd's parser); for manual runs both programs load
|
||||
# ./.env with python-dotenv's rules. Unset keys take the defaults shown.
|
||||
|
||||
# Database Configuration
|
||||
DB_TYPE=postgresql
|
||||
# Options: sqlite, mysql, postgresql, influxdb, victoriametrics
|
||||
|
||||
# SQLite Configuration (default)
|
||||
WATER_DB_PATH=water_levels.db
|
||||
|
||||
# VictoriaMetrics Configuration
|
||||
VM_HOST=localhost
|
||||
VM_PORT=8428
|
||||
VM_URL=
|
||||
|
||||
# InfluxDB Configuration
|
||||
INFLUX_HOST=localhost
|
||||
INFLUX_PORT=8086
|
||||
INFLUX_DATABASE=ping_river_monitoring
|
||||
INFLUX_USERNAME=
|
||||
INFLUX_PASSWORD=
|
||||
|
||||
# PostgreSQL Configuration (Remote Server)
|
||||
# Option 1: Full connection string (URL encode special characters in password)
|
||||
POSTGRES_CONNECTION_STRING=postgresql://username:url_encoded_password@your-postgres-host:5432/water_monitoring
|
||||
|
||||
# Option 2: Individual components (password will be automatically URL encoded)
|
||||
POSTGRES_HOST=your-postgres-host
|
||||
# --- Database (PostgreSQL only) ---------------------------------------------
|
||||
# Either a full connection string (URL-encode special characters in the
|
||||
# password: : %3A @ %40 # %23 ? %3F & %26 / %2F % %25) ...
|
||||
POSTGRES_CONNECTION_STRING=
|
||||
# ... or the parts (the password is encoded for you)
|
||||
POSTGRES_HOST=localhost
|
||||
POSTGRES_PORT=5432
|
||||
POSTGRES_DB=water_monitoring
|
||||
POSTGRES_USER=username
|
||||
POSTGRES_PASSWORD=your:password@with!special#chars
|
||||
POSTGRES_USER=postgres
|
||||
POSTGRES_PASSWORD=
|
||||
|
||||
# Examples for connection string:
|
||||
# - Local: postgresql://postgres:password@localhost:5432/water_monitoring
|
||||
# - Remote: postgresql://user:pass@192.168.1.100:5432/water_monitoring
|
||||
# - With special chars: postgresql://user:my%3Apass%40word@host:5432/db
|
||||
# - With SSL: postgresql://user:pass@host:port/db?sslmode=require
|
||||
# - Connection pooling: postgresql://user:pass@host:port/db?pool_size=20&max_overflow=0
|
||||
# --- What the service runs ---------------------------------------------------
|
||||
# Website and API: the address to listen on (empty = no website)
|
||||
WEB_LISTEN=0.0.0.0:8000
|
||||
# Forecast + ntfy/Matrix notifications after each collection (false = collect only)
|
||||
FORECAST_IN_COLLECTOR=true
|
||||
# Trained models (models/flood_<station>.json.gz from src/ml/export.py)
|
||||
# MODELS_DIR=models
|
||||
|
||||
# Special character URL encoding:
|
||||
# : → %3A @ → %40 # → %23 ? → %3F & → %26 / → %2F % → %25
|
||||
|
||||
# MySQL Configuration
|
||||
MYSQL_CONNECTION_STRING=mysql://user:password@localhost:3306/ping_river_monitoring
|
||||
|
||||
# API Configuration
|
||||
API_HOST=0.0.0.0
|
||||
API_PORT=8000
|
||||
API_WORKERS=1
|
||||
# Public ThaiWater API key used to add Ping-basin water-level sensors.
|
||||
THAIWATER_API_KEY=
|
||||
|
||||
# Data Collection Settings
|
||||
SCRAPING_INTERVAL_HOURS=1
|
||||
# RID poll minute past the hour; retry interval while RID is behind
|
||||
# --- Collection ----------------------------------------------------------------
|
||||
# RID is polled this many minutes past each hour and retried every
|
||||
# RID_RETRY_MINUTES while the current hour is not published yet
|
||||
RID_POLL_OFFSET_MINUTES=5
|
||||
RID_RETRY_MINUTES=10
|
||||
# Days of RID holes the leader re-fetches at startup (0 = off)
|
||||
# RID reports with holes re-fetched at startup (0 = off); older: `ping-collector fill-gaps all`
|
||||
GAP_FILL_DAYS_ON_START=14
|
||||
# false when the Rust ping-collector service collects (rust/collector);
|
||||
# share the status file between both services for /scraping/status
|
||||
COLLECTION_ENABLED=true
|
||||
# SCRAPING_STATUS_FILE=/opt/thailand-water-monitor/collector-status.json
|
||||
# true: ping-collector also forecasts (models/flood_*.json.gz, rust/forecast)
|
||||
# and sends the ntfy notifications; this app serves its stored forecast and
|
||||
# computes itself only when that is older than FORECAST_MAX_AGE_SECONDS.
|
||||
# Set by scripts/install_collector.sh --cutover-forecast.
|
||||
FORECAST_IN_COLLECTOR=false
|
||||
# FORECAST_MAX_AGE_SECONDS=7200
|
||||
# MODELS_DIR=models
|
||||
REQUEST_TIMEOUT=30
|
||||
MAX_RETRIES=3
|
||||
RETRY_DELAY_SECONDS=60
|
||||
|
||||
# Data Retention
|
||||
DATA_RETENTION_DAYS=365
|
||||
|
||||
# Logging Configuration
|
||||
# ENABLE_HII_COLLECTION=true
|
||||
# ENABLE_RESERVOIR_COLLECTION=true
|
||||
# ENABLE_RAIN_COLLECTION=true
|
||||
# HII_BASIN_CODE=6
|
||||
# REQUEST_TIMEOUT=30
|
||||
# Runtime station list written by the admin API; falls back to src/data/stations.json
|
||||
# STATION_CONFIG_PATH=stations.json
|
||||
LOG_LEVEL=INFO
|
||||
LOG_FILE=water_monitor.log
|
||||
|
||||
# Security (for production)
|
||||
SECRET_KEY=your-secret-key-here
|
||||
API_KEY=your-api-key-here
|
||||
|
||||
# Monitoring
|
||||
ENABLE_METRICS=true
|
||||
ENABLE_HEALTH_CHECKS=true
|
||||
|
||||
# Geographic Settings
|
||||
TIMEZONE=Asia/Bangkok
|
||||
DEFAULT_LATITUDE=18.7875
|
||||
DEFAULT_LONGITUDE=99.0045
|
||||
|
||||
# External Services
|
||||
NOTIFICATION_EMAIL=
|
||||
SMTP_SERVER=
|
||||
SMTP_PORT=587
|
||||
SMTP_USERNAME=
|
||||
SMTP_PASSWORD=
|
||||
|
||||
# Public push notifications via self-hosted ntfy (https://ntfy.sh, single binary).
|
||||
# Leave NTFY_SERVER empty to disable. Topics published: <prefix>-<station>-warning,
|
||||
# <prefix>-<station>-danger, <prefix>-warning, <prefix>-danger, <prefix>-p1-outlook,
|
||||
# <prefix>-status. See docs/NOTIFICATIONS.md.
|
||||
# --- Notifications (docs/NOTIFICATIONS.md) ------------------------------------
|
||||
# Public push alerts via self-hosted ntfy; empty NTFY_SERVER = off. Topics:
|
||||
# <prefix>-<station>-warning|danger, <prefix>-warning|danger,
|
||||
# <prefix>-p1-outlook, <prefix>-status.
|
||||
NTFY_SERVER=
|
||||
# Where the monitor POSTs (defaults to NTFY_SERVER). Use the local ntfy
|
||||
# address (loopback or Tailscale IP) so publishing does not depend on
|
||||
# DNS / the reverse proxy being up.
|
||||
# Where the service POSTs (default NTFY_SERVER): the local ntfy address, so
|
||||
# publishing does not depend on DNS or the reverse proxy
|
||||
NTFY_PUBLISH_URL=
|
||||
NTFY_TOPIC_PREFIX=ping
|
||||
NTFY_TOKEN=
|
||||
PUBLIC_URL=https://water.buildfor.life/
|
||||
# ping-collector notifier: report RID lagging this many hours while ThaiWater
|
||||
# telemetry is current (0 = never)
|
||||
# Report RID lagging this many hours while ThaiWater telemetry is current (0 = never)
|
||||
RID_STALE_ALERT_HOURS=18
|
||||
|
||||
# Matrix: ping-collector sends the ntfy transitions to this room too
|
||||
# (docs/MATRIX_QUICK_START.md); off while token or room is empty
|
||||
# Matrix room for a team: the same transitions (docs/MATRIX_QUICK_START.md);
|
||||
# off while token or room is empty
|
||||
MATRIX_HOMESERVER=https://matrix.org
|
||||
MATRIX_ACCESS_TOKEN=
|
||||
MATRIX_ROOM_ID=
|
||||
|
||||
# Website and API served by ping-collector (rust/collector/src/web): the
|
||||
# address to listen on; empty = the Python app serves (port 8000). Set by
|
||||
# scripts/install_collector.sh --cutover-api.
|
||||
WEB_LISTEN=
|
||||
# --- Website and API -------------------------------------------------------------
|
||||
# X-API-Key for station editing, /config and POST /scrape/trigger (empty = those are off)
|
||||
ADMIN_API_KEY=
|
||||
# Comma-separated origins allowed cross-site with credentials (empty: any origin, no credentials)
|
||||
CORS_ALLOW_ORIGINS=
|
||||
# ThaiWater API key for /sensors/thaiwater (extra Ping-basin sensors on the map)
|
||||
THAIWATER_API_KEY=
|
||||
# Self-hosted Umami: server-side tracking of API calls
|
||||
# UMAMI_API_URL=https://stats.buildfor.life/api/send
|
||||
# UMAMI_WEBSITE_ID=00b2be73-8f5f-4400-9029-3be852eb08f7
|
||||
# UMAMI_TRACK_API=true
|
||||
# Response cache TTLs (seconds)
|
||||
# LATEST_CACHE_TTL_SECONDS=45
|
||||
# HII_CACHE_TTL_SECONDS=120
|
||||
# HEALTH_CACHE_TTL_SECONDS=30
|
||||
# A stored forecast older than this is served marked X-Data-Stale: true
|
||||
# FORECAST_MAX_AGE_SECONDS=7200
|
||||
# STATIC_DIR=src/static
|
||||
# ADMIN_API_KEY= # X-API-Key for station editing, /config, /scrape/trigger
|
||||
# Only for `ping-collector serve` running separately from `run`: the status file
|
||||
# SCRAPING_STATUS_FILE=collector-status.json
|
||||
|
||||
# Grafana Integration
|
||||
GRAFANA_URL=http://localhost:3000
|
||||
|
||||
# Alert Configuration
|
||||
ALERT_MAX_AGE_HOURS=2
|
||||
ALERT_CHECK_INTERVAL_MINUTES=15
|
||||
|
||||
# Development Settings
|
||||
DEBUG=false
|
||||
DEVELOPMENT_MODE=false
|
||||
# --- Python tools ----------------------------------------------------------------
|
||||
# Training/evaluation read the database above; this overrides it for them only
|
||||
# FLOOD_ML_DB_URL=
|
||||
+25
-15
@@ -1,13 +1,13 @@
|
||||
name: CI
|
||||
|
||||
# What this checks, on every push and PR to master:
|
||||
# The Python side (src/ml training/evaluation stack, archive backfills), on
|
||||
# every push and PR to master:
|
||||
# 1. formatting contract (black + isort, config in pyproject.toml)
|
||||
# 2. flake8 hard-error gate (syntax, undefined names)
|
||||
# 3. the pytest suite (synthetic data, no DB/network; ~1 min)
|
||||
# Docker build / staging / production / perf jobs from the original template
|
||||
# were removed: there is no registry, no staging host, and production is a
|
||||
# systemd unit deployed by `git pull` on the server (docs/FLOOD_FORECASTING.md
|
||||
# section 6, scripts/install.sh). Re-add a job when the thing it deploys exists.
|
||||
# 3. the pytest suite: synthetic data, no network; the database tests run
|
||||
# against a throwaway PostgreSQL service (TEST_DATABASE_URL, as in rust.yml)
|
||||
# The service itself (ping-collector) is checked by rust.yml. Production is a
|
||||
# systemd unit deployed by `git pull` + scripts/install.sh on the server.
|
||||
|
||||
on:
|
||||
push:
|
||||
@@ -15,7 +15,7 @@ on:
|
||||
pull_request:
|
||||
branches: [master]
|
||||
schedule:
|
||||
# daily, catches dependency drift / upstream API changes in the tests
|
||||
# daily, catches dependency drift
|
||||
- cron: "0 2 * * *"
|
||||
workflow_dispatch:
|
||||
|
||||
@@ -41,17 +41,30 @@ jobs:
|
||||
pip install --root-user-action=ignore black==26.5.1 isort==5.12.0 flake8==6.1.0
|
||||
|
||||
- name: black
|
||||
run: black --check --diff src/ *.py
|
||||
run: black --check --diff src/ scripts/ tests/ rust/forecast/testdata rust/collector/testdata
|
||||
|
||||
- name: isort
|
||||
run: isort --check-only --diff src/ *.py
|
||||
run: isort --check-only --diff src/ scripts/ tests/
|
||||
|
||||
- name: flake8 (errors only)
|
||||
run: flake8 src/ --count --select=E9,F63,F7,F82 --show-source --statistics
|
||||
run: flake8 src/ scripts/ tests/ --count --select=E9,F63,F7,F82 --show-source --statistics
|
||||
|
||||
test:
|
||||
name: Test suite
|
||||
runs-on: ubuntu-latest
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16
|
||||
env:
|
||||
POSTGRES_PASSWORD: ping
|
||||
POSTGRES_DB: ping
|
||||
options: >-
|
||||
--health-cmd pg_isready
|
||||
--health-interval 5s
|
||||
--health-timeout 5s
|
||||
--health-retries 20
|
||||
env:
|
||||
TEST_DATABASE_URL: postgres://postgres:ping@postgres:5432/ping
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
@@ -66,10 +79,7 @@ jobs:
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip --root-user-action=ignore
|
||||
pip install --root-user-action=ignore -r requirements.txt
|
||||
pip install --root-user-action=ignore pytest==9.1.1 pytest-asyncio==0.21.1
|
||||
pip install --root-user-action=ignore -r requirements.txt pytest==9.1.1
|
||||
|
||||
- name: pytest
|
||||
env:
|
||||
DB_TYPE: sqlite
|
||||
run: pytest -q -p no:cacheprovider
|
||||
run: pytest -q -p no:cacheprovider -rs
|
||||
+10
-54
@@ -1,36 +1,23 @@
|
||||
name: Docs
|
||||
|
||||
# Checks that the documentation the project actually ships stays consistent:
|
||||
# - every relative link / image path in docs/*.md and README.md resolves
|
||||
# inside the repo (external URLs are NOT fetched: localhost examples,
|
||||
# rate-limited hosts and the Tailscale-era links made that gate permanently
|
||||
# red, and a 200 on a curl --head proves nothing about a doc anyway)
|
||||
# - the FastAPI app imports and its OpenAPI schema is exportable (that is
|
||||
# the reference at https://water.buildfor.life/docs)
|
||||
# The previous Sphinx/apidoc jobs produced artifacts nobody read and were
|
||||
# removed. Reference docs live in docs/*.md; the public overview is at
|
||||
# https://buildfor.life/docs/tooling/ping-river-monitor/.
|
||||
# Every relative link / image path in the shipped Markdown (README, CLAUDE,
|
||||
# CONTRIBUTING, rust/README, docs/) resolves inside the repo. External URLs
|
||||
# are NOT fetched: localhost examples and rate-limited hosts made that gate
|
||||
# permanently red, and a 200 proves nothing about a doc anyway. The API
|
||||
# reference (/docs, /openapi.json) is generated by ping-collector from its
|
||||
# route table and tested in rust/collector/src/web/tests.rs.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [master, develop]
|
||||
paths:
|
||||
- "docs/**"
|
||||
- "README.md"
|
||||
- "CONTRIBUTING.md"
|
||||
- "src/web_api.py"
|
||||
- "src/schemas.py"
|
||||
- "**.md"
|
||||
- ".gitea/workflows/docs.yml"
|
||||
pull_request:
|
||||
paths:
|
||||
- "docs/**"
|
||||
- "README.md"
|
||||
- "CONTRIBUTING.md"
|
||||
- "**.md"
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
PYTHON_VERSION: "3.11"
|
||||
|
||||
jobs:
|
||||
docs:
|
||||
name: Validate documentation
|
||||
@@ -43,7 +30,8 @@ jobs:
|
||||
python3 - <<'PY'
|
||||
import re, sys, pathlib
|
||||
root = pathlib.Path(".")
|
||||
files = [root / "README.md", root / "CONTRIBUTING.md", *root.glob("docs/**/*.md")]
|
||||
files = [root / "README.md", root / "CLAUDE.md", root / "CONTRIBUTING.md",
|
||||
root / "rust" / "README.md", *root.glob("docs/**/*.md")]
|
||||
link = re.compile(r"!?\[[^\]]*\]\(([^)\s]+)(?:\s+\"[^\"]*\")?\)")
|
||||
bad = []
|
||||
for md in files:
|
||||
@@ -65,35 +53,3 @@ jobs:
|
||||
sys.exit(1)
|
||||
print(f"checked {len(files)} files, all relative links resolve")
|
||||
PY
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
cache: pip
|
||||
cache-dependency-path: requirements.txt
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip --root-user-action=ignore
|
||||
pip install --root-user-action=ignore -r requirements.txt
|
||||
|
||||
- name: OpenAPI schema exports
|
||||
env:
|
||||
DB_TYPE: sqlite
|
||||
run: |
|
||||
python - <<'PY'
|
||||
import json
|
||||
from src.web_api import app
|
||||
spec = app.openapi()
|
||||
paths = sorted(spec["paths"])
|
||||
required = {"/forecast", "/measurements/latest", "/measurements/history/{station_code}", "/stations", "/api/stats", "/health"}
|
||||
missing = required - set(paths)
|
||||
assert not missing, f"documented endpoints missing from the app: {missing}"
|
||||
json.dump(spec, open("openapi.json", "w"), indent=1)
|
||||
print(f"{len(paths)} paths; schema written to openapi.json")
|
||||
PY
|
||||
|
||||
- uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: openapi-${{ github.run_number }}
|
||||
path: openapi.json
|
||||
@@ -1,314 +0,0 @@
|
||||
name: Release - Northern Thailand Ping River Monitor
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "v*.*.*"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version:
|
||||
description: "Release version (e.g., v3.1.3)"
|
||||
required: true
|
||||
type: string
|
||||
|
||||
env:
|
||||
PYTHON_VERSION: "3.11"
|
||||
REGISTRY: git.b4l.co.th
|
||||
IMAGE_NAME: b4l/northern-thailand-ping-river-monitor
|
||||
# GitHub token for better rate limits and authentication
|
||||
GH_TOKEN: ${{ secrets.GH_TOKEN }}
|
||||
|
||||
jobs:
|
||||
# Create release
|
||||
create-release:
|
||||
name: Create Release
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
version: ${{ steps.version.outputs.version }}
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
token: ${{ secrets.GITEA_TOKEN }}
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Get version
|
||||
id: version
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "version=${{ github.event.inputs.version }}" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "version=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Generate changelog
|
||||
id: changelog
|
||||
run: |
|
||||
# Generate changelog from git commits
|
||||
echo "## Changes" > CHANGELOG.md
|
||||
git log --pretty=format:"- %s" $(git describe --tags --abbrev=0 HEAD^)..HEAD >> CHANGELOG.md || echo "- Initial release" >> CHANGELOG.md
|
||||
echo "" >> CHANGELOG.md
|
||||
echo "## Docker Images" >> CHANGELOG.md
|
||||
echo "- \`${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.version.outputs.version }}\`" >> CHANGELOG.md
|
||||
echo "- \`${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest\`" >> CHANGELOG.md
|
||||
|
||||
- name: Create Release
|
||||
uses: actions/create-release@v1
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||
with:
|
||||
tag_name: ${{ steps.version.outputs.version }}
|
||||
release_name: Northern Thailand Ping River Monitor ${{ steps.version.outputs.version }}
|
||||
body_path: CHANGELOG.md
|
||||
draft: false
|
||||
prerelease: false
|
||||
|
||||
# Build and test for release
|
||||
test-release:
|
||||
name: Test Release Build
|
||||
runs-on: ubuntu-latest
|
||||
needs: create-release
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: ["3.9", "3.10", "3.11", "3.12"]
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
token: ${{ secrets.GITEA_TOKEN }}
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip --root-user-action=ignore
|
||||
pip install --root-user-action=ignore -r requirements.txt
|
||||
pip install --root-user-action=ignore -r requirements-dev.txt
|
||||
|
||||
- name: Run full test suite
|
||||
run: |
|
||||
python tests/test_integration.py
|
||||
python tests/test_station_management.py
|
||||
python run.py --test
|
||||
|
||||
- name: Build Python package
|
||||
run: |
|
||||
pip install --root-user-action=ignore build
|
||||
python -m build
|
||||
|
||||
- name: Upload Python package
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: python-package-${{ matrix.python-version }}
|
||||
path: dist/
|
||||
|
||||
# Build release Docker images
|
||||
build-release:
|
||||
name: Build Release Images
|
||||
runs-on: ubuntu-latest
|
||||
needs: [create-release, test-release]
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
token: ${{ secrets.GITEA_TOKEN }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ vars.WORKER_USERNAME}}
|
||||
password: ${{ secrets.CI_BOT_TOKEN }}
|
||||
|
||||
- name: Build and push release images
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: true
|
||||
tags: |
|
||||
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.create-release.outputs.version }}
|
||||
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
|
||||
labels: |
|
||||
org.opencontainers.image.title=Northern Thailand Ping River Monitor
|
||||
org.opencontainers.image.description=Real-time water level monitoring for Ping River Basin
|
||||
org.opencontainers.image.version=${{ needs.create-release.outputs.version }}
|
||||
org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
|
||||
org.opencontainers.image.revision=${{ github.sha }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||
|
||||
# Security scan for release
|
||||
security-scan:
|
||||
name: Security Scan
|
||||
runs-on: ubuntu-latest
|
||||
needs: build-release
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
token: ${{ secrets.GITEA_TOKEN}}
|
||||
|
||||
# Test release deployment locally
|
||||
deploy-release:
|
||||
name: Test Release Deployment
|
||||
runs-on: ubuntu-latest
|
||||
needs: [create-release, build-release, security-scan]
|
||||
environment:
|
||||
name: testing
|
||||
url: http://localhost:8080
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
token: ${{ secrets.GITEA_TOKEN }}
|
||||
|
||||
- name: Log in to Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ vars.WORKER_USERNAME}}
|
||||
password: ${{ secrets.CI_BOT_TOKEN }}
|
||||
|
||||
- name: Deploy to production (Local Test)
|
||||
run: |
|
||||
set -euo pipefail
|
||||
echo "🚀 Testing ${{ needs.create-release.outputs.version }} deployment locally..."
|
||||
|
||||
# Create a dedicated network so we can resolve by container name
|
||||
docker network create ci_net || true
|
||||
|
||||
# Pull the built image
|
||||
docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.create-release.outputs.version }}
|
||||
|
||||
# Stop & remove any existing container
|
||||
docker rm -f ping-river-monitor-test 2>/dev/null || true
|
||||
|
||||
# Start the container on the user-defined network
|
||||
docker run -d \
|
||||
--name ping-river-monitor-test \
|
||||
--network ci_net \
|
||||
-p 8080:8000 \
|
||||
-e LOG_LEVEL=INFO \
|
||||
-e DB_TYPE=sqlite \
|
||||
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.create-release.outputs.version }}
|
||||
|
||||
echo "✅ Container started for testing"
|
||||
|
||||
- name: Health check after deployment
|
||||
run: |
|
||||
set -euo pipefail
|
||||
echo "⏳ Waiting for application to start..."
|
||||
|
||||
# Pull a curl-only image for probing (keeps your app image slim)
|
||||
docker pull curlimages/curl:8.10.1
|
||||
|
||||
# Helper: curl via a sibling container on the SAME Docker network
|
||||
probe() {
|
||||
local url="$1"
|
||||
docker run --rm --network ci_net curlimages/curl:8.10.1 \
|
||||
-sS --max-time 5 --connect-timeout 3 -w "HTTP_CODE:%{http_code}" "$url" || true
|
||||
}
|
||||
|
||||
# Wait for /health (up to ~3m 45s)
|
||||
for i in {1..15}; do
|
||||
echo "🔍 Attempt $i/15: checking http://ping-river-monitor-test:8000/health"
|
||||
resp="$(probe http://ping-river-monitor-test:8000/health)"
|
||||
code="$(echo "$resp" | sed -n 's/.*HTTP_CODE:\([0-9]\+\).*/\1/p')"
|
||||
body="$(echo "$resp" | sed 's/HTTP_CODE:[0-9]*$//')"
|
||||
|
||||
echo "HTTP: ${code:-<none>} | Body: ${body:-<empty>}"
|
||||
|
||||
if [ "${code:-}" = "200" ] && [ -n "${body:-}" ]; then
|
||||
echo "✅ Health endpoint responding successfully"
|
||||
break
|
||||
fi
|
||||
|
||||
echo "❌ Not ready yet. Showing recent logs…"
|
||||
docker logs --tail 20 ping-river-monitor-test || true
|
||||
sleep 15
|
||||
|
||||
if [ "$i" -eq 15 ]; then
|
||||
echo "❌ Health never reached 200. Failing."
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo "🧪 Testing API endpoints…"
|
||||
endpoints=("health" "docs" "stations" "metrics")
|
||||
for ep in "${endpoints[@]}"; do
|
||||
url="http://ping-river-monitor-test:8000/$ep"
|
||||
resp="$(probe "$url")"
|
||||
code="$(echo "$resp" | sed -n 's/.*HTTP_CODE:\([0-9]\+\).*/\1/p')"
|
||||
|
||||
if [ "${code:-}" = "200" ]; then
|
||||
echo "✅ /$ep: OK"
|
||||
else
|
||||
echo "❌ /$ep: FAILED (HTTP ${code:-<none>})"
|
||||
echo "Response: $(echo "$resp" | sed 's/HTTP_CODE:[0-9]*$//')"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo "✅ All health checks passed!"
|
||||
|
||||
- name: Container logs and cleanup
|
||||
if: always()
|
||||
run: |
|
||||
echo "📋 Container logs:"
|
||||
docker logs ping-river-monitor-test || true
|
||||
|
||||
echo "🧹 Cleaning up test container..."
|
||||
docker stop ping-river-monitor-test || true
|
||||
docker rm ping-river-monitor-test || true
|
||||
|
||||
echo "📊 Deployment Test Summary:"
|
||||
echo "Version: ${{ needs.create-release.outputs.version }}"
|
||||
echo "Image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.create-release.outputs.version }}"
|
||||
echo "Status: Container tested successfully"
|
||||
echo "Ready for production deployment"
|
||||
|
||||
# Notify stakeholders
|
||||
notify:
|
||||
name: Notify Release
|
||||
runs-on: ubuntu-latest
|
||||
needs: [create-release, deploy-release]
|
||||
if: always()
|
||||
|
||||
steps:
|
||||
- name: Notify success
|
||||
if: needs.deploy-release.result == 'success'
|
||||
run: |
|
||||
echo "🎉 Release ${{ needs.create-release.outputs.version }} tested successfully!"
|
||||
echo "🧪 Local Test: Passed all health checks"
|
||||
echo "� GDocker Image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.create-release.outputs.version }}"
|
||||
echo "✅ Ready for production deployment"
|
||||
|
||||
# Add notification to Slack, Discord, email, etc.
|
||||
# curl -X POST -H 'Content-type: application/json' \
|
||||
# --data '{"text":"🎉 Northern Thailand Ping River Monitor ${{ needs.create-release.outputs.version }} tested and ready for deployment!"}' \
|
||||
# ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
|
||||
- name: Notify failure
|
||||
if: needs.deploy-release.result == 'failure'
|
||||
run: |
|
||||
echo "❌ Release ${{ needs.create-release.outputs.version }} testing failed!"
|
||||
echo "Please check the logs and fix issues before production deployment."
|
||||
|
||||
# Add failure notification
|
||||
# curl -X POST -H 'Content-type: application/json' \
|
||||
# --data '{"text":"❌ Northern Thailand Ping River Monitor ${{ needs.create-release.outputs.version }} testing failed!"}' \
|
||||
# ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
@@ -5,16 +5,20 @@ Guidance for AI coding agents working in this repository.
|
||||
## What this is
|
||||
|
||||
Flood monitoring and forecasting for the Ping River, Chiang Mai. Public dashboard and
|
||||
API at https://water.buildfor.life/ (never publish the server's private/Tailscale IP).
|
||||
Production: one systemd unit on a small VPS, `/opt/thailand-water-monitor`, user
|
||||
`water-monitor`, interpreter `.venv/bin/python` (uv-managed), updated by `git pull`.
|
||||
API at https://water.buildfor.life/ (never publish the server's private/mesh IP).
|
||||
Production: one systemd unit, `ping-collector.service` (the Rust binary in `rust/`),
|
||||
on a small VPS in `/opt/thailand-water-monitor` (a git checkout), user `water-monitor`;
|
||||
plus the monthly `water-monitor-retrain.timer` (Python, `.venv`, uv-managed). Updated by
|
||||
`git pull`, `cargo build --release`, `sudo bash scripts/install.sh`.
|
||||
|
||||
## Rules
|
||||
|
||||
- Python 3.11 only. `uv sync --python 3.11`; run everything as `uv run ...`.
|
||||
- `make format` (black 88 / isort black profile, config in pyproject.toml) before
|
||||
committing; CI fails on formatting. `make test` must stay green — tests are
|
||||
synthetic-data only, never add one that needs the DB or network.
|
||||
- `make format` (black 88 / isort black profile, config in pyproject.toml; cargo fmt)
|
||||
before committing; CI fails on formatting. `make test` must stay green. Tests use
|
||||
synthetic data, never the network; database tests (both languages) run against a
|
||||
throwaway PostgreSQL schema from `TEST_DATABASE_URL` and skip without it.
|
||||
- PostgreSQL only. No other database backends.
|
||||
- Timestamps everywhere are Asia/Bangkok wall-clock with no offset. The dashboard
|
||||
parses them with `parseTs()` and renders with `timeZone: TZ`; keep it that way.
|
||||
- Model changes go through the rolling-origin harness (`scripts/evaluate_variants.py`)
|
||||
@@ -27,37 +31,41 @@ Production: one systemd unit on a small VPS, `/opt/thailand-water-monitor`, user
|
||||
before. Stage files by name.
|
||||
- Do not add Co-Authored-By trailers.
|
||||
- The dashboard is a single file, `src/static/dashboard.html`, EN + TH via the `t()`
|
||||
table: every user-visible string needs both languages.
|
||||
table: every user-visible string needs both languages. Every path it fetches must be
|
||||
a route in `rust/collector/src/web/mod.rs` (tests/test_dashboard.py checks).
|
||||
- Rust (`rust/`, see rust/README.md): `cargo fmt --all`, `cargo clippy --all-targets
|
||||
-- -D warnings`, `cargo test` (database tests need `TEST_DATABASE_URL`). The Rust
|
||||
collector writes the same tables as the Python collectors: a change to a table,
|
||||
upsert or parser on one side must be made on the other until the Python side is
|
||||
retired (check with a live run of both into two databases).
|
||||
- The forecast is served by Rust (`rust/forecast`) from models trained in Python.
|
||||
A change to `src/ml/features.py`, `predict.py`, the HII fill or the dam frame
|
||||
must be ported in the same change: regenerate
|
||||
`rust/forecast/testdata/gen_forecast_fixtures.py` and keep
|
||||
`cargo test -p ping-forecast` green. Changing thresholds, stages, leads or
|
||||
notify constants: `uv run python -m src.ml.export --config` (a test fails
|
||||
otherwise) and rebuild the collector. Notify texts:
|
||||
`rust/collector/testdata/gen_notify_cases.py`.
|
||||
- API routes live in `rust/collector/src/web` (phase 3). Until the FastAPI app is
|
||||
removed, a route change goes into both, and `scripts/api_parity.py` (both
|
||||
servers on one database) must report no differences. A route's parameters are
|
||||
declared once (`web/params.rs`); that list validates and documents (/docs).
|
||||
-- -D warnings`, `cargo test`; minimum version 1.88
|
||||
(`CARGO_TARGET_DIR=target/msrv cargo +1.88 check --all-targets --locked`). Read
|
||||
`.env` only through `rust/collector/src/envfile.rs` (python-dotenv semantics, never
|
||||
dotenvy: the Python tools read the same file).
|
||||
- The forecast is served by Rust (`rust/forecast`) from models trained in Python;
|
||||
`src/ml/predict.py` is the reference it is tested against. A change to
|
||||
`src/ml/features.py`, `predict.py`, the HII fill or the dam frame must be ported in
|
||||
the same change: regenerate `rust/forecast/testdata/gen_forecast_fixtures.py` and keep
|
||||
`cargo test -p ping-forecast` green. Changing thresholds, stages, leads or the
|
||||
notifier settings (`src/ml/export.py`): `uv run python -m src.ml.export --config`
|
||||
(a test fails otherwise) and rebuild the collector.
|
||||
- Notifier texts are pinned by `rust/collector/testdata/notify_python_cases.json.gz`;
|
||||
a deliberate wording change updates that fixture in the same commit.
|
||||
- API routes live in `rust/collector/src/web`. A route's parameters are declared once
|
||||
(`web/params.rs`); that list validates and documents (/docs). Keep response formats
|
||||
stable: external clients use them.
|
||||
- The Python backfill stores (`src/hii_collector.py`, `src/rid_reservoir.py`,
|
||||
`src/ml/rain.py`) write the same tables as `rust/collector/src/db.rs`: a table or
|
||||
upsert change goes into both.
|
||||
|
||||
## Where things are
|
||||
|
||||
- `src/web_api.py` FastAPI app; `src/water_scraper_v3.py` RID collector;
|
||||
`src/hii_collector.py` ThaiWater/HII; `src/ml/` features/train/evaluate/predict,
|
||||
`rain.py` (Open-Meteo), `dam.py`, `hii_rain.py`.
|
||||
- `scripts/retrain.sh` + `water-monitor-retrain.timer`: monthly retrain with staged
|
||||
promote. `scripts/dev_proxy.py`: serve the working-copy dashboard against the live API.
|
||||
- `rust/collector` (`ping-collector`): Rust data collector; with it deployed the
|
||||
Python app runs `COLLECTION_ENABLED=false`. With `FORECAST_IN_COLLECTOR=true` it
|
||||
also forecasts (`rust/forecast`) and sends ntfy + Matrix; with `WEB_LISTEN` it
|
||||
serves the website and API (`rust/collector/src/web`, the port of
|
||||
`src/web_api.py`; the dashboard stays `src/static/dashboard.html`). Rewrite
|
||||
plan: `docs/RUST_REWRITE.md`.
|
||||
- `rust/collector` (`ping-collector`): collection (`rid.rs`, `hii.rs`, `dams.rs`,
|
||||
`rain.rs`), forecast run (`forecasting.rs`), ntfy/Matrix (`notify.rs`), website/API
|
||||
(`web/`), tables (`db.rs`). `rust/forecast`: inference.
|
||||
- `src/ml/`: features/train/evaluate/predict/export, `rain.py` (Open-Meteo), `dam.py`,
|
||||
`hii_rain.py`. `src/hii_backfill.py`, `src/hii_collector.py`, `src/rid_reservoir.py`:
|
||||
archive backfills (`scripts/backfill_*.py`).
|
||||
- `scripts/install.sh`: install/update on the server. `scripts/retrain.sh` +
|
||||
`water-monitor-retrain.timer`: monthly retrain with staged promote.
|
||||
`scripts/forecast_parity.py`: Rust vs Python forecast on a live database.
|
||||
`scripts/dev_proxy.py`: serve the working-copy dashboard against the live API.
|
||||
- `docs/FLOOD_FORECASTING.md` is the authoritative model write-up; `docs/DATA_SOURCES.md`
|
||||
the source catalog.
|
||||
the source catalog; `docs/NOTIFICATIONS.md` the alerts; `docs/RUST_REWRITE.md` the
|
||||
history of the Python-to-Rust move.
|
||||
+42
-268
@@ -1,277 +1,51 @@
|
||||
# Contributing to Northern Thailand Ping River Monitor
|
||||
# Contributing
|
||||
|
||||
Thank you for your interest in contributing to the Northern Thailand Ping River Monitor! This document provides guidelines and information for contributors.
|
||||
Issues and merge requests on
|
||||
[git.b4l.co.th](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor).
|
||||
The repository has two halves: the service in Rust (`rust/`, see
|
||||
[rust/README.md](rust/README.md)) and the model training/evaluation stack in Python
|
||||
(`src/ml`, see [docs/FLOOD_FORECASTING.md](docs/FLOOD_FORECASTING.md)).
|
||||
|
||||
## 🚀 Getting Started
|
||||
## Setup
|
||||
|
||||
### Prerequisites
|
||||
- Python 3.9 or higher
|
||||
- Git
|
||||
- Basic knowledge of water monitoring systems
|
||||
- Familiarity with FastAPI and time-series databases (optional)
|
||||
- Python 3.11 and [uv](https://docs.astral.sh/uv/) (3.12+ has no pandas 2.0.3 wheels)
|
||||
- Rust >= 1.88 from [rustup](https://rustup.rs), plus `pkg-config` and `libssl-dev` on Linux
|
||||
- Docker (or any PostgreSQL 16) for the database tests
|
||||
|
||||
### Development Setup
|
||||
```bash
|
||||
git clone https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor.git
|
||||
cd Northern-Thailand-Ping-River-Monitor
|
||||
make install # uv sync + pre-commit hooks
|
||||
docker run -d --name ping-pg -e POSTGRES_PASSWORD=ping -e POSTGRES_DB=ping -p 55432:5432 postgres:16
|
||||
export TEST_DATABASE_URL=postgres://postgres:ping@localhost:55432/ping
|
||||
make check # format check, flake8, clippy, pytest, cargo test, Rust 1.88
|
||||
```
|
||||
|
||||
1. **Clone the repository:**
|
||||
```bash
|
||||
git clone https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor.git
|
||||
cd Northern-Thailand-Ping-River-Monitor
|
||||
```
|
||||
Without `TEST_DATABASE_URL` the database tests are skipped, the rest still run.
|
||||
|
||||
2. **Create a virtual environment:**
|
||||
```bash
|
||||
python -m venv venv
|
||||
source venv/bin/activate # On Windows: venv\Scripts\activate
|
||||
```
|
||||
## Before a merge request
|
||||
|
||||
3. **Install dependencies:**
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
- `make format`, then `make check` green. CI runs the same (`.gitea/workflows/`).
|
||||
- One logical change per commit, imperative subject (`fix(collector): ...`,
|
||||
`feat(web): ...`, `docs: ...`).
|
||||
- Dashboard changes (`src/static/dashboard.html`): every visible string in English
|
||||
and Thai (`t()` table); a screenshot in the merge request.
|
||||
- API changes: keep existing fields and formats; external clients read them. New
|
||||
parameters go into the route's declaration in `rust/collector/src/web/params.rs`,
|
||||
which also documents them at `/docs`.
|
||||
- Model changes: evaluate with `scripts/evaluate_variants.py` (rolling origin, judged
|
||||
on first-alert lead and false alarms) and record the result in
|
||||
`docs/FLOOD_FORECASTING.md` section 5, whether it helped or not. A feature change
|
||||
must be ported to `rust/forecast` in the same change (see CLAUDE.md for the
|
||||
fixture and config regeneration steps).
|
||||
- Never commit credentials, `.env`, trained models (`models/*.joblib`,
|
||||
`models/*.json.gz`) or the server's private addresses.
|
||||
|
||||
4. **Run tests:**
|
||||
```bash
|
||||
python test_integration.py
|
||||
```
|
||||
## Useful commands
|
||||
|
||||
5. **Start the development server:**
|
||||
```bash
|
||||
python run.py --web-api
|
||||
```
|
||||
|
||||
## 🔧 Development Guidelines
|
||||
|
||||
### Code Style
|
||||
- Follow PEP 8 Python style guidelines
|
||||
- Use type hints for all function parameters and return values
|
||||
- Write descriptive docstrings for all classes and functions
|
||||
- Keep functions focused and under 50 lines when possible
|
||||
|
||||
### Testing
|
||||
- Write tests for new features
|
||||
- Ensure all existing tests pass before submitting
|
||||
- Test both success and error cases
|
||||
- Include integration tests for API endpoints
|
||||
|
||||
### Documentation
|
||||
- Update README.md for new features
|
||||
- Add docstrings to all new functions and classes
|
||||
- Update API documentation for new endpoints
|
||||
- Include examples in documentation
|
||||
|
||||
## 📝 Contribution Process
|
||||
|
||||
### 1. Issue Creation
|
||||
- Check existing issues before creating new ones
|
||||
- Use clear, descriptive titles
|
||||
- Provide detailed descriptions with steps to reproduce (for bugs)
|
||||
- Include system information and logs when relevant
|
||||
|
||||
### 2. Pull Request Process
|
||||
1. **Fork the repository**
|
||||
2. **Create a feature branch:**
|
||||
```bash
|
||||
git checkout -b feature/your-feature-name
|
||||
```
|
||||
3. **Make your changes**
|
||||
4. **Test thoroughly:**
|
||||
```bash
|
||||
python test_integration.py
|
||||
python test_station_management.py
|
||||
```
|
||||
5. **Commit with clear messages:**
|
||||
```bash
|
||||
git commit -m "Add: New station validation feature"
|
||||
```
|
||||
6. **Push to your fork:**
|
||||
```bash
|
||||
git push origin feature/your-feature-name
|
||||
```
|
||||
7. **Create a pull request**
|
||||
|
||||
### Pull Request Guidelines
|
||||
- Use clear, descriptive titles
|
||||
- Provide detailed descriptions of changes
|
||||
- Reference related issues
|
||||
- Include screenshots for UI changes
|
||||
- Ensure all tests pass
|
||||
- Update documentation as needed
|
||||
|
||||
## 🎯 Areas for Contribution
|
||||
|
||||
### High Priority
|
||||
- **Additional Database Support** - MongoDB, TimescaleDB
|
||||
- **Authentication System** - User management and API keys
|
||||
- **Real-time Alerts** - Email/SMS notifications for critical conditions
|
||||
- **Mobile App** - React Native or Flutter interface
|
||||
- **Advanced Analytics** - Trend analysis and forecasting
|
||||
|
||||
### Medium Priority
|
||||
- **Bulk Station Operations** - Import/export multiple stations
|
||||
- **Map Interface** - Visual station management
|
||||
- **Data Export** - CSV, Excel format support
|
||||
- **Performance Optimization** - Query optimization and caching
|
||||
- **Internationalization** - Multi-language support
|
||||
|
||||
### Low Priority
|
||||
- **Additional Visualizations** - Custom chart types
|
||||
- **Report Generation** - Automated PDF reports
|
||||
- **Integration APIs** - Third-party service connections
|
||||
- **Advanced Logging** - Structured logging with ELK stack
|
||||
|
||||
## 🐛 Bug Reports
|
||||
|
||||
When reporting bugs, please include:
|
||||
|
||||
1. **Environment Information:**
|
||||
- Operating system and version
|
||||
- Python version
|
||||
- Database type and version
|
||||
- Browser (for web interface issues)
|
||||
|
||||
2. **Steps to Reproduce:**
|
||||
- Clear, numbered steps
|
||||
- Expected vs actual behavior
|
||||
- Screenshots or error messages
|
||||
|
||||
3. **Additional Context:**
|
||||
- Log files (sanitized of sensitive data)
|
||||
- Configuration details
|
||||
- Recent changes to the system
|
||||
|
||||
## 💡 Feature Requests
|
||||
|
||||
For feature requests, please provide:
|
||||
|
||||
1. **Use Case:** Why is this feature needed?
|
||||
2. **Description:** What should the feature do?
|
||||
3. **Acceptance Criteria:** How do we know it's complete?
|
||||
4. **Priority:** How important is this feature?
|
||||
5. **Alternatives:** Are there workarounds available?
|
||||
|
||||
## 🔒 Security
|
||||
|
||||
### Reporting Security Issues
|
||||
- **DO NOT** create public issues for security vulnerabilities
|
||||
- Email security issues to: [security contact]
|
||||
- Include detailed information about the vulnerability
|
||||
- Allow time for fixes before public disclosure
|
||||
|
||||
### Security Guidelines
|
||||
- Never commit sensitive data (passwords, API keys)
|
||||
- Use environment variables for configuration
|
||||
- Validate all user inputs
|
||||
- Follow OWASP security guidelines
|
||||
- Keep dependencies updated
|
||||
|
||||
## 📊 Database Contributions
|
||||
|
||||
### Adding New Database Support
|
||||
1. Create adapter in `src/database_adapters.py`
|
||||
2. Implement all required methods
|
||||
3. Add configuration options in `src/config.py`
|
||||
4. Update documentation
|
||||
5. Add integration tests
|
||||
|
||||
### Database Schema Changes
|
||||
- Provide migration scripts
|
||||
- Test with existing data
|
||||
- Document breaking changes
|
||||
- Update all database adapters
|
||||
|
||||
## 🌐 API Contributions
|
||||
|
||||
### Adding New Endpoints
|
||||
1. Define Pydantic models in `src/models.py`
|
||||
2. Implement endpoint in `src/web_api.py`
|
||||
3. Add input validation
|
||||
4. Include error handling
|
||||
5. Write comprehensive tests
|
||||
6. Update API documentation
|
||||
|
||||
### API Guidelines
|
||||
- Use RESTful conventions
|
||||
- Include proper HTTP status codes
|
||||
- Provide clear error messages
|
||||
- Support pagination for large datasets
|
||||
- Include rate limiting considerations
|
||||
|
||||
## 📈 Performance Guidelines
|
||||
|
||||
### Optimization Areas
|
||||
- Database query efficiency
|
||||
- API response times
|
||||
- Memory usage
|
||||
- Concurrent request handling
|
||||
- Data processing speed
|
||||
|
||||
### Monitoring
|
||||
- Use built-in metrics collection
|
||||
- Monitor response times
|
||||
- Track error rates
|
||||
- Watch resource usage
|
||||
- Set up alerts for issues
|
||||
|
||||
## 🧪 Testing Guidelines
|
||||
|
||||
### Test Types
|
||||
- **Unit Tests:** Individual function testing
|
||||
- **Integration Tests:** Component interaction testing
|
||||
- **API Tests:** Endpoint functionality testing
|
||||
- **Performance Tests:** Load and stress testing
|
||||
|
||||
### Test Coverage
|
||||
- Aim for >80% code coverage
|
||||
- Test both success and failure paths
|
||||
- Include edge cases
|
||||
- Test with different database backends
|
||||
|
||||
## 📚 Documentation Standards
|
||||
|
||||
### Code Documentation
|
||||
- Use clear, descriptive variable names
|
||||
- Write comprehensive docstrings
|
||||
- Include type hints
|
||||
- Add inline comments for complex logic
|
||||
|
||||
### User Documentation
|
||||
- Keep README.md updated
|
||||
- Provide clear installation instructions
|
||||
- Include usage examples
|
||||
- Document configuration options
|
||||
|
||||
## 🤝 Community Guidelines
|
||||
|
||||
### Communication
|
||||
- Be respectful and professional
|
||||
- Help newcomers get started
|
||||
- Share knowledge and experiences
|
||||
- Provide constructive feedback
|
||||
|
||||
### Code Reviews
|
||||
- Review code thoroughly
|
||||
- Provide specific, actionable feedback
|
||||
- Be open to suggestions
|
||||
- Focus on code quality and maintainability
|
||||
|
||||
## 📞 Getting Help
|
||||
|
||||
### Resources
|
||||
- **Documentation:** README.md and docs/
|
||||
- **API Reference:** http://localhost:8000/docs
|
||||
- **Issues:** GitHub Issues section
|
||||
- **Discussions:** GitHub Discussions
|
||||
|
||||
### Contact
|
||||
- **General Questions:** Create a GitHub issue
|
||||
- **Security Issues:** [security contact]
|
||||
- **Maintainer:** [maintainer contact]
|
||||
|
||||
## 🎉 Recognition
|
||||
|
||||
Contributors will be recognized in:
|
||||
- README.md contributors section
|
||||
- Release notes for significant contributions
|
||||
- Special recognition for major features
|
||||
|
||||
Thank you for contributing to the Northern Thailand Ping River Monitor! Your contributions help improve water resource monitoring and management in the region.
|
||||
```bash
|
||||
make build && make serve # the website/API from your database on 127.0.0.1:8001
|
||||
uv run python scripts/dev_proxy.py # the working-copy dashboard against the live API
|
||||
uv run python scripts/evaluate_variants.py --stations P.1
|
||||
uv run python -m src.ml.export --config # after changing thresholds or notifier settings
|
||||
```
|
||||
-58
@@ -1,58 +0,0 @@
|
||||
# Multi-stage Docker build for Northern Thailand Ping River Monitor
|
||||
|
||||
# Build stage
|
||||
FROM python:3.11-slim as builder
|
||||
|
||||
# Set working directory
|
||||
WORKDIR /app
|
||||
|
||||
# Install build dependencies
|
||||
RUN apt-get update && apt-get install -y \
|
||||
build-essential \
|
||||
gcc \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Copy requirements and install Python dependencies
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir --user -r requirements.txt
|
||||
|
||||
# Production stage
|
||||
FROM python:3.11-slim
|
||||
|
||||
# Set working directory
|
||||
WORKDIR /app
|
||||
|
||||
# Install runtime dependencies and create user
|
||||
RUN apt-get update && apt-get install -y \
|
||||
wget \
|
||||
curl \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& groupadd -r appuser && useradd -r -g appuser appuser \
|
||||
&& mkdir -p /home/appuser/.local
|
||||
|
||||
# Copy Python packages from builder stage
|
||||
COPY --from=builder /root/.local /home/appuser/.local
|
||||
|
||||
# Copy application code
|
||||
COPY . .
|
||||
|
||||
# Create logs directory and set permissions
|
||||
RUN mkdir -p logs && chown -R appuser:appuser /app /home/appuser/.local
|
||||
|
||||
# Set environment variables
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
ENV TZ=Asia/Bangkok
|
||||
ENV PATH=/home/appuser/.local/bin:$PATH
|
||||
|
||||
# Switch to non-root user
|
||||
USER appuser
|
||||
|
||||
# Health check for API mode
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
|
||||
CMD python -c "import requests; requests.get('http://localhost:8000/health', timeout=5)" 2>/dev/null || exit 1
|
||||
|
||||
# Expose port for API
|
||||
EXPOSE 8000
|
||||
|
||||
# Default command
|
||||
CMD ["python", "run.py", "--web-api"]
|
||||
@@ -1,165 +0,0 @@
|
||||
# Migration to uv
|
||||
|
||||
This document describes the migration from traditional Python package management (pip + requirements.txt) to [uv](https://docs.astral.sh/uv/), a fast Python package installer and resolver.
|
||||
|
||||
## What Changed
|
||||
|
||||
### Files Added
|
||||
- `pyproject.toml` - Modern Python project configuration combining dependencies and metadata
|
||||
- `.python-version` - Specifies Python version for uv
|
||||
- `scripts/setup_uv.sh` - Unix setup script for uv environment
|
||||
- `scripts/setup_uv.bat` - Windows setup script for uv environment
|
||||
- This migration guide
|
||||
|
||||
### Files Modified
|
||||
- `Makefile` - Updated all commands to use `uv run` instead of direct Python execution
|
||||
|
||||
### Files That Can Be Removed (Optional)
|
||||
- `requirements.txt` - Dependencies now in pyproject.toml
|
||||
- `requirements-dev.txt` - Dev dependencies now in pyproject.toml
|
||||
- `setup.py` - Configuration now in pyproject.toml
|
||||
|
||||
## Installation
|
||||
|
||||
### Install uv
|
||||
|
||||
**Unix/macOS:**
|
||||
```bash
|
||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
```
|
||||
|
||||
**Windows (PowerShell):**
|
||||
```powershell
|
||||
powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"
|
||||
```
|
||||
|
||||
### Setup Project
|
||||
|
||||
**Unix/macOS:**
|
||||
```bash
|
||||
# Run the setup script
|
||||
chmod +x scripts/setup_uv.sh
|
||||
./scripts/setup_uv.sh
|
||||
|
||||
# Or manually:
|
||||
uv sync
|
||||
uv run pre-commit install
|
||||
```
|
||||
|
||||
**Windows:**
|
||||
```batch
|
||||
REM Run the setup script
|
||||
scripts\setup_uv.bat
|
||||
|
||||
REM Or manually:
|
||||
uv sync
|
||||
uv run pre-commit install
|
||||
```
|
||||
|
||||
## New Workflow
|
||||
|
||||
### Common Commands
|
||||
|
||||
| Old Command | New Command | Description |
|
||||
|-------------|-------------|-------------|
|
||||
| `pip install -r requirements.txt` | `uv sync --no-dev` | Install production dependencies |
|
||||
| `pip install -r requirements-dev.txt` | `uv sync` | Install all dependencies (including dev) |
|
||||
| `python run.py` | `uv run python run.py` | Run the application |
|
||||
| `pytest` | `uv run pytest` | Run tests |
|
||||
| `black src/` | `uv run black src/` | Format code |
|
||||
|
||||
### Using the Makefile
|
||||
|
||||
The Makefile has been updated to use uv, so all existing commands work the same:
|
||||
|
||||
```bash
|
||||
make install-dev # Install dev dependencies with uv
|
||||
make test # Run tests with uv
|
||||
make run-api # Start API server with uv
|
||||
make lint # Lint code with uv
|
||||
make format # Format code with uv
|
||||
```
|
||||
|
||||
### Adding Dependencies
|
||||
|
||||
**Production dependency:**
|
||||
```bash
|
||||
uv add requests
|
||||
```
|
||||
|
||||
**Development dependency:**
|
||||
```bash
|
||||
uv add --dev pytest
|
||||
```
|
||||
|
||||
**Specific version:**
|
||||
```bash
|
||||
uv add "fastapi==0.104.1"
|
||||
```
|
||||
|
||||
### Managing Python Versions
|
||||
|
||||
uv can automatically manage Python versions:
|
||||
|
||||
```bash
|
||||
# Install and use Python 3.11
|
||||
uv python install 3.11
|
||||
uv sync
|
||||
|
||||
# Use specific Python version
|
||||
uv sync --python 3.11
|
||||
```
|
||||
|
||||
## Benefits of uv
|
||||
|
||||
1. **Speed** - 10-100x faster than pip
|
||||
2. **Reliability** - Better dependency resolution
|
||||
3. **Simplicity** - Single tool for packages and Python versions
|
||||
4. **Reproducibility** - Lock file ensures consistent environments
|
||||
5. **Modern** - Built-in support for pyproject.toml
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Command not found
|
||||
Make sure uv is in your PATH after installation. Restart your terminal or run:
|
||||
```bash
|
||||
source ~/.bashrc # or ~/.zshrc
|
||||
```
|
||||
|
||||
### Lock file conflicts
|
||||
If you encounter lock file issues:
|
||||
```bash
|
||||
rm uv.lock
|
||||
uv sync
|
||||
```
|
||||
|
||||
### Python version issues
|
||||
Ensure the Python version in `.python-version` is available:
|
||||
```bash
|
||||
uv python list
|
||||
uv python install 3.11 # if needed
|
||||
```
|
||||
|
||||
## Rollback (if needed)
|
||||
|
||||
If you need to rollback to the old system:
|
||||
|
||||
1. Use the original requirements files:
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
pip install -r requirements-dev.txt
|
||||
```
|
||||
|
||||
2. Revert the Makefile changes to use `python` instead of `uv run python`
|
||||
|
||||
3. Remove uv-specific files:
|
||||
```bash
|
||||
rm pyproject.toml .python-version uv.lock
|
||||
rm -rf .venv # if created by uv
|
||||
```
|
||||
|
||||
## Additional Resources
|
||||
|
||||
- [uv Documentation](https://docs.astral.sh/uv/)
|
||||
- [Migration Guide](https://docs.astral.sh/uv/guides/projects/)
|
||||
- [pyproject.toml Reference](https://packaging.python.org/en/latest/specifications/pyproject-toml/)
|
||||
@@ -1,181 +1,70 @@
|
||||
# Northern Thailand Ping River Monitor - Makefile
|
||||
# Ping River Monitor. The service is Rust (rust/, ping-collector); Python is
|
||||
# the model training/evaluation stack and the archive backfills (src/).
|
||||
# Database tests (both sides) need TEST_DATABASE_URL, e.g.
|
||||
# docker run -d --name ping-pg -e POSTGRES_PASSWORD=ping -e POSTGRES_DB=ping -p 55432:5432 postgres:16
|
||||
# export TEST_DATABASE_URL=postgres://postgres:ping@localhost:55432/ping
|
||||
|
||||
.PHONY: help install install-dev test test-coverage lint format clean run run-api docker-build docker-run docs
|
||||
PY_SRC := src/ scripts/ tests/ rust/forecast/testdata rust/collector/testdata
|
||||
CARGO := cargo
|
||||
MANIFEST := --manifest-path rust/Cargo.toml
|
||||
|
||||
.PHONY: help install format lint test test-py test-rust build check serve train evaluate export clean
|
||||
|
||||
# Default target
|
||||
help:
|
||||
@echo "Northern Thailand Ping River Monitor - Available Commands:"
|
||||
@echo ""
|
||||
@echo "Setup:"
|
||||
@echo " install Install production dependencies"
|
||||
@echo " install-dev Install development dependencies"
|
||||
@echo ""
|
||||
@echo "Development:"
|
||||
@echo " test Run all tests"
|
||||
@echo " test-cov Run tests with coverage report"
|
||||
@echo " lint Run code linting"
|
||||
@echo " format Format code with black and isort"
|
||||
@echo " clean Clean up temporary files"
|
||||
@echo ""
|
||||
@echo "Running:"
|
||||
@echo " run Run the monitor in continuous mode"
|
||||
@echo " run-api Run the web API server"
|
||||
@echo " run-test Run a single test cycle"
|
||||
@echo " run-status Show system status"
|
||||
@echo ""
|
||||
@echo "Alerting:"
|
||||
@echo " alert-check Check water levels and send alerts"
|
||||
@echo " alert-test Send test Matrix message"
|
||||
@echo ""
|
||||
@echo "Distribution:"
|
||||
@echo " build-exe Build standalone executable"
|
||||
@echo " package Build and create distribution package"
|
||||
@echo ""
|
||||
@echo "Docker:"
|
||||
@echo " docker-build Build Docker image"
|
||||
@echo " docker-run Run with Docker Compose"
|
||||
@echo " docker-stop Stop Docker services"
|
||||
@echo ""
|
||||
@echo "Database:"
|
||||
@echo " setup-postgres Setup PostgreSQL database"
|
||||
@echo " test-postgres Test PostgreSQL connection"
|
||||
@echo " encode-password URL encode password for connection string"
|
||||
@echo " migrate-sqlite Migrate SQLite data to PostgreSQL"
|
||||
@echo " migrate-fast Fast migration with 10K batch size"
|
||||
@echo " analyze-sqlite Analyze SQLite database structure (dry run)"
|
||||
@echo ""
|
||||
@echo "Documentation:"
|
||||
@echo " docs Generate documentation"
|
||||
@echo "make install uv sync (Python 3.11) + pre-commit hooks"
|
||||
@echo "make format black + isort (Python), cargo fmt (Rust)"
|
||||
@echo "make lint the CI checks without tests (format, flake8, clippy)"
|
||||
@echo "make test pytest + cargo test"
|
||||
@echo "make build release build of ping-collector (rust/target/release)"
|
||||
@echo "make check lint + test + the Rust 1.88 minimum-version check"
|
||||
@echo "make serve website/API from the local database on 127.0.0.1:8001"
|
||||
@echo "make train retrain every station's model into models/"
|
||||
@echo "make evaluate rolling-origin harness on P.1 (see scripts/evaluate_variants.py)"
|
||||
@echo "make export export trained models + flood_config.json for the Rust forecaster"
|
||||
|
||||
# Installation
|
||||
install:
|
||||
uv sync --no-dev
|
||||
|
||||
install-dev:
|
||||
uv sync
|
||||
uv sync --python 3.11
|
||||
uv run pre-commit install
|
||||
|
||||
# Testing
|
||||
test:
|
||||
format:
|
||||
uv run black $(PY_SRC)
|
||||
uv run isort src/ scripts/ tests/
|
||||
$(CARGO) fmt --all $(MANIFEST)
|
||||
|
||||
lint:
|
||||
uv run black --check $(PY_SRC)
|
||||
uv run isort --check-only src/ scripts/ tests/
|
||||
uv run flake8 src/ scripts/ tests/ --count --select=E9,F63,F7,F82 --show-source --statistics
|
||||
$(CARGO) fmt --all --check $(MANIFEST)
|
||||
$(CARGO) clippy --all-targets --locked $(MANIFEST) -- -D warnings
|
||||
|
||||
test: test-py test-rust
|
||||
|
||||
test-py:
|
||||
uv run pytest -q
|
||||
|
||||
test-cov:
|
||||
uv run pytest --cov=src --cov-report=html --cov-report=term
|
||||
test-rust:
|
||||
$(CARGO) test --locked $(MANIFEST)
|
||||
|
||||
# Code quality
|
||||
lint:
|
||||
uv run flake8 src/ --max-line-length=100
|
||||
uv run mypy src/
|
||||
build:
|
||||
$(CARGO) build --release --locked $(MANIFEST)
|
||||
|
||||
format:
|
||||
uv run black src/ *.py
|
||||
uv run isort src/ *.py
|
||||
check: lint test
|
||||
CARGO_TARGET_DIR=rust/target/msrv $(CARGO) +1.88 check --all-targets --locked $(MANIFEST)
|
||||
|
||||
serve:
|
||||
rust/target/release/ping-collector serve --listen 127.0.0.1:8001 --no-analytics
|
||||
|
||||
train:
|
||||
uv run python scripts/train_flood_model.py --stations all
|
||||
|
||||
evaluate:
|
||||
uv run python scripts/evaluate_variants.py --stations P.1
|
||||
|
||||
export:
|
||||
uv run python -m src.ml.export
|
||||
uv run python -m src.ml.export --config
|
||||
|
||||
# Cleanup
|
||||
clean:
|
||||
find . -type f -name "*.pyc" -delete
|
||||
find . -type d -name "__pycache__" -delete
|
||||
find . -type f -name "*.log" -delete
|
||||
rm -rf .pytest_cache/
|
||||
rm -rf .mypy_cache/
|
||||
rm -rf htmlcov/
|
||||
rm -rf dist/
|
||||
rm -rf build/
|
||||
rm -rf *.egg-info/
|
||||
|
||||
# Running
|
||||
run:
|
||||
uv run python run.py
|
||||
|
||||
run-api:
|
||||
uv run python run.py --web-api
|
||||
|
||||
run-test:
|
||||
uv run python run.py --test
|
||||
|
||||
run-status:
|
||||
uv run python run.py --status
|
||||
|
||||
# Alerting
|
||||
alert-check:
|
||||
uv run python run.py --alert-check
|
||||
|
||||
alert-test:
|
||||
uv run python run.py --alert-test
|
||||
|
||||
# Docker
|
||||
docker-build:
|
||||
docker build -t ping-river-monitor .
|
||||
|
||||
docker-run:
|
||||
docker-compose -f docker-compose.victoriametrics.yml up -d
|
||||
|
||||
docker-stop:
|
||||
docker-compose -f docker-compose.victoriametrics.yml down
|
||||
|
||||
docker-logs:
|
||||
docker-compose -f docker-compose.victoriametrics.yml logs -f
|
||||
|
||||
# Documentation
|
||||
docs:
|
||||
cd docs && make html
|
||||
|
||||
# Monitoring
|
||||
health-check:
|
||||
curl -f http://localhost:8000/health || exit 1
|
||||
|
||||
metrics:
|
||||
curl -s http://localhost:8000/metrics | jq .
|
||||
|
||||
# Development helpers
|
||||
dev-setup: install-dev
|
||||
cp .env.example .env
|
||||
@echo "Development environment set up!"
|
||||
@echo "Edit .env file with your configuration"
|
||||
|
||||
# Production deployment
|
||||
deploy-check:
|
||||
uv run python run.py --test
|
||||
@echo "Deployment check passed!"
|
||||
|
||||
# Database management
|
||||
setup-postgres:
|
||||
uv run python scripts/setup_postgres.py
|
||||
|
||||
test-postgres:
|
||||
uv run python -c "from scripts.setup_postgres import test_postgres_connection; from src.config import Config; config = Config.get_database_config(); test_postgres_connection(config['connection_string'])"
|
||||
|
||||
migrate-sqlite:
|
||||
uv run python scripts/migrate_sqlite_to_postgres.py
|
||||
|
||||
migrate-fast:
|
||||
uv run python scripts/migrate_sqlite_to_postgres.py --fast
|
||||
|
||||
analyze-sqlite:
|
||||
uv run python scripts/migrate_sqlite_to_postgres.py --dry-run
|
||||
|
||||
# Git helpers
|
||||
git-setup:
|
||||
git remote add origin https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor.git
|
||||
@echo "Git remote configured!"
|
||||
|
||||
# Quick start
|
||||
quick-start: install dev-setup run-test
|
||||
@echo "Quick start completed!"
|
||||
@echo "Run 'make run-api' to start the web interface"
|
||||
|
||||
# Gitea Actions
|
||||
validate-workflows:
|
||||
@echo "Validating Gitea Actions workflows..."
|
||||
@for file in .gitea/workflows/*.yml; do \
|
||||
echo "Checking $$file..."; \
|
||||
uv run python -c "import yaml; yaml.safe_load(open('$$file', encoding='utf-8'))" || exit 1; \
|
||||
done
|
||||
@echo "✅ All workflows are valid"
|
||||
|
||||
workflow-test:
|
||||
@echo "Testing workflow components locally..."
|
||||
make test
|
||||
make lint
|
||||
make format
|
||||
@echo "✅ Workflow test completed"
|
||||
find . -type d -name "__pycache__" -prune -exec rm -rf {} +
|
||||
rm -rf .pytest_cache/ htmlcov/ dist/ build/ *.egg-info/
|
||||
@@ -1,287 +0,0 @@
|
||||
# PostgreSQL Setup for Northern Thailand Ping River Monitor
|
||||
|
||||
This guide helps you configure PostgreSQL as the database backend for the water monitoring system.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- PostgreSQL server running on a remote machine (already available)
|
||||
- Network connectivity to the PostgreSQL server
|
||||
- Database credentials (username, password, host, port)
|
||||
|
||||
## Quick Setup
|
||||
|
||||
### 1. Configure Environment
|
||||
|
||||
Copy the example environment file and configure it:
|
||||
|
||||
```bash
|
||||
cp .env.example .env
|
||||
```
|
||||
|
||||
Edit `.env` and update the PostgreSQL configuration:
|
||||
|
||||
```bash
|
||||
# Database Configuration
|
||||
DB_TYPE=postgresql
|
||||
|
||||
# PostgreSQL Configuration (Remote Server)
|
||||
POSTGRES_CONNECTION_STRING=postgresql://username:password@your-postgres-host:5432/water_monitoring
|
||||
```
|
||||
|
||||
### 2. Run Setup Script
|
||||
|
||||
Use the interactive setup script:
|
||||
|
||||
```bash
|
||||
# Using uv
|
||||
uv run python scripts/setup_postgres.py
|
||||
|
||||
# Or using make
|
||||
make setup-postgres
|
||||
```
|
||||
|
||||
The script will:
|
||||
- Test your database connection
|
||||
- Create the database if it doesn't exist
|
||||
- Initialize the required tables and indexes
|
||||
- Set up sample monitoring stations
|
||||
|
||||
### 3. Test Connection
|
||||
|
||||
Test your PostgreSQL connection:
|
||||
|
||||
```bash
|
||||
make test-postgres
|
||||
```
|
||||
|
||||
### 4. Run the Application
|
||||
|
||||
Start collecting data:
|
||||
|
||||
```bash
|
||||
# Run a test cycle
|
||||
make run-test
|
||||
|
||||
# Start the web API
|
||||
make run-api
|
||||
```
|
||||
|
||||
## Manual Configuration
|
||||
|
||||
If you prefer manual setup, here's what you need:
|
||||
|
||||
### Connection String Format
|
||||
|
||||
```
|
||||
postgresql://username:password@host:port/database
|
||||
```
|
||||
|
||||
**Examples:**
|
||||
- Basic: `postgresql://postgres:mypassword@192.168.1.100:5432/water_monitoring`
|
||||
- With SSL: `postgresql://user:pass@host:5432/db?sslmode=require`
|
||||
- With connection pooling: `postgresql://user:pass@host:5432/db?pool_size=20&max_overflow=0`
|
||||
|
||||
### Environment Variables
|
||||
|
||||
| Variable | Description | Example |
|
||||
|----------|-------------|---------|
|
||||
| `DB_TYPE` | Database type | `postgresql` |
|
||||
| `POSTGRES_CONNECTION_STRING` | Full connection string | See above |
|
||||
|
||||
### Database Schema
|
||||
|
||||
The application uses these main tables:
|
||||
|
||||
1. **stations** - Monitoring station information
|
||||
2. **water_measurements** - Time series water level data
|
||||
3. **alert_thresholds** - Warning/danger level definitions
|
||||
4. **data_quality_log** - Data collection issue tracking
|
||||
|
||||
See `sql/init_postgres.sql` for the complete schema.
|
||||
|
||||
## Connection Options
|
||||
|
||||
### SSL Connection
|
||||
|
||||
For secure connections, add SSL parameters:
|
||||
|
||||
```bash
|
||||
POSTGRES_CONNECTION_STRING=postgresql://user:pass@host:5432/db?sslmode=require
|
||||
```
|
||||
|
||||
SSL modes:
|
||||
- `disable` - No SSL
|
||||
- `require` - Require SSL
|
||||
- `prefer` - Use SSL if available
|
||||
- `verify-ca` - Verify certificate authority
|
||||
- `verify-full` - Full certificate verification
|
||||
|
||||
### Connection Pooling
|
||||
|
||||
For high-performance applications, configure connection pooling:
|
||||
|
||||
```bash
|
||||
POSTGRES_CONNECTION_STRING=postgresql://user:pass@host:5432/db?pool_size=20&max_overflow=0
|
||||
```
|
||||
|
||||
Parameters:
|
||||
- `pool_size` - Number of connections to maintain
|
||||
- `max_overflow` - Additional connections allowed
|
||||
- `pool_timeout` - Seconds to wait for connection
|
||||
- `pool_recycle` - Seconds before connection refresh
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Common Issues
|
||||
|
||||
**1. Connection Refused**
|
||||
```
|
||||
psycopg2.OperationalError: could not connect to server
|
||||
```
|
||||
- Check if PostgreSQL server is running
|
||||
- Verify host/port in connection string
|
||||
- Check firewall settings
|
||||
|
||||
**2. Authentication Failed**
|
||||
```
|
||||
psycopg2.OperationalError: FATAL: password authentication failed
|
||||
```
|
||||
- Verify username/password in connection string
|
||||
- Check PostgreSQL pg_hba.conf configuration
|
||||
- Ensure user has database access permissions
|
||||
|
||||
**3. Database Does Not Exist**
|
||||
```
|
||||
psycopg2.OperationalError: FATAL: database "water_monitoring" does not exist
|
||||
```
|
||||
- Run the setup script to create the database
|
||||
- Or manually create: `CREATE DATABASE water_monitoring;`
|
||||
|
||||
**4. Permission Denied**
|
||||
```
|
||||
psycopg2.ProgrammingError: permission denied for table
|
||||
```
|
||||
- Ensure user has appropriate permissions
|
||||
- Grant access: `GRANT ALL PRIVILEGES ON DATABASE water_monitoring TO username;`
|
||||
|
||||
### Network Configuration
|
||||
|
||||
For remote PostgreSQL servers, ensure:
|
||||
|
||||
1. **PostgreSQL allows remote connections** (`postgresql.conf`):
|
||||
```
|
||||
listen_addresses = '*'
|
||||
port = 5432
|
||||
```
|
||||
|
||||
2. **Client authentication is configured** (`pg_hba.conf`):
|
||||
```
|
||||
# Allow connections from your application server
|
||||
host water_monitoring username your.app.ip/32 md5
|
||||
```
|
||||
|
||||
3. **Firewall allows PostgreSQL port**:
|
||||
```bash
|
||||
# On PostgreSQL server
|
||||
sudo ufw allow 5432/tcp
|
||||
```
|
||||
|
||||
### Performance Tuning
|
||||
|
||||
For optimal performance with time series data:
|
||||
|
||||
1. **Increase work_mem** for sorting operations
|
||||
2. **Tune shared_buffers** for caching
|
||||
3. **Configure maintenance_work_mem** for indexing
|
||||
4. **Set up regular VACUUM and ANALYZE** for statistics
|
||||
|
||||
Example PostgreSQL configuration additions:
|
||||
```
|
||||
# postgresql.conf
|
||||
shared_buffers = 256MB
|
||||
work_mem = 16MB
|
||||
maintenance_work_mem = 256MB
|
||||
effective_cache_size = 1GB
|
||||
```
|
||||
|
||||
## Monitoring
|
||||
|
||||
### Check Application Status
|
||||
|
||||
```bash
|
||||
# View current configuration
|
||||
uv run python -c "from src.config import Config; Config.print_settings()"
|
||||
|
||||
# Test database connection
|
||||
make test-postgres
|
||||
|
||||
# Check latest data
|
||||
psql "postgresql://user:pass@host:5432/water_monitoring" -c "SELECT COUNT(*) FROM water_measurements;"
|
||||
```
|
||||
|
||||
### PostgreSQL Monitoring
|
||||
|
||||
Connect directly to check database status:
|
||||
|
||||
```bash
|
||||
# Connect to database
|
||||
psql "postgresql://username:password@host:5432/water_monitoring"
|
||||
|
||||
# Check table sizes
|
||||
\dt+
|
||||
|
||||
# View latest measurements
|
||||
SELECT * FROM latest_measurements LIMIT 10;
|
||||
|
||||
# Check data quality
|
||||
SELECT issue_type, COUNT(*) FROM data_quality_log
|
||||
WHERE created_at > NOW() - INTERVAL '24 hours'
|
||||
GROUP BY issue_type;
|
||||
```
|
||||
|
||||
## Backup and Maintenance
|
||||
|
||||
### Backup Database
|
||||
|
||||
```bash
|
||||
# Full backup
|
||||
pg_dump "postgresql://user:pass@host:5432/water_monitoring" > backup.sql
|
||||
|
||||
# Data only
|
||||
pg_dump --data-only "postgresql://user:pass@host:5432/water_monitoring" > data_backup.sql
|
||||
```
|
||||
|
||||
### Restore Database
|
||||
|
||||
```bash
|
||||
# Restore full backup
|
||||
psql "postgresql://user:pass@host:5432/water_monitoring" < backup.sql
|
||||
|
||||
# Restore data only
|
||||
psql "postgresql://user:pass@host:5432/water_monitoring" < data_backup.sql
|
||||
```
|
||||
|
||||
### Regular Maintenance
|
||||
|
||||
Set up regular maintenance tasks:
|
||||
|
||||
```sql
|
||||
-- Update table statistics (run weekly)
|
||||
ANALYZE;
|
||||
|
||||
-- Reclaim disk space (run monthly)
|
||||
VACUUM;
|
||||
|
||||
-- Reindex tables (run quarterly)
|
||||
REINDEX DATABASE water_monitoring;
|
||||
```
|
||||
|
||||
## Next Steps
|
||||
|
||||
1. Set up monitoring and alerting
|
||||
2. Configure data retention policies
|
||||
3. Set up automated backups
|
||||
4. Implement connection pooling if needed
|
||||
5. Configure SSL for production use
|
||||
|
||||
For more advanced configuration, see the [PostgreSQL documentation](https://www.postgresql.org/docs/).
|
||||
@@ -2,8 +2,9 @@
|
||||
|
||||
Live water levels, discharge, rainfall and machine-learning flood forecasts for the
|
||||
Ping River basin around Chiang Mai. Collects hourly gauge data from public sources,
|
||||
keeps the full history in PostgreSQL, and serves a bilingual dashboard, an open REST
|
||||
API, and 6/12/24-hour flood-risk forecasts per gauge.
|
||||
keeps the full history in PostgreSQL, serves a bilingual dashboard and an open REST
|
||||
API, forecasts the flood risk per gauge 6/12/24 hours ahead, and pushes alerts when
|
||||
a gauge crosses its warning or danger level.
|
||||
|
||||
**Live: [water.buildfor.life](https://water.buildfor.life/)** · API reference at
|
||||
[/docs](https://water.buildfor.life/docs) · built by [buildfor.life](https://buildfor.life)
|
||||
@@ -11,9 +12,9 @@ after the [October 2024 flood](https://buildfor.life/blog/chiang-mai-flood-2024/
|
||||
background in [Teaching a Model to See the Ping River Rise 13 Hours Early](https://buildfor.life/blog/ping-river-monitor/).
|
||||
|
||||
[](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/actions)
|
||||
[](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/actions)
|
||||
[](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/actions)
|
||||
[](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/actions)
|
||||
[](https://python.org)
|
||||
[](LICENSE)
|
||||
|
||||
## What it does
|
||||
@@ -21,11 +22,12 @@ background in [Teaching a Model to See the Ping River Rise 13 Hours Early](https
|
||||
- **Collects** hourly water level and discharge from 16 Royal Irrigation Department
|
||||
(RID) telemetry gauges, Chiang Dao to the southern basin, since 2018-08; hourly
|
||||
rainfall and water level from 400+ ThaiWater/HII stations; Open-Meteo catchment
|
||||
rainfall (archive + 48 h forecast); daily Mae Ngat reservoir state. Every source and
|
||||
its quirks: [docs/DATA_SOURCES.md](docs/DATA_SOURCES.md).
|
||||
rainfall (archive + 48 h forecast); daily state of the large RID dams, Mae Ngat
|
||||
above all. Every source and its quirks: [docs/DATA_SOURCES.md](docs/DATA_SOURCES.md).
|
||||
- **Fills gaps.** The raw RID grid had readings for ~56 % of hours; a full-history
|
||||
re-fetch plus HII cross-fill brought it to ~93 %. `GET /api/stats` reports the
|
||||
current figure.
|
||||
re-fetch plus HII cross-fill brought it to ~93 %. RID's report often arrives
|
||||
hours late in the evening, so late readings are always stored and holes are
|
||||
re-fetched. `GET /api/stats` reports the current coverage.
|
||||
- **Forecasts.** Per gauge and horizon, a gradient-boosted model predicts the rise
|
||||
within 6/12/24 h and the probability of crossing the station's warning and danger
|
||||
levels. Trained on the monitor's own history plus catchment rain; evaluated
|
||||
@@ -39,42 +41,38 @@ background in [Teaching a Model to See the Ping River Rise 13 Hours Early](https
|
||||
- **Notifies.** Public push alerts over a self-hosted [ntfy](https://ntfy.sh): one
|
||||
message when a gauge crosses its warning or danger level, one all-clear on the
|
||||
way down, an opt-in early-warning topic from the model, nothing in between.
|
||||
Subscribe from the free app, no account. Matrix room alerts for a team are
|
||||
also supported.
|
||||
Subscribe from the free app, no account. A Matrix room can get the same
|
||||
messages for a team. [docs/NOTIFICATIONS.md](docs/NOTIFICATIONS.md).
|
||||
|
||||
## Quick start
|
||||
## How it runs
|
||||
|
||||
Python **3.11** (3.13 breaks the pinned `psycopg2-binary`), PostgreSQL for anything
|
||||
beyond a quick look, [uv](https://docs.astral.sh/uv/).
|
||||
|
||||
```bash
|
||||
git clone https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor.git
|
||||
cd Northern-Thailand-Ping-River-Monitor
|
||||
uv sync --python 3.11
|
||||
cp .env.example .env # DB_TYPE, POSTGRES_CONNECTION_STRING, optional MATRIX_*
|
||||
uv run python run.py --web-api # dashboard + API on http://localhost:8000
|
||||
```text
|
||||
RID gauges · ThaiWater/HII · RID dams · Open-Meteo
|
||||
│ hourly
|
||||
▼
|
||||
┌──────────────────────────────────────┐ ┌───────────────────────────┐
|
||||
│ ping-collector (Rust, one process) │ │ monthly retrain (Python) │
|
||||
│ collect → forecast → notify │◄───────│ scikit-learn, exports │
|
||||
│ website + API on :8000 │ models │ models/flood_*.json.gz │
|
||||
└──────┬──────────────┬────────────────┘ └─────────────┬─────────────┘
|
||||
│ │ │
|
||||
▼ ▼ │
|
||||
ntfy topics, PostgreSQL ◄──────────── history ────────────┘
|
||||
Matrix room (gauges, rain, dams, forecasts, alert state)
|
||||
```
|
||||
|
||||
`DB_TYPE=sqlite` works for the dashboard and API; the forecasting path expects the
|
||||
PostgreSQL history.
|
||||
|
||||
```bash
|
||||
uv run python run.py --status # collector status
|
||||
uv run python run.py --test # one collection cycle
|
||||
uv run python run.py --fill-gaps 7 # re-fetch the last 7 days from RID
|
||||
uv run python run.py --collect-hii # one ThaiWater/HII collection cycle
|
||||
uv run python run.py --alert-check # evaluate thresholds, notify Matrix
|
||||
uv run python scripts/train_flood_model.py --stations all # retrain (~12 min)
|
||||
make test # pytest, synthetic data, no network
|
||||
make format # black + isort (the CI contract)
|
||||
```
|
||||
|
||||
RID collection in `--web-api` mode: polled at :05 past each hour
|
||||
(`RID_POLL_OFFSET_MINUTES`); while RID has not published the current hour yet,
|
||||
retried every `RID_RETRY_MINUTES` (10). Every poll stores all readings the report
|
||||
has, and yesterday's report is re-fetched while it still has holes (RID's evening
|
||||
hours often arrive 1-2 h late). On startup the leader re-fetches holes from the
|
||||
last `GAP_FILL_DAYS_ON_START` (14) days; for older history run `--fill-gaps all`.
|
||||
- **The service** is one Rust binary, `ping-collector` ([rust/README.md](rust/README.md)),
|
||||
run by systemd. It polls RID at :05 past each hour and retries every 10 minutes
|
||||
while the report is behind, fetches HII, the dams and the rain once an hour,
|
||||
forecasts after every cycle that brought new readings, sends the notifications,
|
||||
and serves the dashboard and API, in under 100 MB of memory with every model
|
||||
loaded.
|
||||
- **The models** are trained in Python (`src/ml`, scikit-learn
|
||||
HistGradientBoosting) by a monthly timer and exported for the Rust forecaster,
|
||||
which reproduces Python's features bit for bit and reloads new models without
|
||||
a restart.
|
||||
- Until October 2026 all of this was a Python/FastAPI app; how and why it moved:
|
||||
[docs/RUST_REWRITE.md](docs/RUST_REWRITE.md).
|
||||
|
||||
## API
|
||||
|
||||
@@ -88,76 +86,112 @@ Asia/Bangkok wall-clock without an offset suffix.
|
||||
| `GET /measurements/history/{code}?hours=N` | Hourly history; or `?start=YYYY-MM-DD&end=YYYY-MM-DD`; `limit` ≤ 100000 |
|
||||
| `GET /forecast` | Current flood-risk forecast, every station × horizon, with thresholds and P.1 inundation-stage probabilities |
|
||||
| `GET /api/forecast/history/{code}?hours=N&horizon=24` | Forecasts as issued, for auditing lead time after the fact |
|
||||
| `GET /api/forecast/skill?station_code=P.1` | Issued forecasts vs what happened, per deployed model version |
|
||||
| `GET /api/hii/rainfall/latest`, `/api/hii/waterlevel/latest` | Latest ThaiWater/HII gauge readings |
|
||||
| `GET /api/hii/rainfall/catchment?days=N` | HII gauge catchment-mean rain next to the Open-Meteo series the model uses |
|
||||
| `GET /api/forecast/skill?station_code=P.1` | Issued forecasts vs what happened, per deployed model version |
|
||||
| `GET /api/notifications` | ntfy server and topic names for the subscribe panel |
|
||||
| `GET /api/stats` | Row counts per source, date range, coverage |
|
||||
| `GET /health` | DB / upstream / memory checks |
|
||||
| `GET /health` | Database and data freshness (RID, ThaiWater as backup), the RID API, memory |
|
||||
| `GET /scraping/status` | Collection runs, the last forecast run, service version |
|
||||
|
||||
Interactive reference with schemas: [water.buildfor.life/docs](https://water.buildfor.life/docs).
|
||||
Responses are cached briefly server-side; poll no faster than once a minute — the data
|
||||
changes hourly.
|
||||
Interactive reference with schemas: [water.buildfor.life/docs](https://water.buildfor.life/docs)
|
||||
(also `/redoc`, `/openapi.json`). Responses are cached briefly server-side; poll no
|
||||
faster than once a minute — the data changes hourly. Station editing
|
||||
(`POST/PUT/DELETE /stations`), `/config` and `POST /scrape/trigger` need the
|
||||
server's `ADMIN_API_KEY` in an `X-API-Key` header.
|
||||
|
||||
## Run it locally
|
||||
|
||||
Needs Rust >= 1.88 ([rustup](https://rustup.rs); on Linux also `pkg-config` and
|
||||
`libssl-dev`) and PostgreSQL. Python 3.11 with [uv](https://docs.astral.sh/uv/)
|
||||
only for training models.
|
||||
|
||||
```bash
|
||||
git clone https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor.git
|
||||
cd Northern-Thailand-Ping-River-Monitor
|
||||
docker run -d --name ping-pg -e POSTGRES_PASSWORD=ping -e POSTGRES_DB=ping -p 55432:5432 postgres:16
|
||||
cp .env.example .env
|
||||
# POSTGRES_CONNECTION_STRING=postgresql://postgres:ping@localhost:55432/ping
|
||||
# WEB_LISTEN=127.0.0.1:8000
|
||||
cargo build --release --manifest-path rust/Cargo.toml
|
||||
rust/target/release/ping-collector check-db # create the tables
|
||||
rust/target/release/ping-collector fill-gaps 30 # the last 30 days from RID
|
||||
rust/target/release/ping-collector run # collect, forecast, serve http://127.0.0.1:8000
|
||||
```
|
||||
|
||||
Notifications stay off until `NTFY_SERVER` or `MATRIX_*` is set. Without trained
|
||||
models the forecast is the persistence heuristic, marked `source: "heuristic"`.
|
||||
Training needs years of history (the server's database, or `FLOOD_ML_DB_URL`) and
|
||||
the Open-Meteo archive:
|
||||
|
||||
```bash
|
||||
uv sync --python 3.11
|
||||
uv run python scripts/train_flood_model.py --stations all # writes models/flood_*
|
||||
uv run python scripts/evaluate_variants.py --stations P.1 # the rolling-origin harness
|
||||
```
|
||||
|
||||
`make help` lists the rest (`make check` runs every CI gate locally).
|
||||
|
||||
## Deployment
|
||||
|
||||
Production is a systemd unit on a small VPS behind Cloudflare, updated by `git pull`.
|
||||
`scripts/install.sh` (run as root from a checkout) creates the `water-monitor` user,
|
||||
deploys to `/opt/thailand-water-monitor`, runs `uv sync` into `.venv`, installs
|
||||
`water-monitor.service` and the monthly `water-monitor-retrain.timer`.
|
||||
Production is one systemd unit on a small VPS behind Cloudflare; the app directory
|
||||
`/opt/thailand-water-monitor` is a git checkout. To update:
|
||||
|
||||
```bash
|
||||
cd /opt/thailand-water-monitor && git pull
|
||||
cargo build --release --locked --manifest-path rust/Cargo.toml
|
||||
sudo bash scripts/install.sh
|
||||
sudo systemctl start water-monitor.service
|
||||
systemctl list-timers water-monitor-retrain.timer
|
||||
```
|
||||
|
||||
The retrain timer runs `scripts/retrain.sh`, which trains into `models/.staging`,
|
||||
refuses to promote anything that is not a rain-enabled (`hgb-v3+`) set covering the
|
||||
expected stations, and renames the bundles into place. Details and the operations
|
||||
runbook: [docs/FLOOD_FORECASTING.md](docs/FLOOD_FORECASTING.md) sections 6–8.
|
||||
`scripts/install.sh` (idempotent, also for a fresh host) installs the binary and
|
||||
`ping-collector.service`, syncs the Python environment for the monthly
|
||||
`water-monitor-retrain.timer`, runs a dry run of every source and a forecast check
|
||||
against the Python reference, restarts the service and waits for the website. The
|
||||
retrain trains into `models/.staging`, refuses to promote anything that is not a
|
||||
rain-enabled (`hgb-v3+`) set covering the expected stations, and renames the
|
||||
bundles into place; the service picks them up on its next forecast. Details and
|
||||
the operations runbook: [docs/FLOOD_FORECASTING.md](docs/FLOOD_FORECASTING.md)
|
||||
sections 6–8. Configuration: [.env.example](.env.example).
|
||||
|
||||
## Repository layout
|
||||
|
||||
```
|
||||
src/ collector, API (web_api.py), dashboard (static/dashboard.html)
|
||||
src/ml/ features, training, evaluation harness, prediction, rain/dam/HII loaders
|
||||
scripts/ train_flood_model.py, retrain.sh, evaluate_variants.py, install.sh, dev_proxy.py
|
||||
tests/ pytest suite (synthetic data; no DB or network)
|
||||
docs/ FLOOD_FORECASTING.md, DATA_SOURCES.md, deployment and station guides
|
||||
models/ trained bundles + metrics.json (gitignored) and evaluation results (tracked)
|
||||
.gitea/workflows/ ci (format/lint/tests), security (pip-audit/bandit), docs (link + OpenAPI checks)
|
||||
rust/collector/ ping-collector: collection, forecast run, ntfy/Matrix, website + API (src/web)
|
||||
rust/forecast/ forecast inference from the exported models; parity tests against Python
|
||||
src/static/ the dashboard (single file, EN/TH) and its map data
|
||||
src/data/ the bundled station list
|
||||
src/ml/ features, training, evaluation harness, export; predict.py is the reference
|
||||
src/*.py HII and RID-dam clients/stores for the archive backfills
|
||||
scripts/ install.sh, retrain.sh, train/evaluate/backfill tools, systemd units
|
||||
tests/ pytest (synthetic data; database tests need TEST_DATABASE_URL)
|
||||
docs/ model write-up, data sources, notifications, Matrix, rewrite history
|
||||
models/ trained bundles (gitignored) and evaluation results
|
||||
```
|
||||
|
||||
## Documentation
|
||||
|
||||
- [docs/FLOOD_FORECASTING.md](docs/FLOOD_FORECASTING.md) — the model: data, features, evaluation, measured performance, negatives, deployment, retraining
|
||||
- [docs/DATA_SOURCES.md](docs/DATA_SOURCES.md) — every ingested and candidate source, endpoints, quirks
|
||||
- [docs/STATION_MANAGEMENT_GUIDE.md](docs/STATION_MANAGEMENT_GUIDE.md) — adding/editing gauges
|
||||
- [docs/DATABASE_DEPLOYMENT_GUIDE.md](docs/DATABASE_DEPLOYMENT_GUIDE.md), [POSTGRESQL_SETUP.md](POSTGRESQL_SETUP.md) — database setup
|
||||
- [docs/NOTIFICATIONS.md](docs/NOTIFICATIONS.md) — public push alerts: topics, semantics, ntfy deployment
|
||||
- [docs/MATRIX_QUICK_START.md](docs/MATRIX_QUICK_START.md) — Matrix room alerts for a team
|
||||
- [docs/GAP_FILLING_GUIDE.md](docs/GAP_FILLING_GUIDE.md) — data integrity tooling
|
||||
- [docs/MATRIX_QUICK_START.md](docs/MATRIX_QUICK_START.md) — the same alerts in a Matrix room
|
||||
- [rust/README.md](rust/README.md) — the service: commands, configuration, develop, deploy
|
||||
- [docs/RUST_REWRITE.md](docs/RUST_REWRITE.md) — why and how it moved from Python to Rust
|
||||
- [docs/references/NOTABLE_DOCUMENTS.md](docs/references/NOTABLE_DOCUMENTS.md) — official Thai government resources
|
||||
- Public overview: [buildfor.life/docs/tooling/ping-river-monitor](https://buildfor.life/docs/tooling/ping-river-monitor/)
|
||||
|
||||
Other database backends (VictoriaMetrics, InfluxDB, MySQL, SQLite) and the Grafana
|
||||
dashboards under `grafana/` are supported by the adapters but not what production
|
||||
runs; see [docs/VICTORIAMETRICS_SETUP.md](docs/VICTORIAMETRICS_SETUP.md) if you want them.
|
||||
|
||||
## Contributing
|
||||
|
||||
`make format` before committing (black 88 columns, isort black profile — the CI gate),
|
||||
`make test` must stay green, tests use synthetic data only. See
|
||||
`make format` before committing, `make check` must stay green; see
|
||||
[CONTRIBUTING.md](CONTRIBUTING.md). Issues and merge requests on
|
||||
[git.b4l.co.th](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor).
|
||||
|
||||
## Data sources and thanks
|
||||
|
||||
Royal Irrigation Department (RID) gauge telemetry; Hydro-Informatics Institute (HII) /
|
||||
ThaiWater open API; Open-Meteo; OpenStreetMap contributors for the river geometry;
|
||||
Chiang Mai Municipality for the inundation map the P.1 stages are keyed to. All
|
||||
instruments are theirs; we aggregate, store, fill gaps and forecast.
|
||||
Royal Irrigation Department (RID) gauge telemetry and reservoir reports;
|
||||
Hydro-Informatics Institute (HII) / ThaiWater open API; Open-Meteo; OpenStreetMap
|
||||
contributors for the river geometry; Chiang Mai Municipality for the inundation map
|
||||
the P.1 stages are keyed to. All instruments are theirs; we aggregate, store, fill
|
||||
gaps and forecast.
|
||||
|
||||
## License
|
||||
|
||||
|
||||
@@ -1,278 +0,0 @@
|
||||
# SQLite to PostgreSQL Migration Guide
|
||||
|
||||
This guide helps you migrate your existing SQLite water monitoring data to PostgreSQL.
|
||||
|
||||
## Quick Migration
|
||||
|
||||
### 1. Analyze Your SQLite Database (Optional)
|
||||
|
||||
First, check what's in your SQLite database:
|
||||
|
||||
```bash
|
||||
# Analyze without migrating
|
||||
make analyze-sqlite
|
||||
|
||||
# Or specify a specific SQLite file
|
||||
uv run python scripts/migrate_sqlite_to_postgres.py --dry-run /path/to/your/database.db
|
||||
```
|
||||
|
||||
### 2. Run the Migration
|
||||
|
||||
```bash
|
||||
# Auto-detect SQLite file and migrate
|
||||
make migrate-sqlite
|
||||
|
||||
# Or specify a specific SQLite file
|
||||
uv run python scripts/migrate_sqlite_to_postgres.py /path/to/your/database.db
|
||||
```
|
||||
|
||||
The migration tool will:
|
||||
- ✅ Connect to both databases
|
||||
- ✅ Analyze your SQLite schema automatically
|
||||
- ✅ Migrate station information
|
||||
- ✅ Migrate all measurement data in batches
|
||||
- ✅ Handle different SQLite table structures
|
||||
- ✅ Verify the migration results
|
||||
- ✅ Generate a detailed log file
|
||||
|
||||
## What Gets Migrated
|
||||
|
||||
### Station Data
|
||||
- Station IDs and codes
|
||||
- Thai and English names
|
||||
- Coordinates (latitude/longitude)
|
||||
- Geohash data (if available)
|
||||
- Creation/update timestamps
|
||||
|
||||
### Measurement Data
|
||||
- Water level readings
|
||||
- Discharge measurements
|
||||
- Discharge percentages
|
||||
- Timestamps
|
||||
- Station associations
|
||||
- Data quality status
|
||||
|
||||
## Supported SQLite Schemas
|
||||
|
||||
The migration tool automatically detects and handles various SQLite table structures:
|
||||
|
||||
### Modern Schema
|
||||
```sql
|
||||
-- Stations
|
||||
stations: id, station_code, station_name_th, station_name_en, latitude, longitude, geohash
|
||||
|
||||
-- Measurements
|
||||
water_measurements: timestamp, station_id, water_level, discharge, discharge_percent, status
|
||||
```
|
||||
|
||||
### Legacy Schema
|
||||
```sql
|
||||
-- Stations
|
||||
water_stations: station_id, station_code, station_name, lat, lon
|
||||
|
||||
-- Measurements
|
||||
measurements: timestamp, station_id, water_level, discharge, discharge_percent
|
||||
```
|
||||
|
||||
### Simple Schema
|
||||
```sql
|
||||
-- Any table with basic water level data
|
||||
-- The tool will adapt and map columns automatically
|
||||
```
|
||||
|
||||
## Migration Process
|
||||
|
||||
### Step 1: Database Connection
|
||||
- Connects to your SQLite database
|
||||
- Verifies PostgreSQL connection
|
||||
- Validates configuration
|
||||
|
||||
### Step 2: Schema Analysis
|
||||
- Scans SQLite tables and columns
|
||||
- Reports data counts
|
||||
- Identifies table structures
|
||||
|
||||
### Step 3: Station Migration
|
||||
- Extracts station metadata
|
||||
- Maps to PostgreSQL format
|
||||
- Handles missing data gracefully
|
||||
|
||||
### Step 4: Measurement Migration
|
||||
- Processes data in batches (1000 records at a time)
|
||||
- Converts timestamps correctly
|
||||
- Preserves all measurement values
|
||||
- Shows progress during migration
|
||||
|
||||
### Step 5: Verification
|
||||
- Compares record counts
|
||||
- Validates data integrity
|
||||
- Reports migration statistics
|
||||
|
||||
## Command Options
|
||||
|
||||
```bash
|
||||
# Basic migration (auto-detects SQLite file)
|
||||
uv run python scripts/migrate_sqlite_to_postgres.py
|
||||
|
||||
# Specify SQLite database path
|
||||
uv run python scripts/migrate_sqlite_to_postgres.py /path/to/database.db
|
||||
|
||||
# Dry run (analyze only, no migration)
|
||||
uv run python scripts/migrate_sqlite_to_postgres.py --dry-run
|
||||
|
||||
# Custom batch size for large databases
|
||||
uv run python scripts/migrate_sqlite_to_postgres.py --batch-size 5000
|
||||
```
|
||||
|
||||
## Auto-Detection
|
||||
|
||||
The tool automatically searches for SQLite files in common locations:
|
||||
- `water_levels.db`
|
||||
- `water_monitoring.db`
|
||||
- `database.db`
|
||||
- `../water_levels.db`
|
||||
|
||||
## Migration Output
|
||||
|
||||
The tool provides detailed logging:
|
||||
|
||||
```
|
||||
========================================
|
||||
SQLite to PostgreSQL Migration Tool
|
||||
========================================
|
||||
SQLite database: water_levels.db
|
||||
PostgreSQL: postgresql
|
||||
|
||||
Step 1: Connecting to databases...
|
||||
Connected to SQLite database: water_levels.db
|
||||
Connected to PostgreSQL database
|
||||
|
||||
Step 2: Analyzing SQLite database structure...
|
||||
Table 'stations': 8 columns, 25 rows
|
||||
Table 'water_measurements': 7 columns, 15420 rows
|
||||
|
||||
Step 3: Migrating station data...
|
||||
Migrated 25 stations
|
||||
|
||||
Step 4: Migrating measurement data...
|
||||
Found 15420 measurements to migrate
|
||||
Migrated 1000/15420 measurements
|
||||
Migrated 2000/15420 measurements
|
||||
...
|
||||
Successfully migrated 15420 measurements
|
||||
|
||||
Step 5: Verifying migration...
|
||||
SQLite stations: 25
|
||||
SQLite measurements: 15420
|
||||
PostgreSQL measurements retrieved: 15420
|
||||
Migrated stations: 25
|
||||
Migrated measurements: 15420
|
||||
|
||||
========================================
|
||||
MIGRATION COMPLETED
|
||||
========================================
|
||||
Duration: 0:02:15
|
||||
Stations migrated: 25
|
||||
Measurements migrated: 15420
|
||||
No errors encountered
|
||||
```
|
||||
|
||||
## Error Handling
|
||||
|
||||
The migration tool is robust and handles:
|
||||
- **Missing tables** - Tries alternative table names
|
||||
- **Different column names** - Maps common variations
|
||||
- **Missing data** - Uses sensible defaults
|
||||
- **Invalid timestamps** - Attempts multiple date formats
|
||||
- **Connection issues** - Provides clear error messages
|
||||
- **Large datasets** - Processes in batches to avoid memory issues
|
||||
|
||||
## Log Files
|
||||
|
||||
Migration creates a detailed log file:
|
||||
- `migration.log` - Complete migration log
|
||||
- Shows all operations, errors, and statistics
|
||||
- Useful for troubleshooting
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Common Issues
|
||||
|
||||
**1. SQLite file not found**
|
||||
```
|
||||
SQLite database file not found. Please specify the path:
|
||||
python migrate_sqlite_to_postgres.py /path/to/database.db
|
||||
```
|
||||
**Solution**: Specify the correct path to your SQLite file
|
||||
|
||||
**2. PostgreSQL not configured**
|
||||
```
|
||||
Error: PostgreSQL not configured. Set DB_TYPE=postgresql in your .env file
|
||||
```
|
||||
**Solution**: Ensure your .env file has `DB_TYPE=postgresql`
|
||||
|
||||
**3. Connection failed**
|
||||
```
|
||||
Database connection error: connection refused
|
||||
```
|
||||
**Solution**: Check your PostgreSQL connection settings
|
||||
|
||||
**4. No tables found**
|
||||
```
|
||||
Could not analyze SQLite database structure
|
||||
```
|
||||
**Solution**: Verify your SQLite file contains water monitoring data
|
||||
|
||||
### Performance Tips
|
||||
|
||||
- **Large databases**: Use `--batch-size 5000` for faster processing
|
||||
- **Slow networks**: Reduce batch size to `--batch-size 100`
|
||||
- **Memory issues**: Process smaller batches
|
||||
|
||||
## After Migration
|
||||
|
||||
Once migration is complete:
|
||||
|
||||
1. **Verify data**:
|
||||
```bash
|
||||
make run-test
|
||||
make run-api
|
||||
```
|
||||
|
||||
2. **Check the web interface**: Latest readings should show your migrated data
|
||||
|
||||
3. **Backup your SQLite**: Keep the original file as backup
|
||||
|
||||
4. **Update configurations**: Remove SQLite references from configs
|
||||
|
||||
## Rollback
|
||||
|
||||
If you need to rollback:
|
||||
|
||||
1. **Clear PostgreSQL data**:
|
||||
```sql
|
||||
DELETE FROM water_measurements;
|
||||
DELETE FROM stations;
|
||||
```
|
||||
|
||||
2. **Switch back to SQLite**:
|
||||
```bash
|
||||
# In .env file
|
||||
DB_TYPE=sqlite
|
||||
WATER_DB_PATH=water_levels.db
|
||||
```
|
||||
|
||||
3. **Test the rollback**:
|
||||
```bash
|
||||
make run-test
|
||||
```
|
||||
|
||||
The migration tool is designed to be safe and can be run multiple times - it handles duplicates appropriately.
|
||||
|
||||
## Next Steps
|
||||
|
||||
After successful migration:
|
||||
- Set up automated backups for PostgreSQL
|
||||
- Configure monitoring and alerting
|
||||
- Consider data retention policies
|
||||
- Update documentation references
|
||||
@@ -1,106 +0,0 @@
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
victoriametrics:
|
||||
image: victoriametrics/victoria-metrics:latest
|
||||
container_name: ping_river_victoriametrics
|
||||
ports:
|
||||
- "8428:8428"
|
||||
volumes:
|
||||
- vm_data:/victoria-metrics-data
|
||||
- ./vm-config:/etc/victoriametrics
|
||||
command:
|
||||
- '--storageDataPath=/victoria-metrics-data'
|
||||
- '--retentionPeriod=2y'
|
||||
- '--httpListenAddr=:8428'
|
||||
- '--maxConcurrentInserts=32'
|
||||
- '--search.maxQueryDuration=60s'
|
||||
- '--search.maxConcurrentRequests=16'
|
||||
- '--dedup.minScrapeInterval=30s'
|
||||
- '--memory.allowedPercent=80'
|
||||
- '--loggerLevel=INFO'
|
||||
- '--loggerFormat=json'
|
||||
- '--search.maxSeries=1000000'
|
||||
- '--search.maxPointsPerTimeseries=100000'
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- TZ=Asia/Bangkok
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost:8428/health"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 40s
|
||||
labels:
|
||||
- "com.victoriametrics.service=ping-river-monitoring"
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:latest
|
||||
container_name: ping_river_grafana
|
||||
ports:
|
||||
- "3000:3000"
|
||||
volumes:
|
||||
- grafana_data:/var/lib/grafana
|
||||
- ./grafana/provisioning:/etc/grafana/provisioning
|
||||
- ./grafana/dashboards:/var/lib/grafana/dashboards
|
||||
environment:
|
||||
- GF_SECURITY_ADMIN_PASSWORD=admin_password
|
||||
- GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-worldmap-panel
|
||||
- GF_USERS_ALLOW_SIGN_UP=false
|
||||
- GF_SECURITY_ALLOW_EMBEDDING=true
|
||||
- GF_AUTH_ANONYMOUS_ENABLED=false
|
||||
- TZ=Asia/Bangkok
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
victoriametrics:
|
||||
condition: service_healthy
|
||||
labels:
|
||||
- "com.victoriametrics.service=ping-river-monitoring"
|
||||
|
||||
ping-river-monitor:
|
||||
build: .
|
||||
container_name: ping_river_monitor
|
||||
environment:
|
||||
- DB_TYPE=victoriametrics
|
||||
- VM_HOST=victoriametrics
|
||||
- VM_PORT=8428
|
||||
- TZ=Asia/Bangkok
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
victoriametrics:
|
||||
condition: service_healthy
|
||||
volumes:
|
||||
- ./logs:/app/logs
|
||||
labels:
|
||||
- "com.victoriametrics.service=ping-river-monitoring"
|
||||
|
||||
ping-river-api:
|
||||
build: .
|
||||
container_name: ping_river_api
|
||||
ports:
|
||||
- "8000:8000"
|
||||
environment:
|
||||
- DB_TYPE=victoriametrics
|
||||
- VM_HOST=victoriametrics
|
||||
- VM_PORT=8428
|
||||
- TZ=Asia/Bangkok
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
victoriametrics:
|
||||
condition: service_healthy
|
||||
volumes:
|
||||
- ./logs:/app/logs
|
||||
command: ["python", "-m", "src.main", "--web-api"]
|
||||
labels:
|
||||
- "com.victoriametrics.service=ping-river-monitoring"
|
||||
|
||||
volumes:
|
||||
vm_data:
|
||||
driver: local
|
||||
grafana_data:
|
||||
driver: local
|
||||
|
||||
networks:
|
||||
default:
|
||||
name: ping-river-monitoring
|
||||
driver: bridge
|
||||
@@ -1,447 +0,0 @@
|
||||
# Database Deployment Guide for Thailand Water Monitor
|
||||
|
||||
This guide covers deployment options for storing water monitoring data in production environments.
|
||||
|
||||
## 🏆 Recommendation Summary
|
||||
|
||||
| Database | Best For | Performance | Complexity | Cost |
|
||||
|----------|----------|-------------|------------|------|
|
||||
| **InfluxDB** | Time-series data, dashboards | ⭐⭐⭐⭐⭐ | ⭐⭐⭐ | ⭐⭐⭐ |
|
||||
| **VictoriaMetrics** | High-performance metrics | ⭐⭐⭐⭐⭐ | ⭐⭐⭐⭐ | ⭐⭐⭐⭐ |
|
||||
| **PostgreSQL** | Complex queries, reliability | ⭐⭐⭐⭐ | ⭐⭐ | ⭐⭐⭐⭐ |
|
||||
| **MySQL** | Familiar, existing infrastructure | ⭐⭐⭐ | ⭐⭐ | ⭐⭐⭐⭐ |
|
||||
|
||||
## 1. InfluxDB Deployment (Recommended for Time-Series)
|
||||
|
||||
### Why InfluxDB?
|
||||
- **Purpose-built** for time-series data
|
||||
- **Excellent compression** (10:1 typical ratio)
|
||||
- **Built-in retention policies** and downsampling
|
||||
- **Great Grafana integration** for dashboards
|
||||
- **High write throughput** (100k+ points/second)
|
||||
|
||||
### Docker Deployment
|
||||
|
||||
```yaml
|
||||
# docker-compose.yml
|
||||
version: '3.8'
|
||||
services:
|
||||
influxdb:
|
||||
image: influxdb:1.8
|
||||
container_name: water_influxdb
|
||||
ports:
|
||||
- "8086:8086"
|
||||
volumes:
|
||||
- influxdb_data:/var/lib/influxdb
|
||||
- ./influxdb.conf:/etc/influxdb/influxdb.conf:ro
|
||||
environment:
|
||||
- INFLUXDB_DB=water_monitoring
|
||||
- INFLUXDB_ADMIN_USER=admin
|
||||
- INFLUXDB_ADMIN_PASSWORD=your_secure_password
|
||||
- INFLUXDB_USER=water_user
|
||||
- INFLUXDB_USER_PASSWORD=water_password
|
||||
restart: unless-stopped
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:latest
|
||||
container_name: water_grafana
|
||||
ports:
|
||||
- "3000:3000"
|
||||
volumes:
|
||||
- grafana_data:/var/lib/grafana
|
||||
environment:
|
||||
- GF_SECURITY_ADMIN_PASSWORD=admin_password
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
influxdb_data:
|
||||
grafana_data:
|
||||
```
|
||||
|
||||
### Environment Variables
|
||||
```bash
|
||||
# .env file
|
||||
DB_TYPE=influxdb
|
||||
INFLUX_HOST=localhost
|
||||
INFLUX_PORT=8086
|
||||
INFLUX_DATABASE=water_monitoring
|
||||
INFLUX_USERNAME=water_user
|
||||
INFLUX_PASSWORD=water_password
|
||||
```
|
||||
|
||||
### InfluxDB Configuration
|
||||
```toml
|
||||
# influxdb.conf
|
||||
[meta]
|
||||
dir = "/var/lib/influxdb/meta"
|
||||
|
||||
[data]
|
||||
dir = "/var/lib/influxdb/data"
|
||||
wal-dir = "/var/lib/influxdb/wal"
|
||||
|
||||
# Optimize for time-series data
|
||||
cache-max-memory-size = "1g"
|
||||
cache-snapshot-memory-size = "25m"
|
||||
cache-snapshot-write-cold-duration = "10m"
|
||||
|
||||
# Retention and compression
|
||||
compact-full-write-cold-duration = "4h"
|
||||
max-series-per-database = 1000000
|
||||
max-values-per-tag = 100000
|
||||
|
||||
[coordinator]
|
||||
write-timeout = "10s"
|
||||
max-concurrent-queries = 0
|
||||
query-timeout = "0s"
|
||||
|
||||
[retention]
|
||||
enabled = true
|
||||
check-interval = "30m"
|
||||
|
||||
[http]
|
||||
enabled = true
|
||||
bind-address = ":8086"
|
||||
auth-enabled = true
|
||||
max-body-size = "25000000"
|
||||
max-concurrent-requests = 0
|
||||
max-enqueued-requests = 0
|
||||
```
|
||||
|
||||
### Production Setup Commands
|
||||
```bash
|
||||
# Start services
|
||||
docker-compose up -d
|
||||
|
||||
# Create retention policies
|
||||
docker exec -it water_influxdb influx -username admin -password your_secure_password -execute "
|
||||
CREATE RETENTION POLICY \"raw_data\" ON \"water_monitoring\" DURATION 90d REPLICATION 1 DEFAULT;
|
||||
CREATE RETENTION POLICY \"downsampled\" ON \"water_monitoring\" DURATION 730d REPLICATION 1;
|
||||
"
|
||||
|
||||
# Create continuous queries for downsampling
|
||||
docker exec -it water_influxdb influx -username admin -password your_secure_password -execute "
|
||||
CREATE CONTINUOUS QUERY \"downsample_hourly\" ON \"water_monitoring\"
|
||||
BEGIN
|
||||
SELECT mean(water_level) AS water_level, mean(discharge) AS discharge, mean(discharge_percent) AS discharge_percent
|
||||
INTO \"downsampled\".\"water_data_hourly\"
|
||||
FROM \"water_data\"
|
||||
GROUP BY time(1h), station_code, station_name_en, station_name_th
|
||||
END
|
||||
"
|
||||
```
|
||||
|
||||
## 2. VictoriaMetrics Deployment (High Performance)
|
||||
|
||||
### Why VictoriaMetrics?
|
||||
- **Extremely fast** and resource-efficient
|
||||
- **Better compression** than InfluxDB
|
||||
- **Prometheus-compatible** API
|
||||
- **Lower memory usage**
|
||||
- **Built-in clustering**
|
||||
|
||||
### Docker Deployment
|
||||
```yaml
|
||||
# docker-compose.yml
|
||||
version: '3.8'
|
||||
services:
|
||||
victoriametrics:
|
||||
image: victoriametrics/victoria-metrics:latest
|
||||
container_name: water_victoriametrics
|
||||
ports:
|
||||
- "8428:8428"
|
||||
volumes:
|
||||
- vm_data:/victoria-metrics-data
|
||||
command:
|
||||
- '--storageDataPath=/victoria-metrics-data'
|
||||
- '--retentionPeriod=2y'
|
||||
- '--httpListenAddr=:8428'
|
||||
- '--maxConcurrentInserts=16'
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
vm_data:
|
||||
```
|
||||
|
||||
### Environment Variables
|
||||
```bash
|
||||
# .env file
|
||||
DB_TYPE=victoriametrics
|
||||
VM_HOST=localhost
|
||||
VM_PORT=8428
|
||||
```
|
||||
|
||||
## 3. PostgreSQL Deployment (Relational + Time-Series)
|
||||
|
||||
### Why PostgreSQL?
|
||||
- **Mature and reliable**
|
||||
- **Excellent for complex queries**
|
||||
- **TimescaleDB extension** for time-series optimization
|
||||
- **Strong consistency guarantees**
|
||||
- **Rich ecosystem**
|
||||
|
||||
### Docker Deployment with TimescaleDB
|
||||
```yaml
|
||||
# docker-compose.yml
|
||||
version: '3.8'
|
||||
services:
|
||||
postgres:
|
||||
image: timescale/timescaledb:latest-pg14
|
||||
container_name: water_postgres
|
||||
ports:
|
||||
- "5432:5432"
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
- ./init.sql:/docker-entrypoint-initdb.d/init.sql
|
||||
environment:
|
||||
- POSTGRES_DB=water_monitoring
|
||||
- POSTGRES_USER=water_user
|
||||
- POSTGRES_PASSWORD=secure_password
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
```
|
||||
|
||||
### Database Initialization
|
||||
```sql
|
||||
-- init.sql
|
||||
CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE;
|
||||
|
||||
-- Create hypertable for time-series optimization
|
||||
CREATE TABLE water_measurements (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
timestamp TIMESTAMPTZ NOT NULL,
|
||||
station_id INT NOT NULL,
|
||||
water_level NUMERIC(10,3),
|
||||
discharge NUMERIC(10,2),
|
||||
discharge_percent NUMERIC(5,2),
|
||||
status VARCHAR(20) DEFAULT 'active',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Convert to hypertable (TimescaleDB)
|
||||
SELECT create_hypertable('water_measurements', 'timestamp', chunk_time_interval => INTERVAL '1 day');
|
||||
|
||||
-- Create indexes
|
||||
CREATE INDEX idx_water_measurements_station_time ON water_measurements (station_id, timestamp DESC);
|
||||
CREATE INDEX idx_water_measurements_timestamp ON water_measurements (timestamp DESC);
|
||||
|
||||
-- Create retention policy (keep raw data for 2 years)
|
||||
SELECT add_retention_policy('water_measurements', INTERVAL '2 years');
|
||||
|
||||
-- Create continuous aggregates for performance
|
||||
CREATE MATERIALIZED VIEW water_measurements_hourly
|
||||
WITH (timescaledb.continuous) AS
|
||||
SELECT
|
||||
time_bucket('1 hour', timestamp) AS bucket,
|
||||
station_id,
|
||||
AVG(water_level) as avg_water_level,
|
||||
MAX(water_level) as max_water_level,
|
||||
MIN(water_level) as min_water_level,
|
||||
AVG(discharge) as avg_discharge,
|
||||
MAX(discharge) as max_discharge,
|
||||
MIN(discharge) as min_discharge,
|
||||
AVG(discharge_percent) as avg_discharge_percent
|
||||
FROM water_measurements
|
||||
GROUP BY bucket, station_id;
|
||||
|
||||
-- Refresh policy for continuous aggregates
|
||||
SELECT add_continuous_aggregate_policy('water_measurements_hourly',
|
||||
start_offset => INTERVAL '1 day',
|
||||
end_offset => INTERVAL '1 hour',
|
||||
schedule_interval => INTERVAL '1 hour');
|
||||
```
|
||||
|
||||
### Environment Variables
|
||||
```bash
|
||||
# .env file
|
||||
DB_TYPE=postgresql
|
||||
POSTGRES_CONNECTION_STRING=postgresql://water_user:secure_password@localhost:5432/water_monitoring
|
||||
```
|
||||
|
||||
## 4. MySQL Deployment (Traditional Relational)
|
||||
|
||||
### Docker Deployment
|
||||
```yaml
|
||||
# docker-compose.yml
|
||||
version: '3.8'
|
||||
services:
|
||||
mysql:
|
||||
image: mysql:8.0
|
||||
container_name: water_mysql
|
||||
ports:
|
||||
- "3306:3306"
|
||||
volumes:
|
||||
- mysql_data:/var/lib/mysql
|
||||
- ./mysql.cnf:/etc/mysql/conf.d/mysql.cnf
|
||||
- ./init.sql:/docker-entrypoint-initdb.d/init.sql
|
||||
environment:
|
||||
- MYSQL_ROOT_PASSWORD=root_password
|
||||
- MYSQL_DATABASE=water_monitoring
|
||||
- MYSQL_USER=water_user
|
||||
- MYSQL_PASSWORD=water_password
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
mysql_data:
|
||||
```
|
||||
|
||||
### MySQL Configuration
|
||||
```ini
|
||||
# mysql.cnf
|
||||
[mysqld]
|
||||
# Optimize for time-series data
|
||||
innodb_buffer_pool_size = 1G
|
||||
innodb_log_file_size = 256M
|
||||
innodb_flush_log_at_trx_commit = 2
|
||||
innodb_flush_method = O_DIRECT
|
||||
|
||||
# Partitioning support
|
||||
partition = ON
|
||||
|
||||
# Query cache
|
||||
query_cache_type = 1
|
||||
query_cache_size = 128M
|
||||
|
||||
# Connection settings
|
||||
max_connections = 200
|
||||
connect_timeout = 10
|
||||
wait_timeout = 600
|
||||
```
|
||||
|
||||
### Environment Variables
|
||||
```bash
|
||||
# .env file
|
||||
DB_TYPE=mysql
|
||||
MYSQL_CONNECTION_STRING=mysql://water_user:water_password@localhost:3306/water_monitoring
|
||||
```
|
||||
|
||||
## 5. Installation and Dependencies
|
||||
|
||||
### Required Python Packages
|
||||
```bash
|
||||
# Base requirements
|
||||
pip install requests schedule
|
||||
|
||||
# Database-specific packages
|
||||
pip install influxdb # For InfluxDB
|
||||
pip install sqlalchemy pymysql # For MySQL
|
||||
pip install sqlalchemy psycopg2-binary # For PostgreSQL
|
||||
# VictoriaMetrics uses HTTP API (no extra packages needed)
|
||||
```
|
||||
|
||||
### Updated requirements.txt
|
||||
```txt
|
||||
requests>=2.28.0
|
||||
schedule>=1.2.0
|
||||
pandas>=1.5.0
|
||||
|
||||
# Database adapters (install as needed)
|
||||
influxdb>=5.3.1
|
||||
sqlalchemy>=1.4.0
|
||||
pymysql>=1.0.2
|
||||
psycopg2-binary>=2.9.0
|
||||
```
|
||||
|
||||
## 6. Production Deployment Examples
|
||||
|
||||
### Using InfluxDB (Recommended)
|
||||
```bash
|
||||
# Set environment variables
|
||||
export DB_TYPE=influxdb
|
||||
export INFLUX_HOST=your-influx-server.com
|
||||
export INFLUX_PORT=8086
|
||||
export INFLUX_DATABASE=water_monitoring
|
||||
export INFLUX_USERNAME=water_user
|
||||
export INFLUX_PASSWORD=your_secure_password
|
||||
|
||||
# Run the scraper
|
||||
python water_scraper_v3.py
|
||||
```
|
||||
|
||||
### Using PostgreSQL with TimescaleDB
|
||||
```bash
|
||||
# Set environment variables
|
||||
export DB_TYPE=postgresql
|
||||
export POSTGRES_CONNECTION_STRING=postgresql://water_user:password@your-postgres-server.com:5432/water_monitoring
|
||||
|
||||
# Run the scraper
|
||||
python water_scraper_v3.py
|
||||
```
|
||||
|
||||
### Using VictoriaMetrics
|
||||
```bash
|
||||
# Set environment variables
|
||||
export DB_TYPE=victoriametrics
|
||||
export VM_HOST=your-vm-server.com
|
||||
export VM_PORT=8428
|
||||
|
||||
# Run the scraper
|
||||
python water_scraper_v3.py
|
||||
```
|
||||
|
||||
## 7. Monitoring and Alerting
|
||||
|
||||
### Grafana Dashboard Setup
|
||||
1. **Add Data Source**: Configure your database as a Grafana data source
|
||||
2. **Import Dashboard**: Use pre-built water monitoring dashboards
|
||||
3. **Set Alerts**: Configure alerts for abnormal water levels or discharge rates
|
||||
|
||||
### Example Grafana Queries
|
||||
|
||||
#### InfluxDB Queries
|
||||
```sql
|
||||
-- Current water levels
|
||||
SELECT last("water_level") FROM "water_data" GROUP BY "station_code"
|
||||
|
||||
-- Discharge trends (last 24h)
|
||||
SELECT mean("discharge") FROM "water_data" WHERE time >= now() - 24h GROUP BY time(1h), "station_code"
|
||||
```
|
||||
|
||||
#### PostgreSQL/TimescaleDB Queries
|
||||
```sql
|
||||
-- Current water levels
|
||||
SELECT DISTINCT ON (station_id)
|
||||
station_id, water_level, discharge, timestamp
|
||||
FROM water_measurements
|
||||
ORDER BY station_id, timestamp DESC;
|
||||
|
||||
-- Hourly averages (last 24h)
|
||||
SELECT
|
||||
time_bucket('1 hour', timestamp) as hour,
|
||||
station_id,
|
||||
AVG(water_level) as avg_level,
|
||||
AVG(discharge) as avg_discharge
|
||||
FROM water_measurements
|
||||
WHERE timestamp >= NOW() - INTERVAL '24 hours'
|
||||
GROUP BY hour, station_id
|
||||
ORDER BY hour DESC;
|
||||
```
|
||||
|
||||
## 8. Performance Optimization Tips
|
||||
|
||||
### For All Databases
|
||||
- **Batch inserts**: Insert multiple measurements at once
|
||||
- **Connection pooling**: Reuse database connections
|
||||
- **Indexing**: Ensure proper indexes on timestamp and station_id
|
||||
- **Retention policies**: Automatically delete old data
|
||||
|
||||
### InfluxDB Specific
|
||||
- Use **tags** for metadata (station codes, names)
|
||||
- Use **fields** for numeric values (water levels, discharge)
|
||||
- Configure **retention policies** and **continuous queries**
|
||||
- Enable **compression** for long-term storage
|
||||
|
||||
### PostgreSQL/TimescaleDB Specific
|
||||
- Use **hypertables** for automatic partitioning
|
||||
- Create **continuous aggregates** for common queries
|
||||
- Configure **compression** for older chunks
|
||||
- Use **parallel queries** for large datasets
|
||||
|
||||
### VictoriaMetrics Specific
|
||||
- Use **labels** efficiently (similar to Prometheus)
|
||||
- Configure **retention periods** appropriately
|
||||
- Use **downsampling** for long-term storage
|
||||
- Enable **deduplication** if needed
|
||||
|
||||
This deployment guide provides production-ready configurations for all supported database backends. Choose the one that best fits your infrastructure and requirements.
|
||||
+15
-14
@@ -23,7 +23,7 @@ The only source persisted to the database and used by the ML pipeline.
|
||||
| Params | `DW[UtokID]=1`, `DW[BasinID]=6` (Ping), `DW[TimeCurrent]=<Buddhist-calendar date>`, `rows=100` |
|
||||
| Variables | Water level (m, gauge datum), discharge (m³/s, `'***'` = malformed), discharge % of channel capacity |
|
||||
| Stations | 16 P-series gauges (P.1 anchor at Nawarat Bridge; see `src/data/stations.json`) |
|
||||
| Client | `src/water_scraper_v3.py` |
|
||||
| Client | `rust/collector/src/rid.rs` (`ping-collector`; `fill-gaps` re-fetches old reports) |
|
||||
|
||||
Human-facing page: <https://hyd-app-db.rid.go.th/hydro1h.html>
|
||||
|
||||
@@ -117,9 +117,9 @@ GET .../rain_24h_graph?station_type=tele_rainfall&station_id=418&start_date=...&
|
||||
|
||||
### 2.2 `twa-api-public.thaiwater.net` — auth-gated (x-api-key / session) 🔒
|
||||
|
||||
The layer our existing `src/thaiwater.py` client uses
|
||||
(`GET /v2/waterlevel` with `x-api-key: $THAIWATER_API_KEY`; wired to
|
||||
`GET /sensors/thaiwater` in the web API, display-only, never persisted).
|
||||
The layer the website's `GET /sensors/thaiwater` uses (`GET /v2/waterlevel`
|
||||
with an `x-api-key` header from `THAIWATER_API_KEY`;
|
||||
`rust/collector/src/web/hii.rs`, display-only, never persisted).
|
||||
Without a key: HTTP 401/500. No public key-registration page was found —
|
||||
obtain a sanctioned key from HII (<https://hii.or.th>).
|
||||
|
||||
@@ -242,21 +242,22 @@ Oct 2024 flood). Mae Kuang Udom Thara is the second upstream reservoir.
|
||||
| Source | Status | Action |
|
||||
|---|---|---|
|
||||
| RID hourly gauges | ✅ Ingested (hourly → PostgreSQL) | — |
|
||||
| ThaiWater `/v2/waterlevel` | 🟡 Display-only (`src/thaiwater.py`, needs `THAIWATER_API_KEY`, never persisted) | Optionally persist |
|
||||
| HII `rain_24h` | ✅ Ingested hourly via `src/hii_collector.py` → `hii_rain_stations` + `hii_rainfall` (Ping-filtered; ~300 stations) | — |
|
||||
| HII `waterlevel_load` | ✅ Ingested hourly via `src/hii_collector.py` → `hii_wl_stations` + `hii_waterlevel` (125 Ping stations, m MSL; `rid_code` column maps mirrors like `ridhydro_P.1` → `P.1`, `offset_msl` converts MSL → gauge datum) | — |
|
||||
| ThaiWater `/v2/waterlevel` | 🟡 Display-only (`/sensors/thaiwater`, needs `THAIWATER_API_KEY`, never persisted) | Optionally persist |
|
||||
| HII `rain_24h` | ✅ Ingested hourly by ping-collector (`rust/collector/src/hii.rs`) → `hii_rain_stations` + `hii_rainfall` (Ping-filtered; ~300 stations) | — |
|
||||
| HII `waterlevel_load` | ✅ Ingested hourly by ping-collector (`rust/collector/src/hii.rs`) → `hii_wl_stations` + `hii_waterlevel` (125 Ping stations, m MSL; `rid_code` column maps mirrors like `ridhydro_P.1` → `P.1`, `offset_msl` converts MSL → gauge datum) | — |
|
||||
| HII `waterlevel_graph` | ✅ Backfill via `scripts/backfill_hii_waterlevel.py` → `hii_waterlevel` (hourly MSL + discharge, archive ≥2019; full-year windows per request; upserts never overwrite live-snapshot columns) | Run once on the box: `python scripts/backfill_hii_waterlevel.py` (defaults: 2019-01-01 → today, RID-mirror + key stations; `--stations P.1,P.67`, `--all` for every Ping station) |
|
||||
| Mae Ngat reservoir (app.rid.go.th) | ✅ Ingested (2026-08-13) | Daily storage/inflow/outflow for all large dams → `rid_reservoir_daily`; candidate model features for next retrain |
|
||||
| Satellite QPE (GSMaP/IMERG) | ❌ | Backfill training rainfall (GEE) |
|
||||
| Open-Meteo forecasts | ❌ | Add forecast features (live + 2021 archive for training) |
|
||||
| Open-Meteo rain | ✅ Hourly by ping-collector → `openmeteo_rain` (5 catchment points, past 5 days + 48 h forecast); the 2021+ archive trains the hgb-v3 rain features | — |
|
||||
| HII API key (dams, forecasts) | ❌ | Contact HII for sanctioned access |
|
||||
|
||||
**Collector configuration** (`src/hii_collector.py`): runs automatically every
|
||||
scraping cycle (hourly cadence, even while the RID scraper is in 1-minute retry
|
||||
mode) from both `--web-api` and continuous-monitoring modes; one-shot via
|
||||
`python -m src.main --collect-hii`. Env vars: `ENABLE_HII_COLLECTION`
|
||||
(default `true`), `HII_BASIN_CODE` (default `6` = Ping). Requires a SQL
|
||||
`DB_TYPE` (sqlite/postgresql/mysql); tables are created automatically.
|
||||
**Collector configuration** (ping-collector): HII, the dams and Open-Meteo are
|
||||
fetched once per clock hour, also while RID is being retried; one-shot via
|
||||
`ping-collector once`. Env vars: `ENABLE_HII_COLLECTION`,
|
||||
`ENABLE_RESERVOIR_COLLECTION`, `ENABLE_RAIN_COLLECTION` (default `true`),
|
||||
`HII_BASIN_CODE` (default `6` = Ping). Tables are created automatically
|
||||
(PostgreSQL). The archive backfills stay in Python (`scripts/backfill_*.py`,
|
||||
using `src/hii_collector.py` and `src/rid_reservoir.py`).
|
||||
|
||||
**Caveats**: `api-v3` is an undocumented backend — no SLA, no ToS, may change
|
||||
without notice. Poll hourly at most, cache aggressively, and pursue official
|
||||
|
||||
@@ -1,329 +0,0 @@
|
||||
# Debian/Linux Troubleshooting Guide
|
||||
|
||||
This guide addresses common issues when running the Thailand Water Monitor on Debian and other Linux distributions.
|
||||
|
||||
## Fixed Issues
|
||||
|
||||
### SQLAlchemy Connection Error (RESOLVED)
|
||||
|
||||
**Error Message:**
|
||||
```
|
||||
2025-07-24 19:48:31,920 - ERROR - Failed to connect to SQLITE: 'Connection' object has no attribute 'commit'
|
||||
2025-07-24 19:48:32,740 - ERROR - Error saving to SQLITE: 'Connection' object has no attribute 'commit'
|
||||
```
|
||||
|
||||
**Root Cause:**
|
||||
This error occurred due to incompatibility between the database adapter code and newer versions of SQLAlchemy. The code was calling `conn.commit()` on a connection object that doesn't have a `commit()` method in newer SQLAlchemy versions.
|
||||
|
||||
**Solution Applied:**
|
||||
Changed from `engine.connect()` to `engine.begin()` context manager, which automatically handles transactions:
|
||||
|
||||
```python
|
||||
# OLD (problematic) code:
|
||||
with self.engine.connect() as conn:
|
||||
conn.execute(text(sql))
|
||||
conn.commit() # This fails in newer SQLAlchemy
|
||||
|
||||
# NEW (fixed) code:
|
||||
with self.engine.begin() as conn:
|
||||
conn.execute(text(sql))
|
||||
# Transaction automatically committed when context exits
|
||||
```
|
||||
|
||||
**Status:** ✅ **FIXED** - The issue has been resolved in the current version.
|
||||
|
||||
## Installation on Debian/Ubuntu
|
||||
|
||||
### System Requirements
|
||||
|
||||
```bash
|
||||
# Update package list
|
||||
sudo apt update
|
||||
|
||||
# Install Python and pip
|
||||
sudo apt install python3 python3-pip python3-venv
|
||||
|
||||
# Install system dependencies for database drivers
|
||||
sudo apt install build-essential python3-dev
|
||||
|
||||
# For MySQL support (optional)
|
||||
sudo apt install default-libmysqlclient-dev
|
||||
|
||||
# For PostgreSQL support (optional)
|
||||
sudo apt install libpq-dev
|
||||
```
|
||||
|
||||
### Python Environment Setup
|
||||
|
||||
```bash
|
||||
# Create virtual environment
|
||||
python3 -m venv water_monitor_env
|
||||
|
||||
# Activate virtual environment
|
||||
source water_monitor_env/bin/activate
|
||||
|
||||
# Install requirements
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
### Running the Monitor
|
||||
|
||||
```bash
|
||||
# Test run
|
||||
python water_scraper_v3.py --test
|
||||
|
||||
# Run with specific database
|
||||
export DB_TYPE=sqlite
|
||||
python water_scraper_v3.py
|
||||
|
||||
# Run demo
|
||||
python demo_databases.py
|
||||
```
|
||||
|
||||
## Common Linux Issues
|
||||
|
||||
### 1. Permission Errors
|
||||
|
||||
**Error:**
|
||||
```
|
||||
PermissionError: [Errno 13] Permission denied: 'water_levels.db'
|
||||
```
|
||||
|
||||
**Solution:**
|
||||
```bash
|
||||
# Check current directory permissions
|
||||
ls -la
|
||||
|
||||
# Create data directory with proper permissions
|
||||
mkdir -p data
|
||||
chmod 755 data
|
||||
|
||||
# Set database path to data directory
|
||||
export WATER_DB_PATH=data/water_levels.db
|
||||
```
|
||||
|
||||
### 2. Missing System Dependencies
|
||||
|
||||
**Error:**
|
||||
```
|
||||
ImportError: No module named '_sqlite3'
|
||||
```
|
||||
|
||||
**Solution:**
|
||||
```bash
|
||||
# Install SQLite development headers
|
||||
sudo apt install libsqlite3-dev
|
||||
|
||||
# Reinstall Python if needed
|
||||
sudo apt install python3-sqlite
|
||||
```
|
||||
|
||||
### 3. Network/Firewall Issues
|
||||
|
||||
**Error:**
|
||||
```
|
||||
requests.exceptions.ConnectionError: HTTPSConnectionPool
|
||||
```
|
||||
|
||||
**Solution:**
|
||||
```bash
|
||||
# Test network connectivity
|
||||
curl -I https://hyd-app-db.rid.go.th/hydro1h.html
|
||||
|
||||
# Check firewall rules
|
||||
sudo ufw status
|
||||
|
||||
# Allow outbound HTTPS if needed
|
||||
sudo ufw allow out 443
|
||||
```
|
||||
|
||||
### 4. Systemd Service Setup
|
||||
|
||||
Create service file `/etc/systemd/system/water-monitor.service`:
|
||||
|
||||
```ini
|
||||
[Unit]
|
||||
Description=Thailand Water Level Monitor
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=water-monitor
|
||||
Group=water-monitor
|
||||
WorkingDirectory=/opt/water_level_monitor
|
||||
Environment=PATH=/opt/water_level_monitor/venv/bin
|
||||
Environment=DB_TYPE=sqlite
|
||||
Environment=WATER_DB_PATH=/opt/water_level_monitor/data/water_levels.db
|
||||
ExecStart=/opt/water_level_monitor/venv/bin/python water_scraper_v3.py
|
||||
Restart=always
|
||||
RestartSec=60
|
||||
|
||||
# Security settings
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
ReadWritePaths=/opt/water_level_monitor/data
|
||||
ReadWritePaths=/opt/water_level_monitor/logs
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
```
|
||||
|
||||
Enable and start:
|
||||
```bash
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable water-monitor.service
|
||||
sudo systemctl start water-monitor.service
|
||||
sudo systemctl status water-monitor.service
|
||||
```
|
||||
|
||||
### 5. Log Rotation
|
||||
|
||||
Create `/etc/logrotate.d/water-monitor`:
|
||||
|
||||
```
|
||||
/opt/water_level_monitor/water_monitor.log {
|
||||
daily
|
||||
missingok
|
||||
rotate 30
|
||||
compress
|
||||
delaycompress
|
||||
notifempty
|
||||
create 644 water-monitor water-monitor
|
||||
postrotate
|
||||
systemctl reload water-monitor.service
|
||||
endscript
|
||||
}
|
||||
```
|
||||
|
||||
## Database-Specific Issues
|
||||
|
||||
### SQLite
|
||||
|
||||
**Issue:** Database locked
|
||||
```bash
|
||||
# Check for processes using the database
|
||||
sudo lsof /path/to/water_levels.db
|
||||
|
||||
# Kill processes if needed
|
||||
sudo pkill -f water_scraper_v3.py
|
||||
```
|
||||
|
||||
### VictoriaMetrics with HTTPS
|
||||
|
||||
**Configuration:**
|
||||
```bash
|
||||
export DB_TYPE=victoriametrics
|
||||
export VM_HOST=https://your-vm-server.com
|
||||
export VM_PORT=443
|
||||
```
|
||||
|
||||
**Test connection:**
|
||||
```bash
|
||||
curl -k https://your-vm-server.com/health
|
||||
```
|
||||
|
||||
## Performance Optimization
|
||||
|
||||
### 1. System Tuning
|
||||
|
||||
```bash
|
||||
# Increase file descriptor limits
|
||||
echo "* soft nofile 65536" >> /etc/security/limits.conf
|
||||
echo "* hard nofile 65536" >> /etc/security/limits.conf
|
||||
|
||||
# Optimize network settings
|
||||
echo "net.core.rmem_max = 16777216" >> /etc/sysctl.conf
|
||||
echo "net.core.wmem_max = 16777216" >> /etc/sysctl.conf
|
||||
sysctl -p
|
||||
```
|
||||
|
||||
### 2. Database Optimization
|
||||
|
||||
```bash
|
||||
# For SQLite
|
||||
export SQLITE_CACHE_SIZE=10000
|
||||
export SQLITE_SYNCHRONOUS=NORMAL
|
||||
|
||||
# Monitor database size
|
||||
du -h data/water_levels.db
|
||||
```
|
||||
|
||||
## Monitoring and Maintenance
|
||||
|
||||
### Health Check Script
|
||||
|
||||
Create `health_check.sh`:
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
LOG_FILE="/opt/water_level_monitor/water_monitor.log"
|
||||
SERVICE_NAME="water-monitor"
|
||||
|
||||
# Check if service is running
|
||||
if ! systemctl is-active --quiet $SERVICE_NAME; then
|
||||
echo "ERROR: $SERVICE_NAME is not running"
|
||||
systemctl restart $SERVICE_NAME
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check recent log entries
|
||||
RECENT_ERRORS=$(tail -n 100 $LOG_FILE | grep -c "ERROR")
|
||||
if [ $RECENT_ERRORS -gt 5 ]; then
|
||||
echo "WARNING: $RECENT_ERRORS errors found in recent logs"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "OK: Service is healthy"
|
||||
exit 0
|
||||
```
|
||||
|
||||
### Cron Job for Health Checks
|
||||
|
||||
```bash
|
||||
# Add to crontab
|
||||
*/5 * * * * /opt/water_level_monitor/health_check.sh >> /var/log/water-monitor-health.log 2>&1
|
||||
```
|
||||
|
||||
## Getting Help
|
||||
|
||||
### Debug Information
|
||||
|
||||
```bash
|
||||
# System information
|
||||
uname -a
|
||||
python3 --version
|
||||
pip list | grep -E "(sqlalchemy|requests|influxdb)"
|
||||
|
||||
# Service logs
|
||||
journalctl -u water-monitor.service -f
|
||||
|
||||
# Application logs
|
||||
tail -f water_monitor.log
|
||||
|
||||
# Database information
|
||||
sqlite3 water_levels.db ".schema"
|
||||
sqlite3 water_levels.db "SELECT COUNT(*) FROM water_measurements;"
|
||||
```
|
||||
|
||||
### Common Commands
|
||||
|
||||
```bash
|
||||
# Restart service
|
||||
sudo systemctl restart water-monitor.service
|
||||
|
||||
# View logs
|
||||
sudo journalctl -u water-monitor.service --since "1 hour ago"
|
||||
|
||||
# Test configuration
|
||||
python config.py
|
||||
|
||||
# Test database connection
|
||||
python demo_databases.py
|
||||
|
||||
# Manual data fetch
|
||||
python water_scraper_v3.py --test
|
||||
```
|
||||
|
||||
This troubleshooting guide should help resolve most common issues encountered when running the Thailand Water Monitor on Debian and other Linux distributions.
|
||||
+46
-48
@@ -62,8 +62,8 @@ P.5 before 2022-01-01; the rest are handled by the per-row coverage gates.
|
||||
|
||||
**How `src/ml/data.py` loads it.** `resolve_db_url()` picks a connection string in
|
||||
priority order: an explicit `--db-url` argument, then the `FLOOD_ML_DB_URL`
|
||||
environment variable, then `Config.get_database_config()` when `DB_TYPE` is
|
||||
`postgresql`, else `None`. `load_measurements()` then tries three tiers:
|
||||
environment variable, then the `POSTGRES_*` settings (`src/config.py`, the same
|
||||
`.env` the service reads), else `None`. `load_measurements()` then tries three tiers:
|
||||
|
||||
1. **PostgreSQL** (`_fetch_from_db`) — the primary path. NULL discharge stays
|
||||
NULL, which matters because the models must learn from the real missingness
|
||||
@@ -614,32 +614,28 @@ fix, the served features match what this section evaluates.
|
||||
|
||||
## 6. Deployment
|
||||
|
||||
### API
|
||||
### Serving
|
||||
|
||||
With `FORECAST_IN_COLLECTOR=true` the forecast is computed by `ping-collector`
|
||||
(`rust/forecast`, a parity-tested port of this serving path) from
|
||||
The forecast is computed by `ping-collector` (`rust/forecast`, a parity-tested
|
||||
port of `src/ml/predict.py`, which stays as the reference implementation) from
|
||||
`models/flood_<station>.json.gz`, which `train_all()` exports next to each
|
||||
joblib bundle; `/forecast` serves its `forecast_current` row and computes
|
||||
in-process only when that is older than `FORECAST_MAX_AGE_SECONDS` (2 h). See
|
||||
`rust/README.md`. The rest of this section describes the in-process path.
|
||||
joblib bundle. It runs after every collection cycle that brought newer
|
||||
readings and at least hourly, stores the rows in `forecast_history` (section 9's
|
||||
skill tracking) and `forecast_current`, and `GET /forecast` serves the stored
|
||||
row: one JSON row per station × horizon with the fields listed in section 1.
|
||||
A stored forecast older than `FORECAST_MAX_AGE_SECONDS` (2 h, i.e. the service
|
||||
has stopped forecasting) is still served, marked `X-Data-Stale: true`. See
|
||||
`rust/README.md`.
|
||||
|
||||
`GET /forecast` (`src/web_api.py`) returns one JSON row per station × horizon with
|
||||
the fields listed in section 1. Results are cached in-process for
|
||||
`FORECAST_TTL = 900` seconds (15 minutes), which matches the data cadence — the
|
||||
underlying readings do not update faster than hourly. Inference runs in a thread
|
||||
via `asyncio.to_thread` so it never blocks the event loop.
|
||||
A station without a model bundle, or whose bundle needs a feature the Rust side
|
||||
cannot build, gets the heuristic, as in Python. So **before the first training
|
||||
run `/forecast` returns 200 with an all-heuristic payload**, provided there is
|
||||
recent gauge data; you get an empty `200 []` only when there is no recent data
|
||||
at all. Judge deployment state by the `source` field, not the status code.
|
||||
|
||||
Failure modes: **503** if the `src.ml` package cannot be imported (missing
|
||||
scikit-learn, say), **502** on any other exception.
|
||||
|
||||
One behaviour worth knowing, because the code comments suggest otherwise: the
|
||||
endpoint's `FileNotFoundError` ("No trained flood models found") and `RuntimeError`
|
||||
handlers are unreachable — nothing in `src/ml/` raises either, and
|
||||
`predict._forecast_station` checks `bundle_path.exists()` and falls back to the
|
||||
heuristic instead. So **before the first training run `/forecast` returns 200 with
|
||||
an all-heuristic payload**, not a 503, provided there is recent gauge data; you
|
||||
get an empty `200 []` only when there is no recent data at all. Judge deployment
|
||||
state by the `source` field, not the status code.
|
||||
The Python serving path (`predict.get_latest_forecasts`) is not called by the
|
||||
service any more; `scripts/forecast_parity.py` runs both on the live database
|
||||
and `scripts/install.sh` does that on every install.
|
||||
|
||||
### Dashboard
|
||||
|
||||
@@ -661,13 +657,14 @@ The server already has the PostgreSQL connection configured, so no host override
|
||||
is needed:
|
||||
|
||||
```bash
|
||||
cd /path/to/Northern-Thailand-Ping-River-Monitor
|
||||
python scripts/train_flood_model.py --stations all
|
||||
cd /opt/thailand-water-monitor
|
||||
uv run python scripts/train_flood_model.py --stations all
|
||||
```
|
||||
|
||||
`resolve_db_url()` picks up `Config.get_database_config()` automatically when
|
||||
`DB_TYPE=postgresql`. The run writes fifteen `models/flood_{station}.joblib`
|
||||
bundles plus `models/metrics.json`.
|
||||
`resolve_db_url()` picks up the `POSTGRES_*` settings from `.env`. The run writes
|
||||
fifteen `models/flood_{station}.joblib` bundles, their `.json.gz` exports for the
|
||||
service, and `models/metrics.json`. The monthly timer does the same through
|
||||
`scripts/retrain.sh` with a staged promote (section 7).
|
||||
|
||||
### Artifacts and dependencies
|
||||
|
||||
@@ -733,7 +730,10 @@ gap between fetch and first model could not be reconstructed from the surviving
|
||||
artifacts. Do not attribute it to per-station training cost. Either way the
|
||||
conclusion holds: **retraining is minutes, not tens of minutes.**
|
||||
|
||||
**Inference** (15 bundles, 16 stations × 3 horizons = 48 rows):
|
||||
**Inference in Python** (15 bundles, 16 stations × 3 horizons = 48 rows; the
|
||||
retired in-process serving path, measured before the Rust forecaster took
|
||||
over: there the whole run takes milliseconds, 0.25 ms per station against
|
||||
158 ms, and the 15 exported models need about 70 MB resident):
|
||||
|
||||
| Measurement | Value |
|
||||
|---|---|
|
||||
@@ -753,9 +753,9 @@ one-time lazy `sklearn.ensemble` import — the remaining fourteen average 159 m
|
||||
Of the ~640 ms warm compute, model prediction is ~525 ms, the hourly grid 47 ms,
|
||||
and feature building 71 ms across all sixteen stations.
|
||||
|
||||
**Live-endpoint measurements** (a second, independent benchmark run against a real
|
||||
uvicorn instance of the app, same day, same workstation, RSS summed over the
|
||||
process tree):
|
||||
**Live-endpoint measurements of the retired Python API** (a second, independent
|
||||
benchmark run against a real uvicorn instance of the app, same day, same
|
||||
workstation, RSS summed over the process tree):
|
||||
|
||||
| Measurement | Value |
|
||||
|---|---|
|
||||
@@ -775,14 +775,11 @@ Training shows the opposite (2.3× slower single-threaded), so set the variable
|
||||
per process, never globally.
|
||||
|
||||
**Server sizing, in plain terms:** this is a small workload and almost any server
|
||||
runs it. **RAM is the binding constraint, not CPU.** Budget about **1 GB for the
|
||||
API process** so the ~335 MB steady state has headroom on top of the rest of the
|
||||
app; training peaks at only ~210–315 MB and can share the same box. No GPU
|
||||
anywhere. Pin thread counts per process — `OMP_NUM_THREADS=1` in the serving
|
||||
unit, `OMP_NUM_THREADS=4` for retraining so it cannot monopolise every core while
|
||||
the API is serving. And since a cold call costs seconds against a 2.4 ms cache
|
||||
hit, consider warming `/forecast` once at startup rather than letting a user
|
||||
absorb it.
|
||||
runs it. The service (`ping-collector`, Rust) stays under 100 MB resident with
|
||||
every model loaded (its unit caps it at 512 MB); the monthly retrain peaks at
|
||||
~210–315 MB and shares the same box. No GPU anywhere. The retrain unit sets
|
||||
`OMP_NUM_THREADS=4` so it cannot monopolise every core while the site is
|
||||
serving.
|
||||
|
||||
## 7. Retraining policy
|
||||
|
||||
@@ -821,15 +818,15 @@ fires `water-monitor-retrain.service` on the 1st of every month at 03:30 server
|
||||
time (`Persistent=true`, so a missed run catches up at boot). The unit runs
|
||||
`scripts/retrain.sh` as the service user with `OMP_NUM_THREADS=4`, `Nice=15`:
|
||||
|
||||
1. trains all stations into `models/.staging/` (the API keeps serving the old
|
||||
bundles throughout);
|
||||
1. trains all stations into `models/.staging/` (the service keeps serving the
|
||||
old bundles throughout);
|
||||
2. refuses to promote unless `metrics.json` reports a `hgb-v3+` version and at
|
||||
least 14 trained stations (exit 3, staging discarded, old models untouched);
|
||||
3. renames the new bundles into `models/`, moving the previous generation to
|
||||
`models/.previous/` for rollback.
|
||||
|
||||
No API restart: `predict.py` reloads bundles by mtime on the next hourly
|
||||
precompute. `systemctl list-timers water-monitor-retrain.timer` shows the next
|
||||
No restart: ping-collector reloads changed `.json.gz` exports by mtime on its
|
||||
next forecast run. `systemctl list-timers water-monitor-retrain.timer` shows the next
|
||||
run; `sudo systemctl start water-monitor-retrain.service` runs it now (after a
|
||||
flood, say); `journalctl -u water-monitor-retrain` has the log. The installer
|
||||
(`scripts/install.sh`) enables the timer.
|
||||
@@ -912,9 +909,10 @@ non-model-backed, and all of them are visible in the payload:
|
||||
horizon's head is in the bundle's `skipped_heads`, almost always because the
|
||||
station had fewer than 30 positive examples for that threshold.
|
||||
- A whole station flipping to heuristic after a code change — the feature-name
|
||||
check in `_model_forecast` caught a mismatch between the live feature builder
|
||||
and the stored `feature_names`. The fix is to retrain; the log line names the
|
||||
missing columns.
|
||||
check caught a mismatch between the feature builder and the stored
|
||||
`feature_names` (in Python `_model_forecast`; in Rust a bundle needing a
|
||||
feature `rust/forecast` cannot build). The fix is to retrain or to port the
|
||||
feature; the log line names the missing columns.
|
||||
|
||||
A live example from the 2026-08-10 cache: of 48 forecast rows, 42 came from
|
||||
models and 6 were heuristic — three for P.4A, which has no bundle by design, and
|
||||
|
||||
@@ -1,275 +0,0 @@
|
||||
# Gap Filling and Data Integrity Guide
|
||||
|
||||
This guide explains the enhanced gap-filling functionality that addresses data gaps and missing timestamps in the Thailand Water Monitor.
|
||||
|
||||
## ✅ **Issues Resolved**
|
||||
|
||||
### **1. Data Gaps Problem**
|
||||
- **Before**: Tool only fetched current day data, leaving gaps in historical records
|
||||
- **After**: Automatically detects and fills missing timestamps for the last 7 days
|
||||
|
||||
### **2. Missing Midnight Timestamps**
|
||||
- **Before**: Jump from 23:00 to 01:00 (missing 00:00 midnight data)
|
||||
- **After**: Specifically checks for and fills midnight hour gaps
|
||||
|
||||
### **3. Changed Values**
|
||||
- **Before**: No mechanism to update existing data if values changed on the server
|
||||
- **After**: Compares existing data with fresh API data and updates changed values
|
||||
|
||||
## 🔧 **New Features**
|
||||
|
||||
### **Command Line Interface**
|
||||
```bash
|
||||
# Check for missing data gaps
|
||||
python water_scraper_v3.py --check-gaps [days]
|
||||
|
||||
# Fill missing data gaps
|
||||
python water_scraper_v3.py --fill-gaps [days]
|
||||
|
||||
# Update existing data with latest values
|
||||
python water_scraper_v3.py --update-data [days]
|
||||
|
||||
# Run single test cycle
|
||||
python water_scraper_v3.py --test
|
||||
|
||||
# Show help
|
||||
python water_scraper_v3.py --help
|
||||
```
|
||||
|
||||
### **Automatic Gap Detection**
|
||||
The system now automatically:
|
||||
- Generates expected hourly timestamps for the specified time range
|
||||
- Compares with existing database records
|
||||
- Identifies missing timestamps
|
||||
- Groups missing data by date for efficient API calls
|
||||
|
||||
### **Intelligent Gap Filling**
|
||||
- **Historical Data Fetching**: Retrieves data for specific dates to fill gaps
|
||||
- **Selective Insertion**: Only inserts data for actually missing timestamps
|
||||
- **API Rate Limiting**: Includes delays between API calls to be respectful
|
||||
- **Error Handling**: Continues processing even if some dates fail
|
||||
|
||||
### **Data Update Mechanism**
|
||||
- **Change Detection**: Compares water levels, discharge rates, and percentages
|
||||
- **Precision Checking**: Uses appropriate thresholds (0.001m for water level, 0.1 cms for discharge)
|
||||
- **Selective Updates**: Only updates records where values have actually changed
|
||||
|
||||
## 📊 **Test Results**
|
||||
|
||||
### **Before Enhancement**
|
||||
```
|
||||
Found 22 missing timestamps in the last 2 days:
|
||||
2025-07-23: Missing hours [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]
|
||||
2025-07-24: Missing hours [0, 20, 21, 22, 23]
|
||||
2025-07-25: Missing hours [0, 9]
|
||||
```
|
||||
|
||||
### **After Gap Filling**
|
||||
```
|
||||
Gap filling completed. Filled 96 missing data points
|
||||
|
||||
Remaining gaps:
|
||||
2025-07-24: Missing hours [10]
|
||||
2025-07-25: Missing hours [0, 10]
|
||||
```
|
||||
|
||||
**Improvement**: Reduced from 22 missing timestamps to 3 (86% improvement)
|
||||
|
||||
## 🚀 **Enhanced Scraping Cycle**
|
||||
|
||||
The regular scraping cycle now includes three phases:
|
||||
|
||||
### **Phase 1: Current Data Collection**
|
||||
```python
|
||||
# Fetch and save current data
|
||||
water_data = self.fetch_water_data()
|
||||
success = self.save_to_database(water_data)
|
||||
```
|
||||
|
||||
### **Phase 2: Gap Filling (Last 7 Days)**
|
||||
```python
|
||||
# Check for and fill missing data
|
||||
filled_count = self.fill_data_gaps(days_back=7)
|
||||
```
|
||||
|
||||
### **Phase 3: Data Updates (Last 2 Days)**
|
||||
```python
|
||||
# Update existing data with latest values
|
||||
updated_count = self.update_existing_data(days_back=2)
|
||||
```
|
||||
|
||||
## 🔧 **Technical Improvements**
|
||||
|
||||
### **Database Connection Handling**
|
||||
- **SQLite Optimization**: Added timeout and thread safety parameters
|
||||
- **Retry Logic**: Exponential backoff for database lock errors
|
||||
- **Transaction Management**: Proper use of `engine.begin()` for automatic commits
|
||||
|
||||
### **Error Recovery**
|
||||
```python
|
||||
# Retry logic with exponential backoff
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
success = self.db_adapter.save_measurements(water_data)
|
||||
if success:
|
||||
return True
|
||||
except Exception as e:
|
||||
if "database is locked" in str(e).lower():
|
||||
time.sleep(2 ** attempt) # 1s, 2s, 4s delays
|
||||
continue
|
||||
```
|
||||
|
||||
### **Memory Efficiency**
|
||||
- **Selective Data Processing**: Only processes data for missing timestamps
|
||||
- **Batch Processing**: Groups operations by date to minimize API calls
|
||||
- **Resource Management**: Proper cleanup and connection handling
|
||||
|
||||
## 📋 **Usage Examples**
|
||||
|
||||
### **Daily Maintenance**
|
||||
```bash
|
||||
# Check for gaps in the last week
|
||||
python water_scraper_v3.py --check-gaps 7
|
||||
|
||||
# Fill any found gaps
|
||||
python water_scraper_v3.py --fill-gaps 7
|
||||
|
||||
# Update recent data for accuracy
|
||||
python water_scraper_v3.py --update-data 2
|
||||
```
|
||||
|
||||
### **Historical Data Recovery**
|
||||
```bash
|
||||
# Check for gaps in the last month
|
||||
python water_scraper_v3.py --check-gaps 30
|
||||
|
||||
# Fill gaps for the last month (be patient, this takes time)
|
||||
python water_scraper_v3.py --fill-gaps 30
|
||||
```
|
||||
|
||||
### **Production Monitoring**
|
||||
```bash
|
||||
# Quick test to ensure system is working
|
||||
python water_scraper_v3.py --test
|
||||
|
||||
# Check for recent gaps
|
||||
python water_scraper_v3.py --check-gaps 1
|
||||
```
|
||||
|
||||
## 🔍 **Monitoring and Alerts**
|
||||
|
||||
### **Gap Detection Output**
|
||||
```
|
||||
Found 22 missing timestamps:
|
||||
2025-07-23: Missing hours [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]
|
||||
2025-07-24: Missing hours [0, 20, 21, 22, 23]
|
||||
2025-07-25: Missing hours [0, 9]
|
||||
```
|
||||
|
||||
### **Gap Filling Progress**
|
||||
```
|
||||
Fetching data for 2025-07-24 to fill 5 missing timestamps
|
||||
Successfully fetched 368 data points from API for 2025-07-24
|
||||
Filled 80 data points for 2025-07-24
|
||||
Gap filling completed. Filled 96 missing data points
|
||||
```
|
||||
|
||||
### **Update Detection**
|
||||
```
|
||||
Checking for updates on 2025-07-24
|
||||
Update needed for P.1 at 2025-07-24 15:00:00
|
||||
Updated 5 measurements for 2025-07-24
|
||||
Data update completed. Updated 5 measurements
|
||||
```
|
||||
|
||||
## ⚙️ **Configuration Options**
|
||||
|
||||
### **Environment Variables**
|
||||
```bash
|
||||
# Database configuration
|
||||
export DB_TYPE=sqlite
|
||||
export WATER_DB_PATH=water_monitoring.db
|
||||
|
||||
# Gap filling settings (can be added to config.py)
|
||||
export GAP_FILL_DAYS=7 # Days to check for gaps
|
||||
export UPDATE_DAYS=2 # Days to check for updates
|
||||
export API_DELAY=1 # Seconds between API calls
|
||||
export MAX_RETRIES=3 # Database retry attempts
|
||||
```
|
||||
|
||||
### **Customizable Parameters**
|
||||
- **Gap Check Period**: Default 7 days, configurable via command line
|
||||
- **Update Period**: Default 2 days, configurable via command line
|
||||
- **API Rate Limiting**: 1-second delay between calls (configurable)
|
||||
- **Retry Logic**: 3 attempts with exponential backoff (configurable)
|
||||
|
||||
## 🛠️ **Troubleshooting**
|
||||
|
||||
### **Common Issues**
|
||||
|
||||
#### **Database Locked Errors**
|
||||
```
|
||||
ERROR - Error saving to SQLITE: database is locked
|
||||
```
|
||||
**Solution**: The retry logic now handles this automatically with exponential backoff.
|
||||
|
||||
#### **API Rate Limiting**
|
||||
```
|
||||
WARNING - Too many requests to API
|
||||
```
|
||||
**Solution**: Increase delay between API calls or reduce the number of days processed at once.
|
||||
|
||||
#### **Missing Data Still Present**
|
||||
```
|
||||
Found X missing timestamps after gap filling
|
||||
```
|
||||
**Possible Causes**:
|
||||
- Data not available on the Thai government server for those timestamps
|
||||
- Network issues during API calls
|
||||
- API returned empty data for those specific times
|
||||
|
||||
### **Debug Commands**
|
||||
```bash
|
||||
# Enable debug logging
|
||||
export LOG_LEVEL=DEBUG
|
||||
python water_scraper_v3.py --check-gaps 1
|
||||
|
||||
# Test specific date range
|
||||
python water_scraper_v3.py --fill-gaps 1
|
||||
|
||||
# Check database directly
|
||||
sqlite3 water_monitoring.db "SELECT COUNT(*) FROM water_measurements;"
|
||||
sqlite3 water_monitoring.db "SELECT timestamp, COUNT(*) FROM water_measurements GROUP BY timestamp ORDER BY timestamp DESC LIMIT 10;"
|
||||
```
|
||||
|
||||
## 📈 **Performance Metrics**
|
||||
|
||||
### **Gap Filling Efficiency**
|
||||
- **API Calls**: Grouped by date to minimize requests
|
||||
- **Processing Speed**: ~100-400 data points per API call
|
||||
- **Success Rate**: 86% gap reduction in test case
|
||||
- **Resource Usage**: Minimal memory footprint with selective processing
|
||||
|
||||
### **Database Performance**
|
||||
- **SQLite Optimization**: Connection pooling and timeout handling
|
||||
- **Transaction Efficiency**: Batch inserts with proper transaction management
|
||||
- **Retry Success**: Automatic recovery from temporary lock conditions
|
||||
|
||||
## 🎯 **Best Practices**
|
||||
|
||||
### **Regular Maintenance**
|
||||
1. **Daily**: Run `--check-gaps 1` to monitor recent data quality
|
||||
2. **Weekly**: Run `--fill-gaps 7` to catch any missed data
|
||||
3. **Monthly**: Run `--update-data 7` to ensure data accuracy
|
||||
|
||||
### **Production Deployment**
|
||||
1. **Automated Scheduling**: Use cron or systemd timers for regular gap checks
|
||||
2. **Monitoring**: Set up alerts for excessive missing data
|
||||
3. **Backup**: Regular database backups before major gap-filling operations
|
||||
|
||||
### **Data Quality Assurance**
|
||||
1. **Validation**: Check for reasonable value ranges after gap filling
|
||||
2. **Comparison**: Compare filled data with nearby timestamps for consistency
|
||||
3. **Documentation**: Log all gap-filling activities for audit trails
|
||||
|
||||
This enhanced gap-filling system ensures comprehensive and accurate water level monitoring with minimal data loss and automatic recovery capabilities.
|
||||
@@ -1,300 +0,0 @@
|
||||
# 🔄 Gitea Actions Workflows - Northern Thailand Ping River Monitor
|
||||
|
||||
## 📋 Overview
|
||||
|
||||
This document describes the Gitea Actions workflows configured for the Northern Thailand Ping River Monitor project. These workflows provide comprehensive CI/CD, security scanning, and documentation generation.
|
||||
|
||||
## 🚀 Available Workflows
|
||||
|
||||
### 1. **CI/CD Pipeline** (`.gitea/workflows/ci.yml`)
|
||||
|
||||
**Triggers:**
|
||||
- Push to `main` or `develop` branches
|
||||
- Pull requests to `main`
|
||||
- Daily scheduled runs at 2 AM UTC
|
||||
|
||||
**Jobs:**
|
||||
- **Test Suite**: Multi-version Python testing (3.9-3.12)
|
||||
- **Code Quality**: Linting, formatting, and type checking
|
||||
- **Build**: Docker image creation and testing
|
||||
- **Integration Test**: Testing with VictoriaMetrics service
|
||||
- **Deploy Staging**: Automatic deployment to staging (develop branch)
|
||||
- **Deploy Production**: Manual deployment to production (main branch)
|
||||
- **Performance Test**: Load testing after production deployment
|
||||
|
||||
**Key Features:**
|
||||
- ✅ Multi-Python version testing
|
||||
- ✅ Docker multi-architecture builds (amd64, arm64)
|
||||
- ✅ Service integration testing
|
||||
- ✅ Automatic staging deployment
|
||||
- ✅ Manual production approval
|
||||
- ✅ Performance validation
|
||||
|
||||
### 2. **Security & Dependency Updates** (`.gitea/workflows/security.yml`)
|
||||
|
||||
**Triggers:**
|
||||
- Daily scheduled runs at 3 AM UTC
|
||||
- Manual dispatch
|
||||
- Changes to requirements files or Dockerfile
|
||||
|
||||
**Jobs:**
|
||||
- **Dependency Scan**: Safety, Bandit, Semgrep security scans
|
||||
- **Docker Security**: Trivy vulnerability scanning
|
||||
- **License Check**: License compliance verification
|
||||
- **Dependency Updates**: Automated update detection
|
||||
- **Code Quality**: Complexity and maintainability analysis
|
||||
|
||||
**Key Features:**
|
||||
- 🔒 Daily security scans
|
||||
- 📦 Dependency vulnerability detection
|
||||
- 📄 License compliance checking
|
||||
- 🔄 Automated update notifications
|
||||
- 📊 Code quality metrics
|
||||
|
||||
### 3. **Release Workflow** (`.gitea/workflows/release.yml`)
|
||||
|
||||
**Triggers:**
|
||||
- Git tags matching `v*.*.*` pattern
|
||||
- Manual dispatch with version input
|
||||
|
||||
**Jobs:**
|
||||
- **Create Release**: Automated release creation with changelog
|
||||
- **Test Release**: Comprehensive testing across Python versions
|
||||
- **Build Release**: Multi-architecture Docker images with proper tags
|
||||
- **Security Scan**: Trivy security scanning of release images
|
||||
- **Deploy Release**: Production deployment with health checks
|
||||
- **Validate Release**: Post-deployment validation and testing
|
||||
|
||||
**Key Features:**
|
||||
- 🏷️ Automated release creation
|
||||
- 📝 Changelog generation
|
||||
- 🐳 Multi-architecture Docker builds
|
||||
- 🔒 Security scanning
|
||||
- ✅ Comprehensive validation
|
||||
|
||||
### 4. **Documentation** (`.gitea/workflows/docs.yml`)
|
||||
|
||||
**Triggers:**
|
||||
- Changes to documentation files
|
||||
- Changes to Python source files
|
||||
- Manual dispatch
|
||||
|
||||
**Jobs:**
|
||||
- **Validate Docs**: Link checking and structure validation
|
||||
- **Generate API Docs**: OpenAPI specification generation
|
||||
- **Build Sphinx Docs**: Comprehensive API documentation
|
||||
- **Documentation Summary**: Build status and artifact summary
|
||||
|
||||
**Key Features:**
|
||||
- 📚 Automated API documentation
|
||||
- 🔗 Link validation
|
||||
- 📖 Sphinx documentation generation
|
||||
- ✅ Documentation completeness checking
|
||||
|
||||
## 🔧 Workflow Configuration
|
||||
|
||||
### **Required Secrets**
|
||||
|
||||
Configure these secrets in your Gitea repository settings:
|
||||
|
||||
```bash
|
||||
GITEA_TOKEN # Gitea access token for container registry
|
||||
GITHUB_TOKEN # GitHub personal access token for better rate limits
|
||||
SLACK_WEBHOOK_URL # Optional: Slack notifications
|
||||
STAGING_WEBHOOK_URL # Optional: Staging deployment webhook
|
||||
PRODUCTION_WEBHOOK_URL # Optional: Production deployment webhook
|
||||
```
|
||||
|
||||
**Setting up GitHub Token:**
|
||||
1. Go to GitHub → Settings → Developer settings → Personal access tokens
|
||||
2. Generate a new token with `public_repo` scope (for public repos) or `repo` scope (for private repos)
|
||||
3. Copy the token
|
||||
4. In your Gitea repository → Settings → Secrets → Add `GITHUB_TOKEN` with the copied value
|
||||
|
||||
### **Environment Variables**
|
||||
|
||||
Key environment variables used across workflows:
|
||||
|
||||
```yaml
|
||||
PYTHON_VERSION: '3.11' # Default Python version
|
||||
REGISTRY: git.b4l.co.th # Container registry
|
||||
IMAGE_NAME: grabowski/northern-thailand-ping-river-monitor
|
||||
```
|
||||
|
||||
## 📊 Workflow Status
|
||||
|
||||
### **CI/CD Pipeline Status**
|
||||
- **Test Coverage**: Multi-version Python testing
|
||||
- **Code Quality**: Automated linting and formatting
|
||||
- **Security**: Integrated security scanning
|
||||
- **Deployment**: Automated staging, manual production
|
||||
|
||||
### **Security Monitoring**
|
||||
- **Daily Scans**: Automated vulnerability detection
|
||||
- **Dependency Updates**: Proactive update notifications
|
||||
- **License Compliance**: Automated license checking
|
||||
- **Code Quality**: Continuous quality monitoring
|
||||
|
||||
### **Release Management**
|
||||
- **Automated Releases**: Tag-based release creation
|
||||
- **Multi-Architecture**: Support for amd64 and arm64
|
||||
- **Security Validation**: Pre-deployment security checks
|
||||
- **Health Monitoring**: Post-deployment validation
|
||||
|
||||
## 🚀 Usage Examples
|
||||
|
||||
### **Triggering Workflows**
|
||||
|
||||
**Manual CI/CD Run:**
|
||||
```bash
|
||||
# Push to trigger CI/CD
|
||||
git push origin main
|
||||
|
||||
# Create pull request to trigger testing
|
||||
git checkout -b feature/new-feature
|
||||
git push origin feature/new-feature
|
||||
# Create PR in Gitea UI
|
||||
```
|
||||
|
||||
**Manual Security Scan:**
|
||||
```bash
|
||||
# Trigger via Gitea Actions UI
|
||||
# Go to Actions → Security & Dependency Updates → Run workflow
|
||||
```
|
||||
|
||||
**Creating a Release:**
|
||||
```bash
|
||||
# Create and push a tag
|
||||
git tag v3.1.1
|
||||
git push origin v3.1.1
|
||||
|
||||
# Or use manual dispatch in Gitea Actions UI
|
||||
```
|
||||
|
||||
### **Monitoring Workflow Results**
|
||||
|
||||
**Check Workflow Status:**
|
||||
1. Navigate to your repository in Gitea
|
||||
2. Click on "Actions" tab
|
||||
3. View workflow runs and their status
|
||||
|
||||
**Download Artifacts:**
|
||||
1. Click on a completed workflow run
|
||||
2. Scroll to "Artifacts" section
|
||||
3. Download reports and logs
|
||||
|
||||
**View Security Reports:**
|
||||
1. Go to Security workflow runs
|
||||
2. Download security-reports artifacts
|
||||
3. Review JSON reports for vulnerabilities
|
||||
|
||||
## 🔍 Troubleshooting
|
||||
|
||||
### **Common Issues**
|
||||
|
||||
**Workflow Fails on Dependencies:**
|
||||
```bash
|
||||
# Check requirements.txt for version conflicts
|
||||
pip-compile requirements.in
|
||||
```
|
||||
|
||||
**Docker Build Fails:**
|
||||
```bash
|
||||
# Test Docker build locally
|
||||
make docker-build
|
||||
docker run --rm ping-river-monitor python run.py --test
|
||||
```
|
||||
|
||||
**Security Scan Failures:**
|
||||
```bash
|
||||
# Run security scans locally
|
||||
safety check -r requirements.txt
|
||||
bandit -r src/
|
||||
```
|
||||
|
||||
**Test Failures:**
|
||||
```bash
|
||||
# Run tests locally
|
||||
make test
|
||||
python tests/test_integration.py
|
||||
```
|
||||
|
||||
### **Debugging Workflows**
|
||||
|
||||
**Enable Debug Logging:**
|
||||
Add to workflow file:
|
||||
```yaml
|
||||
env:
|
||||
ACTIONS_STEP_DEBUG: true
|
||||
ACTIONS_RUNNER_DEBUG: true
|
||||
```
|
||||
|
||||
**Check Workflow Logs:**
|
||||
1. Go to failed workflow run
|
||||
2. Click on failed job
|
||||
3. Expand failed step to see detailed logs
|
||||
|
||||
**Validate Workflow Syntax:**
|
||||
```bash
|
||||
# Validate YAML syntax
|
||||
make validate-workflows
|
||||
```
|
||||
|
||||
## 📈 Performance Optimization
|
||||
|
||||
### **Caching Strategy**
|
||||
- **Pip Cache**: Cached across workflow runs
|
||||
- **Docker Layer Cache**: GitHub Actions cache for faster builds
|
||||
- **Dependency Cache**: Cached based on requirements.txt hash
|
||||
|
||||
### **Parallel Execution**
|
||||
- **Matrix Builds**: Multiple Python versions tested in parallel
|
||||
- **Independent Jobs**: Security scans run independently of tests
|
||||
- **Conditional Execution**: Jobs skip when not needed
|
||||
|
||||
### **Resource Management**
|
||||
- **Timeout Settings**: Prevent hanging workflows
|
||||
- **Resource Limits**: Appropriate runner sizing
|
||||
- **Artifact Cleanup**: Automatic cleanup of old artifacts
|
||||
|
||||
## 🔒 Security Best Practices
|
||||
|
||||
### **Secret Management**
|
||||
- Use Gitea repository secrets for sensitive data
|
||||
- Never commit secrets to repository
|
||||
- Rotate secrets regularly
|
||||
- Use least-privilege access tokens
|
||||
|
||||
### **Container Security**
|
||||
- Multi-stage Docker builds for smaller images
|
||||
- Non-root user in containers
|
||||
- Regular base image updates
|
||||
- Vulnerability scanning before deployment
|
||||
|
||||
### **Code Security**
|
||||
- Automated security scanning in CI/CD
|
||||
- Dependency vulnerability monitoring
|
||||
- License compliance checking
|
||||
- Code quality enforcement
|
||||
|
||||
## 📚 Additional Resources
|
||||
|
||||
### **Gitea Actions Documentation**
|
||||
- [Gitea Actions Overview](https://docs.gitea.io/en-us/usage/actions/)
|
||||
- [Workflow Syntax](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions)
|
||||
- [Available Actions](https://github.com/marketplace?type=actions)
|
||||
|
||||
### **Project-Specific Resources**
|
||||
- [Contributing Guide](../CONTRIBUTING.md)
|
||||
|
||||
### **Monitoring and Alerts**
|
||||
- Workflow status badges in README
|
||||
- Email notifications for failures
|
||||
- Slack/Discord integration for team updates
|
||||
- Grafana dashboards for deployment metrics
|
||||
|
||||
---
|
||||
|
||||
**Workflow Version**: v3.1.3
|
||||
**Last Updated**: 2025-08-12
|
||||
**Maintained By**: Ping River Monitor Team
|
||||
@@ -32,13 +32,9 @@ curl -X POST https://matrix.org/_matrix/client/v3/login \
|
||||
## Step 4: Configure .env File
|
||||
Add these to your `.env` file:
|
||||
```bash
|
||||
# Matrix Alerting Configuration
|
||||
MATRIX_HOMESERVER=https://matrix.org
|
||||
MATRIX_ACCESS_TOKEN=syt_your_access_token_here
|
||||
MATRIX_ROOM_ID=!your_room_id:matrix.org
|
||||
|
||||
# Grafana Integration (optional)
|
||||
GRAFANA_URL=http://localhost:3000
|
||||
```
|
||||
|
||||
## Step 5: Restart and check
|
||||
@@ -71,8 +67,8 @@ so a Matrix outage is retried on Matrix only and does not re-send ntfy.
|
||||
|
||||
The old `run.py --alert-check` cron job (every 15 minutes, repeating an alert
|
||||
for as long as a level stayed high) is replaced by this;
|
||||
`scripts/install_collector.sh` comments it out of the crontab once
|
||||
ping-collector sends Matrix, keeping a backup next to the app.
|
||||
`scripts/install.sh` comments it out of the crontab if it is still there,
|
||||
keeping a backup next to the app.
|
||||
|
||||
## Troubleshooting
|
||||
- **HTTP 401/403** in `journalctl -u ping-collector`: the access token is invalid
|
||||
|
||||
+33
-25
@@ -36,12 +36,11 @@ the public dashboard.
|
||||
|
||||
## How it runs
|
||||
|
||||
Since phase 2 of the Rust rewrite the notifier runs in `ping-collector`
|
||||
(`rust/collector/src/notify.rs`) when `FORECAST_IN_COLLECTOR=true`, after each
|
||||
forecast run. It is a port of `src/notify.py` with the same topics, texts and
|
||||
state keys (checked word for word against Python on 751 generated messages),
|
||||
so switching between the two sends nothing twice. One difference: the Rust
|
||||
notifier also uses ThaiWater (HII) telemetry. For the stations whose HII twin
|
||||
The notifier runs in `ping-collector` (`rust/collector/src/notify.rs`) when
|
||||
`FORECAST_IN_COLLECTOR=true`, after each forecast run. It is a port of the
|
||||
Python notifier it replaced in 2026-10, with the same topics, texts and state
|
||||
keys (checked word for word on 751 generated messages), so the switch sent
|
||||
nothing twice. One difference: it also uses ThaiWater (HII) telemetry. For the stations whose HII twin
|
||||
is the same sensor (P.1, P.103, P.20, P.4A, P.67, P.75, P.82, P.84, P.92, and
|
||||
P.81 bias-corrected), a reading newer than RID's drives the level alerts; the
|
||||
message then says "ThaiWater telemetry; the RID report is not in yet". RID's
|
||||
@@ -52,12 +51,12 @@ once after `RID_STALE_ALERT_HOURS` (18, 0 = off). With `MATRIX_*` set, the
|
||||
same transitions also go to a Matrix room, each once (per station, outlook,
|
||||
feed; not the per-topic summaries), with their own `matrix:` state keys.
|
||||
|
||||
With `FORECAST_IN_COLLECTOR=false`, `src/notify.py` is called once per collection cycle inside the API process
|
||||
(leader only), right after the forecast precompute, so it sees exactly the
|
||||
readings and forecasts the dashboard shows. Per-key last-sent state is stored
|
||||
in the `notification_state` table of the monitor's own database, so a restart
|
||||
or redeploy never re-sends and never misses a crossing that happened while
|
||||
the service was down (the next cycle compares against the persisted state).
|
||||
It runs in the same process that serves the dashboard, right after the
|
||||
forecast, so it sees exactly the readings and forecasts the dashboard shows.
|
||||
Per-key last-sent state is stored in the `notification_state` table of the
|
||||
monitor's own database, so a restart or redeploy never re-sends and never
|
||||
misses a crossing that happened while the service was down (the next cycle
|
||||
compares against the persisted state).
|
||||
|
||||
If ntfy is unreachable the transition is **not** recorded, so it is retried
|
||||
on the next cycle rather than silently lost. Any other failure in the notify
|
||||
@@ -86,8 +85,8 @@ token, and appends `NTFY_SERVER` (public URL for subscribers),
|
||||
and `NTFY_TOKEN` to `.env` if they are not there yet. Then:
|
||||
|
||||
```bash
|
||||
systemctl restart water-monitor
|
||||
journalctl -u water-monitor -n 20 | grep ntfy # "ntfy notifications: https://... topics ping-*"
|
||||
systemctl restart ping-collector
|
||||
journalctl -u ping-collector -n 20 | grep ntfy # "ntfy notifications: publish to ..., topics ping-*"
|
||||
curl -s 'https://ntfy.buildfor.life/ping-status/json?poll=1' # anonymous read works
|
||||
```
|
||||
|
||||
@@ -120,21 +119,30 @@ holds back an alert. Test the pipeline before the domain is live with
|
||||
| `NTFY_TOKEN` | *(empty)* | bearer token if the server requires auth to publish (it does, see above) |
|
||||
| `PUBLIC_URL` | `https://water.buildfor.life/` | click-through target in messages |
|
||||
|
||||
Tunables in `src/notify.py`: `CLEAR_MARGIN_M` (0.10), `OUTLOOK_ON` / `OUTLOOK_OFF`
|
||||
(0.50 / 0.25), stale feed threshold (3 h, argument to `evaluate`).
|
||||
| `RID_STALE_ALERT_HOURS` | `18` | report RID lagging this long while ThaiWater is current (0 = never) |
|
||||
| `MATRIX_HOMESERVER` / `MATRIX_ACCESS_TOKEN` / `MATRIX_ROOM_ID` | *(empty = off)* | the team room (docs/MATRIX_QUICK_START.md) |
|
||||
|
||||
Tunables in `src/ml/export.py` (compiled into ping-collector through
|
||||
`rust/forecast/flood_config.json`: regenerate with
|
||||
`uv run python -m src.ml.export --config` and rebuild): `CLEAR_MARGIN_M`
|
||||
(0.10), `OUTLOOK_ON` / `OUTLOOK_OFF` (0.50 / 0.25), `FEED_STALE_AFTER_H` (3),
|
||||
the capacity guard and the station names in the messages.
|
||||
|
||||
## Testing
|
||||
|
||||
`tests/test_notify.py` covers the state machine: quiet river sends nothing;
|
||||
crossing once, then silence while above, then all-clear; hysteresis on the way
|
||||
down; escalation to danger and back; basin digest grouping; outlook on/off;
|
||||
heuristic forecasts ignored; stale feed and recovery; state survives a restart
|
||||
through sqlite; a failed publish is retried next cycle.
|
||||
The tests in `rust/collector/src/notify.rs` cover the state machine: quiet
|
||||
river sends nothing; crossing once, then silence while above, then all-clear;
|
||||
hysteresis on the way down; escalation to danger and back; outlook on/off with
|
||||
heuristic forecasts ignored; HII readings replacing a lagging RID reading;
|
||||
stale feed only when both sources are; RID lagging reported once; a failed
|
||||
publish retried next cycle; the Matrix rendering; and the 652-step replay of
|
||||
the Python notifier's messages. `notification_state` round-trips against
|
||||
PostgreSQL in `db_tests.rs`.
|
||||
|
||||
To exercise the real path against a real ntfy locally: run `ntfy serve` (any
|
||||
platform, same binary), set `NTFY_SERVER`/`NTFY_TOKEN`, seed readings, and
|
||||
poll the topic JSON. `scripts/e2e_notify.py` does exactly that if you want a
|
||||
template.
|
||||
`ping-collector notify --dry-run` shows the readings judged and what would be
|
||||
sent right now, changing nothing. To exercise the real path against a real
|
||||
ntfy locally: run `ntfy serve` (any platform, same binary), point
|
||||
`NTFY_SERVER`/`NTFY_TOKEN` at it and poll the topic JSON.
|
||||
|
||||
## Why ntfy and not …
|
||||
|
||||
|
||||
+25
-10
@@ -1,9 +1,17 @@
|
||||
# A Rust rewrite: assessment
|
||||
# The Rust rewrite
|
||||
|
||||
Status: **in progress.** Phase 1 (collector) is in production since 2026-10-17,
|
||||
phase 2 (forecast + ntfy) and phase 3 (website/API + Matrix) since
|
||||
2026-10-18; see
|
||||
[rust/README.md](../rust/README.md). Spike: [spikes/rust-forecast](../spikes/rust-forecast/README.md).
|
||||
Status: **done** (2026-10-18). The service is `ping-collector`
|
||||
([rust/README.md](../rust/README.md)); Python keeps model training,
|
||||
evaluation and the archive backfills. Phases 1-3 replaced the Python
|
||||
collector, the forecast + notifier and the website/API one at a time, each
|
||||
checked against the Python side on production data before the switch;
|
||||
phase 4 removed the Python service code. The migration tooling
|
||||
(`scripts/collector_parity.py`, `scripts/api_parity.py`,
|
||||
`scripts/install_collector.sh` with its cutover/rollback modes, the
|
||||
`spikes/rust-forecast` spike) is in git history before the cleanup commit.
|
||||
|
||||
The assessment below was written before phase 1 and is kept as the record of
|
||||
why and how.
|
||||
|
||||
## Short version
|
||||
|
||||
@@ -87,7 +95,7 @@ This slows model iteration, which is the most active part of the repo.
|
||||
| 1 | `ping-collector`: RID, HII, dams, Open-Meteo | in production since 2026-10-17 23:09 (Python: `COLLECTION_ENABLED=false`) |
|
||||
| 2 | Forecast crate + ntfy | in production since 2026-10-18 15:08 (`FORECAST_IN_COLLECTOR=true`); live check on production data: 48/48 rows identical, 96/96 probabilities bit-identical |
|
||||
| 3 | Website + read API in `ping-collector`, Matrix like ntfy | in production since 2026-10-18 16:40 (`WEB_LISTEN`, `water-monitor.service` disabled); on production data 53 of 53 compared routes identical to FastAPI (`scripts/api_parity.py`) |
|
||||
| 4 | Cleanup | |
|
||||
| 4 | Cleanup: Python service code removed | done 2026-10-18 |
|
||||
|
||||
0. **Spike: done.** P.1 features + HGB inference, bit-identical.
|
||||
1. **Collector daemon** (`rust/collector`, ~1.5k LOC incl. tests): RID with the
|
||||
@@ -96,7 +104,7 @@ This slows model iteration, which is the most active part of the repo.
|
||||
API runs with `COLLECTION_ENABLED=false` and re-forecasts when the newest
|
||||
RID reading in the database advances. ntfy moved to phase 2: the outlook
|
||||
alert needs the forecasts, which are still Python. Cutover and rollback
|
||||
are one command each (`scripts/install_collector.sh`).
|
||||
were one command each (`scripts/install_collector.sh`, since removed).
|
||||
2. **Forecast crate + ntfy** (`rust/forecast`, run by `ping-collector` with
|
||||
`FORECAST_IN_COLLECTOR=true`): all stations, HII/rain/dam inputs, heuristic
|
||||
fallback, probabilities and stages, `forecast_history`, `forecast_current`
|
||||
@@ -130,9 +138,16 @@ This slows model iteration, which is the most active part of the repo.
|
||||
their own `matrix:` state keys seeded from ntfy's, replacing the
|
||||
`run.py --alert-check` cron job that repeated alerts every 15 minutes.
|
||||
Python keeps training, evaluation and the monthly retrain timer.
|
||||
4. **Cleanup:** drop FastAPI/uvicorn, the Python collector, notifier, CLI and
|
||||
the multi-backend adapters (SQLite, MySQL, InfluxDB, VictoriaMetrics) and
|
||||
demo databases; the Python environment shrinks to the training stack.
|
||||
4. **Cleanup** (done): FastAPI/uvicorn, the Python collector, notifier, Matrix
|
||||
alerting, CLI (`run.py`), the multi-backend adapters (SQLite, MySQL,
|
||||
InfluxDB, VictoriaMetrics), demo databases, Docker/Grafana files and the
|
||||
migration scripts are gone, about 17k lines. The Python environment is the
|
||||
training stack (pandas, scikit-learn, SQLAlchemy/psycopg2): `src/ml`, the
|
||||
backfill clients/stores (`src/hii_collector.py`, `src/rid_reservoir.py`)
|
||||
and `src/ml/predict.py` as the reference implementation the Rust forecaster
|
||||
is tested against. PostgreSQL only; the Python database tests run against
|
||||
a throwaway schema (`TEST_DATABASE_URL`, as the Rust ones). One installer,
|
||||
`scripts/install.sh`.
|
||||
|
||||
Decided before phase 3 (2026-10-18): Matrix stays and behaves like ntfy;
|
||||
`/docs` stays; SQLite dev mode goes; the monthly retrain stays on the same
|
||||
|
||||
@@ -1,241 +0,0 @@
|
||||
# 🏔️ Station Management Guide - Northern Thailand Ping River Monitor
|
||||
|
||||
## 🎯 **Overview**
|
||||
|
||||
The Northern Thailand Ping River Monitor now includes comprehensive station management capabilities, allowing you to dynamically add, update, and remove monitoring stations through the web API.
|
||||
|
||||
## 🌊 **Current Coverage**
|
||||
|
||||
The system currently monitors **16 water stations** along the Ping River Basin:
|
||||
|
||||
### **Upper Ping River (Chiang Mai Province)**
|
||||
- **P.20** - Ban Chiang Dao (บ้านเชียงดาว)
|
||||
- **P.75** - Ban Chai Lat (บ้านช่อแล)
|
||||
- **P.92** - Ban Muang Aut (บ้านเมืองกึ๊ด)
|
||||
- **P.4A** - Ban Mae Taeng (บ้านแม่แตง)
|
||||
- **P.67** - Ban Tae (บ้านแม่แต)
|
||||
- **P.21** - Ban Rim Tai (บ้านริมใต้)
|
||||
- **P.103** - Ring Bridge 3 (สะพานวงแหวนรอบ 3)
|
||||
|
||||
### **Middle Ping River**
|
||||
- **P.1** - Nawarat Bridge (สะพานนวรัฐ) - *Main reference station*
|
||||
- **P.82** - Ban Sob win (บ้านสบวิน)
|
||||
- **P.84** - Ban Panton (บ้านพันตน)
|
||||
- **P.81** - Ban Pong (บ้านโป่ง)
|
||||
- **P.5** - Tha Nang Bridge (สะพานท่านาง)
|
||||
|
||||
### **Lower Ping River**
|
||||
- **P.77** - Baan Sop Mae Sapuord (บ้านสบแม่สะป๊วด)
|
||||
- **P.87** - Ban Pa Sang (บ้านป่าซาง)
|
||||
- **P.76** - Banb Mae I Hai (บ้านแม่อีไฮ)
|
||||
- **P.85** - Baan Lai Kaew (บ้านหล่ายแก้ว)
|
||||
|
||||
## 🔧 **Station Management API**
|
||||
|
||||
### **List All Stations**
|
||||
```bash
|
||||
GET /stations
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
[
|
||||
{
|
||||
"station_id": 1,
|
||||
"station_code": "P.20",
|
||||
"thai_name": "บ้านเชียงดาว",
|
||||
"english_name": "Ban Chiang Dao",
|
||||
"latitude": 19.36731448032191,
|
||||
"longitude": 98.9688487015384,
|
||||
"geohash": null,
|
||||
"status": "active"
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
### **Get Specific Station**
|
||||
```bash
|
||||
GET /stations/{station_id}
|
||||
```
|
||||
|
||||
### **Add New Station**
|
||||
```bash
|
||||
POST /stations
|
||||
Content-Type: application/json
|
||||
|
||||
{
|
||||
"station_code": "P.NEW",
|
||||
"thai_name": "สถานีใหม่",
|
||||
"english_name": "New Station",
|
||||
"latitude": 18.7875,
|
||||
"longitude": 99.0045,
|
||||
"geohash": "w5q6uuhvfcfp25",
|
||||
"status": "active"
|
||||
}
|
||||
```
|
||||
|
||||
### **Update Station Information**
|
||||
```bash
|
||||
PUT /stations/{station_id}
|
||||
Content-Type: application/json
|
||||
|
||||
{
|
||||
"thai_name": "ชื่อใหม่",
|
||||
"english_name": "Updated Name",
|
||||
"latitude": 18.8000,
|
||||
"longitude": 99.0100
|
||||
}
|
||||
```
|
||||
|
||||
### **Delete Station**
|
||||
```bash
|
||||
DELETE /stations/{station_id}
|
||||
```
|
||||
|
||||
## 🧪 **Testing Station Management**
|
||||
|
||||
Use the provided test script to verify station management functionality:
|
||||
|
||||
```bash
|
||||
# Test all station management endpoints
|
||||
python test_station_management.py
|
||||
```
|
||||
|
||||
This will:
|
||||
1. List existing stations
|
||||
2. Create a test station
|
||||
3. Retrieve station details
|
||||
4. Update station information
|
||||
5. Verify changes
|
||||
6. Delete the test station
|
||||
7. Confirm deletion
|
||||
|
||||
## 📊 **Station Data Model**
|
||||
|
||||
### **Required Fields**
|
||||
- `station_code`: Unique identifier (e.g., "P.1", "P.20")
|
||||
- `thai_name`: Thai language name
|
||||
- `english_name`: English language name
|
||||
|
||||
### **Optional Fields**
|
||||
- `latitude`: GPS latitude coordinate (-90 to 90)
|
||||
- `longitude`: GPS longitude coordinate (-180 to 180)
|
||||
- `geohash`: Geohash string for location
|
||||
- `status`: Station status ("active", "inactive", "maintenance", "error")
|
||||
|
||||
### **Validation Rules**
|
||||
- Station codes must be unique
|
||||
- Latitude must be between -90 and 90
|
||||
- Longitude must be between -180 and 180
|
||||
- Names cannot be empty
|
||||
- Status must be valid enum value
|
||||
|
||||
## 🌐 **Web Interface**
|
||||
|
||||
Access the station management interface through the web dashboard:
|
||||
|
||||
1. **Start the API server:**
|
||||
```bash
|
||||
python run.py --web-api
|
||||
```
|
||||
|
||||
2. **Open your browser:**
|
||||
- Dashboard: http://localhost:8000
|
||||
- API Documentation: http://localhost:8000/docs
|
||||
|
||||
3. **Use the interactive API docs** to test station management endpoints
|
||||
|
||||
## 🔄 **Integration with Data Collection**
|
||||
|
||||
- **Dynamic Station Discovery**: New stations are automatically included in data collection
|
||||
- **Real-time Updates**: Station information changes are reflected immediately
|
||||
- **Data Continuity**: Historical data is preserved when updating station details
|
||||
- **Error Handling**: Invalid stations are skipped during data collection
|
||||
|
||||
## 📍 **Geographic Coverage**
|
||||
|
||||
The Ping River Basin monitoring network covers:
|
||||
|
||||
- **Total Distance**: ~400 km from Chiang Dao to Nakhon Sawan
|
||||
- **Elevation Range**: 300m to 1,200m above sea level
|
||||
- **Catchment Area**: ~25,000 km²
|
||||
- **Major Cities**: Chiang Mai, Lamphun, Tak, Nakhon Sawan
|
||||
|
||||
## 🚀 **Usage Examples**
|
||||
|
||||
### **Add a New Upstream Station**
|
||||
```bash
|
||||
curl -X POST "http://localhost:8000/stations" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"station_code": "P.UPSTREAM",
|
||||
"thai_name": "สถานีต้นน้ำ",
|
||||
"english_name": "Upstream Station",
|
||||
"latitude": 19.5000,
|
||||
"longitude": 98.9000,
|
||||
"status": "active"
|
||||
}'
|
||||
```
|
||||
|
||||
### **Update Station Coordinates**
|
||||
```bash
|
||||
curl -X PUT "http://localhost:8000/stations/1" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"latitude": 19.3700,
|
||||
"longitude": 98.9700
|
||||
}'
|
||||
```
|
||||
|
||||
### **Mark Station for Maintenance**
|
||||
```bash
|
||||
curl -X PUT "http://localhost:8000/stations/5" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"status": "maintenance"
|
||||
}'
|
||||
```
|
||||
|
||||
## 🔒 **Best Practices**
|
||||
|
||||
### **Station Naming**
|
||||
- Use consistent code format (P.XX)
|
||||
- Include both Thai and English names
|
||||
- Use descriptive location names
|
||||
|
||||
### **Coordinate Accuracy**
|
||||
- Use high-precision GPS coordinates (6+ decimal places)
|
||||
- Verify coordinates match actual station location
|
||||
- Include geohash for efficient spatial queries
|
||||
|
||||
### **Status Management**
|
||||
- Set status to "maintenance" during repairs
|
||||
- Use "inactive" for temporarily offline stations
|
||||
- Use "error" for stations with data quality issues
|
||||
|
||||
### **Data Integrity**
|
||||
- Test new stations before adding to production
|
||||
- Backup station configuration before major changes
|
||||
- Monitor data quality after station updates
|
||||
|
||||
## 🎯 **Future Enhancements**
|
||||
|
||||
Planned improvements for station management:
|
||||
|
||||
1. **Bulk Operations** - Import/export multiple stations
|
||||
2. **Station Groups** - Organize stations by river section
|
||||
3. **Automated Validation** - GPS coordinate verification
|
||||
4. **Historical Tracking** - Track station configuration changes
|
||||
5. **Alert Integration** - Notifications for station status changes
|
||||
6. **Map Interface** - Visual station management on interactive map
|
||||
|
||||
## 📞 **Support**
|
||||
|
||||
For station management issues:
|
||||
|
||||
1. Check the API documentation at `/docs`
|
||||
2. Run the test script: `python test_station_management.py`
|
||||
3. Review logs for error details
|
||||
4. Verify station data format and validation rules
|
||||
|
||||
The station management system provides flexible control over your monitoring network while maintaining data integrity and system reliability.
|
||||
@@ -1,443 +0,0 @@
|
||||
# VictoriaMetrics Setup Guide for Thailand Water Monitor
|
||||
|
||||
This guide provides comprehensive instructions for setting up VictoriaMetrics as the time-series database backend for the Thailand Water Monitor.
|
||||
|
||||
## Why VictoriaMetrics?
|
||||
|
||||
VictoriaMetrics is an excellent choice for water monitoring data because:
|
||||
|
||||
- **High Performance**: Up to 10x faster than InfluxDB
|
||||
- **Low Resource Usage**: Uses 10x less RAM than Prometheus
|
||||
- **Better Compression**: 70x better compression than Prometheus
|
||||
- **Prometheus Compatible**: Drop-in replacement for Prometheus
|
||||
- **Easy to Deploy**: Single binary, no dependencies
|
||||
- **Cost Effective**: Open source with commercial support available
|
||||
|
||||
## Quick Start
|
||||
|
||||
### 1. Environment Variables
|
||||
|
||||
Set these environment variables to configure VictoriaMetrics:
|
||||
|
||||
```bash
|
||||
# Windows (PowerShell)
|
||||
$env:DB_TYPE="victoriametrics"
|
||||
$env:VM_HOST="localhost"
|
||||
$env:VM_PORT="8428"
|
||||
|
||||
# Linux/Mac
|
||||
export DB_TYPE=victoriametrics
|
||||
export VM_HOST=localhost
|
||||
export VM_PORT=8428
|
||||
```
|
||||
|
||||
### 2. Start VictoriaMetrics with Docker
|
||||
|
||||
```bash
|
||||
# Simple setup
|
||||
docker run -d \
|
||||
--name victoriametrics \
|
||||
-p 8428:8428 \
|
||||
-v victoria-metrics-data:/victoria-metrics-data \
|
||||
victoriametrics/victoria-metrics:latest \
|
||||
--storageDataPath=/victoria-metrics-data \
|
||||
--retentionPeriod=2y \
|
||||
--httpListenAddr=:8428
|
||||
|
||||
# Verify it's running
|
||||
curl http://localhost:8428/health
|
||||
```
|
||||
|
||||
### 3. Run the Water Monitor
|
||||
|
||||
```bash
|
||||
python water_scraper_v3.py
|
||||
```
|
||||
|
||||
### 4. Access Grafana Dashboard
|
||||
|
||||
```bash
|
||||
# Start with Docker Compose (includes Grafana)
|
||||
docker-compose -f docker-compose.victoriametrics.yml up -d
|
||||
|
||||
# Access Grafana at http://localhost:3000
|
||||
# Username: admin
|
||||
# Password: admin_password
|
||||
```
|
||||
|
||||
## Production Setup
|
||||
|
||||
### Docker Compose Configuration
|
||||
|
||||
Use the provided `docker-compose.victoriametrics.yml` file:
|
||||
|
||||
```bash
|
||||
# Start the complete stack
|
||||
docker-compose -f docker-compose.victoriametrics.yml up -d
|
||||
|
||||
# Check status
|
||||
docker-compose -f docker-compose.victoriametrics.yml ps
|
||||
|
||||
# View logs
|
||||
docker-compose -f docker-compose.victoriametrics.yml logs -f
|
||||
```
|
||||
|
||||
### Manual VictoriaMetrics Configuration
|
||||
|
||||
#### High-Performance Configuration
|
||||
|
||||
```bash
|
||||
docker run -d \
|
||||
--name victoriametrics \
|
||||
-p 8428:8428 \
|
||||
-v victoria-metrics-data:/victoria-metrics-data \
|
||||
victoriametrics/victoria-metrics:latest \
|
||||
--storageDataPath=/victoria-metrics-data \
|
||||
--retentionPeriod=2y \
|
||||
--httpListenAddr=:8428 \
|
||||
--maxConcurrentInserts=32 \
|
||||
--search.maxQueryDuration=60s \
|
||||
--search.maxConcurrentRequests=16 \
|
||||
--dedup.minScrapeInterval=30s \
|
||||
--memory.allowedPercent=80 \
|
||||
--loggerLevel=INFO \
|
||||
--loggerFormat=json \
|
||||
--search.maxSeries=1000000 \
|
||||
--search.maxPointsPerTimeseries=100000
|
||||
```
|
||||
|
||||
#### Configuration Parameters Explained
|
||||
|
||||
| Parameter | Description | Recommended Value |
|
||||
|-----------|-------------|-------------------|
|
||||
| `--storageDataPath` | Data storage directory | `/victoria-metrics-data` |
|
||||
| `--retentionPeriod` | How long to keep data | `2y` (2 years) |
|
||||
| `--httpListenAddr` | HTTP listen address | `:8428` |
|
||||
| `--maxConcurrentInserts` | Max concurrent inserts | `32` |
|
||||
| `--search.maxQueryDuration` | Max query duration | `60s` |
|
||||
| `--search.maxConcurrentRequests` | Max concurrent queries | `16` |
|
||||
| `--dedup.minScrapeInterval` | Deduplication interval | `30s` |
|
||||
| `--memory.allowedPercent` | Max memory usage | `80` |
|
||||
| `--loggerLevel` | Log level | `INFO` |
|
||||
| `--search.maxSeries` | Max time series | `1000000` |
|
||||
|
||||
### Kubernetes Deployment
|
||||
|
||||
```yaml
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: victoriametrics
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: victoriametrics
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: victoriametrics
|
||||
spec:
|
||||
containers:
|
||||
- name: victoriametrics
|
||||
image: victoriametrics/victoria-metrics:latest
|
||||
ports:
|
||||
- containerPort: 8428
|
||||
args:
|
||||
- --storageDataPath=/victoria-metrics-data
|
||||
- --retentionPeriod=2y
|
||||
- --httpListenAddr=:8428
|
||||
- --maxConcurrentInserts=32
|
||||
volumeMounts:
|
||||
- name: storage
|
||||
mountPath: /victoria-metrics-data
|
||||
resources:
|
||||
requests:
|
||||
memory: "512Mi"
|
||||
cpu: "500m"
|
||||
limits:
|
||||
memory: "2Gi"
|
||||
cpu: "2000m"
|
||||
volumes:
|
||||
- name: storage
|
||||
persistentVolumeClaim:
|
||||
claimName: victoriametrics-pvc
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: victoriametrics
|
||||
spec:
|
||||
selector:
|
||||
app: victoriametrics
|
||||
ports:
|
||||
- port: 8428
|
||||
targetPort: 8428
|
||||
type: ClusterIP
|
||||
```
|
||||
|
||||
## Data Queries
|
||||
|
||||
### HTTP API Queries
|
||||
|
||||
VictoriaMetrics provides a Prometheus-compatible HTTP API:
|
||||
|
||||
```bash
|
||||
# Current water levels for all stations
|
||||
curl "http://localhost:8428/api/v1/query?query=water_level"
|
||||
|
||||
# Water levels for specific station
|
||||
curl "http://localhost:8428/api/v1/query?query=water_level{station_code=\"P.1\"}"
|
||||
|
||||
# Average discharge over last hour
|
||||
curl "http://localhost:8428/api/v1/query?query=avg_over_time(water_discharge[1h])"
|
||||
|
||||
# High discharge alerts (>80%)
|
||||
curl "http://localhost:8428/api/v1/query?query=water_discharge_percent>80"
|
||||
|
||||
# Time range query (last 6 hours)
|
||||
START=$(date -d '6 hours ago' +%s)
|
||||
END=$(date +%s)
|
||||
curl "http://localhost:8428/api/v1/query_range?query=water_level&start=${START}&end=${END}&step=300"
|
||||
```
|
||||
|
||||
### PromQL Examples
|
||||
|
||||
```promql
|
||||
# Current water levels
|
||||
water_level
|
||||
|
||||
# Water level trends (last 24h)
|
||||
water_level[24h]
|
||||
|
||||
# Discharge rates by station
|
||||
water_discharge{station_code="P.1"}
|
||||
|
||||
# Average discharge across all stations
|
||||
avg(water_discharge)
|
||||
|
||||
# Stations with high discharge (>80%)
|
||||
water_discharge_percent > 80
|
||||
|
||||
# Rate of change in water level
|
||||
rate(water_level[5m])
|
||||
|
||||
# Maximum water level in last hour
|
||||
max_over_time(water_level[1h])
|
||||
|
||||
# Stations with increasing water levels
|
||||
increase(water_level[1h]) > 0
|
||||
```
|
||||
|
||||
## Grafana Integration
|
||||
|
||||
### Data Source Configuration
|
||||
|
||||
1. **Add VictoriaMetrics as Prometheus Data Source**:
|
||||
- URL: `http://localhost:8428` (or `http://victoriametrics:8428` in Docker)
|
||||
- Access: Server (default)
|
||||
- HTTP Method: POST
|
||||
|
||||
2. **Import Dashboard**:
|
||||
- Use the provided `water-monitoring-dashboard.json`
|
||||
- Or create custom dashboards with the queries above
|
||||
|
||||
### Dashboard Panels
|
||||
|
||||
The included dashboard provides:
|
||||
|
||||
- **Time Series**: Water levels and discharge over time
|
||||
- **Table**: Current status of all stations
|
||||
- **Pie Chart**: Discharge percentage distribution
|
||||
- **Gauge**: Average discharge percentage
|
||||
- **Variables**: Filter by station
|
||||
|
||||
## Monitoring and Maintenance
|
||||
|
||||
### Health Checks
|
||||
|
||||
```bash
|
||||
# Check VictoriaMetrics health
|
||||
curl http://localhost:8428/health
|
||||
|
||||
# Check metrics endpoint
|
||||
curl http://localhost:8428/metrics
|
||||
|
||||
# Check configuration
|
||||
curl http://localhost:8428/api/v1/status/config
|
||||
```
|
||||
|
||||
### Performance Monitoring
|
||||
|
||||
```bash
|
||||
# Query performance stats
|
||||
curl http://localhost:8428/api/v1/status/tsdb
|
||||
|
||||
# Memory usage
|
||||
curl http://localhost:8428/api/v1/status/runtime
|
||||
|
||||
# Active queries
|
||||
curl http://localhost:8428/api/v1/status/active_queries
|
||||
```
|
||||
|
||||
### Backup and Restore
|
||||
|
||||
```bash
|
||||
# Create backup
|
||||
docker exec victoriametrics /usr/bin/vmbackup \
|
||||
-storageDataPath=/victoria-metrics-data \
|
||||
-dst=fs:///backup/$(date +%Y%m%d)
|
||||
|
||||
# Restore from backup
|
||||
docker exec victoriametrics /usr/bin/vmrestore \
|
||||
-src=fs:///backup/20250724 \
|
||||
-storageDataPath=/victoria-metrics-data
|
||||
```
|
||||
|
||||
### Log Analysis
|
||||
|
||||
```bash
|
||||
# View logs
|
||||
docker logs victoriametrics
|
||||
|
||||
# Follow logs
|
||||
docker logs -f victoriametrics
|
||||
|
||||
# Search for errors
|
||||
docker logs victoriametrics 2>&1 | grep ERROR
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Common Issues
|
||||
|
||||
1. **Connection Refused**:
|
||||
```bash
|
||||
# Check if VictoriaMetrics is running
|
||||
docker ps | grep victoriametrics
|
||||
|
||||
# Check port binding
|
||||
netstat -tlnp | grep 8428
|
||||
```
|
||||
|
||||
2. **High Memory Usage**:
|
||||
```bash
|
||||
# Reduce memory limit
|
||||
docker run ... --memory.allowedPercent=60 ...
|
||||
```
|
||||
|
||||
3. **Slow Queries**:
|
||||
```bash
|
||||
# Increase query timeout
|
||||
docker run ... --search.maxQueryDuration=120s ...
|
||||
```
|
||||
|
||||
4. **Data Not Appearing**:
|
||||
```bash
|
||||
# Check if data is being written
|
||||
curl "http://localhost:8428/api/v1/query?query=up"
|
||||
|
||||
# Check water monitor logs
|
||||
tail -f water_monitor.log
|
||||
```
|
||||
|
||||
### Performance Tuning
|
||||
|
||||
1. **For High Write Load**:
|
||||
```bash
|
||||
--maxConcurrentInserts=64
|
||||
--insert.maxQueueDuration=60s
|
||||
```
|
||||
|
||||
2. **For High Query Load**:
|
||||
```bash
|
||||
--search.maxConcurrentRequests=32
|
||||
--search.maxQueryDuration=120s
|
||||
```
|
||||
|
||||
3. **For Large Datasets**:
|
||||
```bash
|
||||
--search.maxSeries=10000000
|
||||
--search.maxPointsPerTimeseries=1000000
|
||||
```
|
||||
|
||||
## Security
|
||||
|
||||
### Authentication
|
||||
|
||||
VictoriaMetrics doesn't have built-in authentication. Use a reverse proxy:
|
||||
|
||||
```nginx
|
||||
server {
|
||||
listen 80;
|
||||
server_name victoriametrics.example.com;
|
||||
|
||||
auth_basic "VictoriaMetrics";
|
||||
auth_basic_user_file /etc/nginx/.htpasswd;
|
||||
|
||||
location / {
|
||||
proxy_pass http://localhost:8428;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### TLS/SSL
|
||||
|
||||
```bash
|
||||
# Use nginx or traefik for TLS termination
|
||||
# Or use VictoriaMetrics with TLS:
|
||||
docker run ... \
|
||||
-v /path/to/cert.pem:/cert.pem \
|
||||
-v /path/to/key.pem:/key.pem \
|
||||
victoriametrics/victoria-metrics:latest \
|
||||
--tls \
|
||||
--tlsCertFile=/cert.pem \
|
||||
--tlsKeyFile=/key.pem
|
||||
```
|
||||
|
||||
## Scaling
|
||||
|
||||
### Cluster Setup
|
||||
|
||||
For high availability and horizontal scaling:
|
||||
|
||||
```bash
|
||||
# Start multiple VictoriaMetrics instances
|
||||
docker run -d --name vm1 -p 8428:8428 victoriametrics/victoria-metrics:latest
|
||||
docker run -d --name vm2 -p 8429:8428 victoriametrics/victoria-metrics:latest
|
||||
|
||||
# Use load balancer to distribute queries
|
||||
# Use vminsert/vmselect/vmstorage for true clustering
|
||||
```
|
||||
|
||||
### Resource Requirements
|
||||
|
||||
| Data Points/Hour | RAM | CPU | Storage/Day |
|
||||
|------------------|-----|-----|-------------|
|
||||
| 1,000 | 100MB | 0.1 CPU | 10MB |
|
||||
| 10,000 | 500MB | 0.5 CPU | 100MB |
|
||||
| 100,000 | 2GB | 1 CPU | 1GB |
|
||||
| 1,000,000 | 8GB | 2 CPU | 10GB |
|
||||
|
||||
## Migration
|
||||
|
||||
### From InfluxDB
|
||||
|
||||
```bash
|
||||
# Export from InfluxDB
|
||||
influx -database water_monitoring -execute "SELECT * FROM water_data" -format csv > data.csv
|
||||
|
||||
# Import to VictoriaMetrics (convert to Prometheus format first)
|
||||
# Use vmctl tool for migration
|
||||
```
|
||||
|
||||
### From Prometheus
|
||||
|
||||
```bash
|
||||
# Use vmctl for direct migration
|
||||
vmctl prometheus --prom-snapshot=/path/to/prometheus/data --vm-addr=http://localhost:8428
|
||||
```
|
||||
|
||||
This comprehensive setup guide should help you configure VictoriaMetrics for optimal performance with the Thailand Water Monitor system.
|
||||
@@ -1,543 +0,0 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 0,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"liveNow": false,
|
||||
"panels": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "victoriametrics-uid"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"vis": false
|
||||
},
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 2,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "m"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 1,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "victoriametrics-uid"
|
||||
},
|
||||
"expr": "water_level",
|
||||
"interval": "",
|
||||
"legendFormat": "{{station_code}} - {{station_name_en}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Water Levels by Station",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "victoriametrics-uid"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"vis": false
|
||||
},
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 2,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "cms"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 0
|
||||
},
|
||||
"id": 2,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "victoriametrics-uid"
|
||||
},
|
||||
"expr": "water_discharge",
|
||||
"interval": "",
|
||||
"legendFormat": "{{station_code}} - {{station_name_en}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Water Discharge by Station",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "victoriametrics-uid"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"custom": {
|
||||
"align": "auto",
|
||||
"displayMode": "auto",
|
||||
"inspect": false
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "Water Level"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "unit",
|
||||
"value": "m"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "Discharge"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "unit",
|
||||
"value": "cms"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "Discharge %"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "unit",
|
||||
"value": "percent"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 8
|
||||
},
|
||||
"id": 3,
|
||||
"options": {
|
||||
"showHeader": true,
|
||||
"sortBy": [
|
||||
{
|
||||
"desc": false,
|
||||
"displayName": "Station"
|
||||
}
|
||||
]
|
||||
},
|
||||
"pluginVersion": "8.5.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "victoriametrics-uid"
|
||||
},
|
||||
"expr": "water_level",
|
||||
"format": "table",
|
||||
"instant": true,
|
||||
"interval": "",
|
||||
"legendFormat": "",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "victoriametrics-uid"
|
||||
},
|
||||
"expr": "water_discharge",
|
||||
"format": "table",
|
||||
"instant": true,
|
||||
"interval": "",
|
||||
"legendFormat": "",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "victoriametrics-uid"
|
||||
},
|
||||
"expr": "water_discharge_percent",
|
||||
"format": "table",
|
||||
"instant": true,
|
||||
"interval": "",
|
||||
"legendFormat": "",
|
||||
"refId": "C"
|
||||
}
|
||||
],
|
||||
"title": "Current Station Status",
|
||||
"transformations": [
|
||||
{
|
||||
"id": "merge",
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": "organize",
|
||||
"options": {
|
||||
"excludeByName": {
|
||||
"Time": true,
|
||||
"__name__": true,
|
||||
"job": true,
|
||||
"instance": true
|
||||
},
|
||||
"indexByName": {
|
||||
"station_code": 0,
|
||||
"station_name_en": 1,
|
||||
"station_name_th": 2,
|
||||
"Value #A": 3,
|
||||
"Value #B": 4,
|
||||
"Value #C": 5
|
||||
},
|
||||
"renameByName": {
|
||||
"Value #A": "Water Level",
|
||||
"Value #B": "Discharge",
|
||||
"Value #C": "Discharge %",
|
||||
"station_code": "Station",
|
||||
"station_name_en": "English Name",
|
||||
"station_name_th": "Thai Name"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"type": "table"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "victoriametrics-uid"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"vis": false
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"unit": "percent"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 16
|
||||
},
|
||||
"id": 4,
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "right"
|
||||
},
|
||||
"pieType": "pie",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "victoriametrics-uid"
|
||||
},
|
||||
"expr": "water_discharge_percent",
|
||||
"interval": "",
|
||||
"legendFormat": "{{station_code}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Discharge Percentage Distribution",
|
||||
"type": "piechart"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "victoriametrics-uid"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 70
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 90
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "percent"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 16
|
||||
},
|
||||
"id": 5,
|
||||
"options": {
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showThresholdLabels": false,
|
||||
"showThresholdMarkers": true,
|
||||
"text": {}
|
||||
},
|
||||
"pluginVersion": "8.5.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "victoriametrics-uid"
|
||||
},
|
||||
"expr": "avg(water_discharge_percent)",
|
||||
"interval": "",
|
||||
"legendFormat": "Average Discharge %",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Average Discharge Percentage",
|
||||
"type": "gauge"
|
||||
}
|
||||
],
|
||||
"refresh": "30s",
|
||||
"schemaVersion": 36,
|
||||
"style": "dark",
|
||||
"tags": [
|
||||
"water-monitoring",
|
||||
"thailand",
|
||||
"victoriametrics"
|
||||
],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"current": {
|
||||
"selected": false,
|
||||
"text": "All",
|
||||
"value": "$__all"
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "victoriametrics-uid"
|
||||
},
|
||||
"definition": "label_values(water_level, station_code)",
|
||||
"hide": 0,
|
||||
"includeAll": true,
|
||||
"label": "Station",
|
||||
"multi": true,
|
||||
"name": "station",
|
||||
"options": [],
|
||||
"query": {
|
||||
"query": "label_values(water_level, station_code)",
|
||||
"refId": "StandardVariableQuery"
|
||||
},
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"skipUrlSync": false,
|
||||
"sort": 1,
|
||||
"type": "query"
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": {
|
||||
"from": "now-6h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "Asia/Bangkok",
|
||||
"title": "Thailand Water Level Monitoring",
|
||||
"uid": "water-monitoring-dashboard",
|
||||
"version": 1,
|
||||
"weekStart": ""
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: 'water-monitoring'
|
||||
orgId: 1
|
||||
folder: 'Water Monitoring'
|
||||
type: file
|
||||
disableDeletion: false
|
||||
updateIntervalSeconds: 10
|
||||
allowUiUpdates: true
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
@@ -1,17 +0,0 @@
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: VictoriaMetrics
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://victoriametrics:8428
|
||||
isDefault: true
|
||||
editable: true
|
||||
jsonData:
|
||||
httpMethod: POST
|
||||
queryTimeout: 60s
|
||||
timeInterval: 30s
|
||||
customQueryParameters: ''
|
||||
secureJsonData: {}
|
||||
version: 1
|
||||
uid: victoriametrics-uid
|
||||
+9
-75
@@ -5,122 +5,56 @@ build-backend = "setuptools.build_meta"
|
||||
[project]
|
||||
name = "northern-thailand-ping-river-monitor"
|
||||
version = "3.1.3"
|
||||
description = "Real-time water level monitoring system for the Ping River Basin in Northern Thailand"
|
||||
description = "Ping River Monitor: flood-model training, evaluation and archive backfills (the service, ping-collector, is in rust/)"
|
||||
readme = "README.md"
|
||||
license = {text = "MIT"}
|
||||
authors = [
|
||||
{name = "Ping River Monitor Team", email = "contact@example.com"}
|
||||
{name = "buildfor.life"}
|
||||
]
|
||||
keywords = [
|
||||
"water monitoring",
|
||||
"hydrology",
|
||||
"flood forecasting",
|
||||
"thailand",
|
||||
"ping river",
|
||||
"environmental monitoring",
|
||||
"time series",
|
||||
"fastapi",
|
||||
"real-time data"
|
||||
]
|
||||
classifiers = [
|
||||
"Development Status :: 4 - Beta",
|
||||
"Development Status :: 5 - Production/Stable",
|
||||
"Intended Audience :: Science/Research",
|
||||
"Intended Audience :: System Administrators",
|
||||
"Topic :: Scientific/Engineering :: Hydrology",
|
||||
"Topic :: System :: Monitoring",
|
||||
"Programming Language :: Python :: 3",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Operating System :: OS Independent",
|
||||
"Environment :: Web Environment",
|
||||
"Framework :: FastAPI"
|
||||
]
|
||||
# 3.11 only: pandas 2.0.3 ships no 3.12 wheels; psycopg2-binary 2.9.9 breaks on 3.13
|
||||
requires-python = ">=3.11,<3.12"
|
||||
dependencies = [
|
||||
# Core dependencies
|
||||
"requests==2.34.2",
|
||||
"schedule==1.2.0",
|
||||
"pandas==2.0.3",
|
||||
"numpy>=1.24,<2",
|
||||
# Flood forecasting (ML)
|
||||
"numpy>=1.24,<2", # pandas 2.0.3 wheels are ABI-incompatible with numpy 2.x
|
||||
"scikit-learn==1.9.0",
|
||||
# Web API framework
|
||||
"fastapi==0.141.1",
|
||||
"uvicorn[standard]==0.52.4",
|
||||
"pydantic==2.13.5",
|
||||
# Database adapters
|
||||
"sqlalchemy==2.0.23",
|
||||
"influxdb==5.3.1",
|
||||
"pymysql==1.2.0",
|
||||
"psycopg2-binary==2.9.9",
|
||||
# Monitoring and metrics
|
||||
"psutil==5.9.6"
|
||||
# .env loading; rust/collector/src/envfile.rs ports exactly this version
|
||||
"python-dotenv==1.1.1",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
# Testing
|
||||
"pytest==9.1.1",
|
||||
"pytest-cov==4.1.0",
|
||||
"pytest-asyncio==0.21.1",
|
||||
# Code formatting and linting
|
||||
"black==26.5.1",
|
||||
"flake8==6.1.0",
|
||||
"isort==5.12.0",
|
||||
"mypy==1.7.1",
|
||||
# Pre-commit hooks
|
||||
"pre-commit==3.5.0",
|
||||
# Development tools
|
||||
"ipython==8.17.2",
|
||||
"jupyter==1.0.0",
|
||||
# Type stubs
|
||||
"types-requests==2.33.0.20260906",
|
||||
"types-python-dateutil==2.8.19.14"
|
||||
]
|
||||
docs = [
|
||||
"sphinx==7.2.6",
|
||||
"sphinx-rtd-theme==1.3.0",
|
||||
"sphinx-autodoc-typehints==1.25.2"
|
||||
]
|
||||
all = [
|
||||
"influxdb==5.3.1",
|
||||
"pymysql==1.2.0",
|
||||
"psycopg2-binary==2.9.9"
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
ping-river-monitor = "src.main:main"
|
||||
ping-river-api = "src.web_api:main"
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor"
|
||||
Homepage = "https://water.buildfor.life/"
|
||||
Repository = "https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor"
|
||||
Issues = "https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/issues"
|
||||
Documentation = "https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/wiki"
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
# Testing
|
||||
"pytest==9.1.1",
|
||||
"pytest-cov==4.1.0",
|
||||
"pytest-asyncio==0.21.1",
|
||||
# Code formatting and linting
|
||||
"black==26.5.1",
|
||||
"flake8==6.1.0",
|
||||
"isort==5.12.0",
|
||||
"mypy==1.7.1",
|
||||
# Pre-commit hooks
|
||||
"pre-commit==3.5.0",
|
||||
# Development tools
|
||||
"ipython==8.17.2",
|
||||
"jupyter==1.0.0",
|
||||
# Type stubs
|
||||
"types-requests==2.33.0.20260906",
|
||||
"types-python-dateutil==2.8.19.14",
|
||||
# Documentation
|
||||
"sphinx==7.2.6",
|
||||
"sphinx-rtd-theme==1.3.0",
|
||||
"sphinx-autodoc-typehints==1.25.2",
|
||||
"pyinstaller>=6.16.0",
|
||||
]
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
|
||||
@@ -1,29 +1,8 @@
|
||||
# Development dependencies
|
||||
-r requirements.txt
|
||||
|
||||
# Testing
|
||||
pytest==9.1.1
|
||||
pytest-cov==4.1.0
|
||||
pytest-asyncio==0.21.1
|
||||
|
||||
# Code formatting and linting
|
||||
black==26.5.1
|
||||
flake8==6.1.0
|
||||
isort==5.12.0
|
||||
mypy==1.7.1
|
||||
|
||||
# Pre-commit hooks
|
||||
pre-commit==3.5.0
|
||||
|
||||
# Documentation
|
||||
sphinx==7.2.6
|
||||
sphinx-rtd-theme==1.3.0
|
||||
sphinx-autodoc-typehints==1.25.2
|
||||
|
||||
# Development tools
|
||||
ipython==8.17.2
|
||||
jupyter==1.0.0
|
||||
|
||||
# Type stubs
|
||||
types-requests==2.33.0.20260906
|
||||
types-python-dateutil==2.8.19.14
|
||||
+3
-28
@@ -1,34 +1,9 @@
|
||||
# Core dependencies
|
||||
# Python side: flood-model training, evaluation, backfills (pyproject.toml is
|
||||
# the source; uv.lock pins the rest). The service is ping-collector (rust/).
|
||||
requests==2.34.2
|
||||
schedule==1.2.0
|
||||
pandas==2.0.3
|
||||
numpy>=1.24,<2 # pandas 2.0.3 wheels are ABI-incompatible with numpy 2.x
|
||||
|
||||
# Flood forecasting (ML)
|
||||
scikit-learn==1.9.0
|
||||
|
||||
# Web API framework
|
||||
fastapi==0.141.1
|
||||
uvicorn[standard]==0.52.4
|
||||
pydantic==2.13.5
|
||||
|
||||
# Database adapters
|
||||
sqlalchemy==2.0.23
|
||||
influxdb==5.3.1
|
||||
pymysql==1.2.0
|
||||
psycopg2-binary==2.9.9
|
||||
|
||||
# Monitoring and metrics
|
||||
psutil==5.9.6
|
||||
|
||||
# Development dependencies (optional)
|
||||
pytest==9.1.1
|
||||
pytest-cov==4.1.0
|
||||
black==26.5.1
|
||||
flake8==6.1.0
|
||||
mypy==1.7.1
|
||||
pre-commit==3.5.0
|
||||
|
||||
# Documentation
|
||||
sphinx==7.2.6
|
||||
sphinx-rtd-theme==1.3.0
|
||||
python-dotenv==1.1.1
|
||||
@@ -1,15 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Simple startup script for Thailand Water Monitor
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
# Add src directory to Python path
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "src"))
|
||||
|
||||
if __name__ == "__main__":
|
||||
from src.main import main
|
||||
|
||||
main()
|
||||
+81
-125
@@ -1,29 +1,24 @@
|
||||
# Rust services
|
||||
# Rust: the service
|
||||
|
||||
Rewrite of the Ping River Monitor in Rust, one service at a time. The plan,
|
||||
costs and the parity rules are in [docs/RUST_REWRITE.md](../docs/RUST_REWRITE.md).
|
||||
Since October 2026 the whole Ping River Monitor service is one Rust binary,
|
||||
`ping-collector`; Python is left with model training and evaluation
|
||||
(`src/ml`) and the archive backfills. How it got here, and the parity rules
|
||||
the rewrite followed: [docs/RUST_REWRITE.md](../docs/RUST_REWRITE.md).
|
||||
|
||||
| Crate | Replaces | Status |
|
||||
| --- | --- | --- |
|
||||
| `collector` (`ping-collector`) | RID, HII/ThaiWater, RID dams and Open-Meteo collection in the Python web app | phase 1: in production since 2026-10-17 |
|
||||
| `forecast` (`ping-forecast`, library) | the flood forecast (`src/ml` serving path); run by `ping-collector` with ntfy | phase 2: in production since 2026-10-18 |
|
||||
| `collector`: `src/web` | the FastAPI app (`src/web_api.py`): dashboard, read API, `/docs` | phase 3: in production since 2026-10-18 |
|
||||
| Crate | What |
|
||||
| --- | --- |
|
||||
| `collector` (`ping-collector`) | collection (RID, HII/ThaiWater, RID dams, Open-Meteo), the forecast run, ntfy + Matrix notifications, the website and API (`collector/src/web`) |
|
||||
| `forecast` (`ping-forecast`, library) | flood-forecast inference from the models trained in Python: features, HII gap fill, gradient-boosted trees, heuristic fallback |
|
||||
|
||||
## ping-collector
|
||||
|
||||
Collects into the same PostgreSQL tables as the Python app, so the two can be
|
||||
swapped (and swapped back) with no data migration. The Python web app keeps
|
||||
serving the API, forecasting and sending ntfy alerts with
|
||||
`COLLECTION_ENABLED=false`, and re-forecasts whenever the newest RID reading in
|
||||
the database advances.
|
||||
|
||||
```text
|
||||
ping-collector run # the service: startup gap fill, RID at :05 (+10 min retries
|
||||
# while RID is behind), HII/dams/rain once per clock hour;
|
||||
# with FORECAST_IN_COLLECTOR=true also forecast + ntfy/Matrix;
|
||||
# with WEB_LISTEN the website and API
|
||||
ping-collector run # the service (systemd): startup gap fill, RID at :05 (+10 min
|
||||
# retries while RID is behind), HII/dams/rain once per clock
|
||||
# hour; FORECAST_IN_COLLECTOR=true: forecast + ntfy/Matrix
|
||||
# after each cycle with newer readings; WEB_LISTEN: website/API
|
||||
ping-collector serve [--listen A] [--no-analytics]
|
||||
# the website and API only (default 127.0.0.1:8001)
|
||||
# the website and API alone (default 127.0.0.1:8001)
|
||||
ping-collector once [--dry-run] # one cycle of every source; --dry-run writes nothing
|
||||
# and compares the RID fetch with the database
|
||||
ping-collector fill-gaps <days|all> # re-fetch RID reports with holes
|
||||
@@ -33,131 +28,98 @@ ping-collector forecast [--json] [--save] [--now T] [--rain-file F]
|
||||
ping-collector notify --dry-run # the readings the notifier judges, and what it would send
|
||||
```
|
||||
|
||||
Configuration is the Python app's `.env` (read from the working directory, or
|
||||
`--env-file`) with python-dotenv's rules, verified on 614 generated inputs
|
||||
(`collector/testdata/`): an unparsable line is skipped with a warning giving
|
||||
only its line number, never its value. Variables already set (e.g. by systemd's
|
||||
`EnvironmentFile=`) win. Used keys:
|
||||
`POSTGRES_CONNECTION_STRING` (or `POSTGRES_HOST/PORT/DB/USER/PASSWORD`),
|
||||
`RID_POLL_OFFSET_MINUTES` (5), `RID_RETRY_MINUTES` (10),
|
||||
`GAP_FILL_DAYS_ON_START` (14), `ENABLE_HII_COLLECTION`,
|
||||
`ENABLE_RESERVOIR_COLLECTION`, `ENABLE_RAIN_COLLECTION`, `HII_BASIN_CODE` (6),
|
||||
`STATION_CONFIG_PATH` (falls back to the bundled `src/data/stations.json`),
|
||||
`SCRAPING_STATUS_FILE` (status JSON for `/scraping/status` when the website runs elsewhere),
|
||||
`LOG_LEVEL` / `RUST_LOG`; for phase 2 `FORECAST_IN_COLLECTOR` (false),
|
||||
`MODELS_DIR` (`models`), `NTFY_SERVER` / `NTFY_PUBLISH_URL` / `NTFY_TOPIC_PREFIX`
|
||||
/ `NTFY_TOKEN` / `PUBLIC_URL` (as the Python app), `RID_STALE_ALERT_HOURS` (18,
|
||||
0 = off). All timestamps are Asia/Bangkok wall-clock, whatever the server's
|
||||
time zone.
|
||||
Configuration: the environment, or `.env` in the working directory
|
||||
(`--env-file` for another), every key with its default in
|
||||
[.env.example](../.env.example). `.env` is parsed with python-dotenv's rules
|
||||
(`collector/src/envfile.rs`, verified on 614 generated inputs) because the
|
||||
Python tools read the same file: an unparsable line is skipped with a warning
|
||||
giving only its line number, never its value. Variables already set (e.g. by
|
||||
systemd's `EnvironmentFile=`) win. All timestamps are Asia/Bangkok wall-clock,
|
||||
whatever the server's time zone.
|
||||
|
||||
Not ported: the one-off archive backfills (`scripts/backfill_*.py`), which
|
||||
stay in Python.
|
||||
### Collection
|
||||
|
||||
### Verified
|
||||
The tables, upserts and parsers are the ones the Python collectors had (the
|
||||
DDL is in `collector/src/db.rs`; the Python backfill stores create the same).
|
||||
RID's evening report often lags 1-2 h: every poll stores every reading the
|
||||
report has, yesterday's report is re-fetched while it has holes, and the
|
||||
startup gap fill repairs the last `GAP_FILL_DAYS_ON_START` days.
|
||||
|
||||
- 20 tests: parsers, gap/schedule logic, and database tests against a real
|
||||
PostgreSQL (late reports, midnight recovery, upsert revisions, NUMERIC
|
||||
rounding, dam metadata preservation).
|
||||
- Live parity: the Python collectors and `ping-collector` run back-to-back
|
||||
against the real upstream APIs into two empty databases produced identical
|
||||
rows in all 9 tables (16 stations, 1,104 RID readings, 103 + 76 HII rows,
|
||||
35 dams x 2 days, 168 rain hours). Rerun with
|
||||
`scripts/collector_parity.py` after changing a parser, table or upsert on
|
||||
either side.
|
||||
- The Python API with `COLLECTION_ENABLED=false` served `/health`,
|
||||
`/measurements/latest`, `/api/stats`, `/scraping/status` and model
|
||||
forecasts from a database filled only by `ping-collector`.
|
||||
### Forecast and notifications
|
||||
|
||||
## Forecast and notifications (phase 2)
|
||||
After every cycle that brought newer RID or HII readings (and at least
|
||||
hourly) the forecast runs over the last 336 h, is stored in `forecast_history`
|
||||
(for skill tracking) and `forecast_current`, and the notifier compares the
|
||||
readings and forecasts with `notification_state`.
|
||||
|
||||
With `FORECAST_IN_COLLECTOR=true` in the shared `.env`, `ping-collector` runs
|
||||
the flood forecast after every cycle that brought newer RID or HII readings
|
||||
(and at least hourly), stores it in `forecast_history` and `forecast_current`
|
||||
(the Python API serves the latter as `/forecast`, falling back to computing
|
||||
itself when it is older than `FORECAST_MAX_AGE_SECONDS`), and sends the ntfy
|
||||
notifications. The Python app then neither forecasts nor notifies.
|
||||
|
||||
- **Models** stay trained in Python. `src/ml/export.py` writes
|
||||
- **Models** are trained in Python. `src/ml/export.py` writes
|
||||
`models/flood_<station>.json.gz` next to each joblib bundle (`train_all()`
|
||||
does it, `retrain.sh` promotes it, `python -m src.ml.export` exports
|
||||
existing bundles); the collector reloads changed files on its next run.
|
||||
- **Shared constants** (thresholds, flood stages, upstream leads, notify
|
||||
does it; `python -m src.ml.export` for existing bundles); the service
|
||||
reloads changed files on its next run, so a retrain needs no restart.
|
||||
- **Shared constants** (thresholds, flood stages, upstream leads, notifier
|
||||
settings) are compiled in from `forecast/flood_config.json`, generated by
|
||||
`uv run python -m src.ml.export --config`. `tests/test_model_export.py`
|
||||
fails while it differs from Python, and a change needs a collector rebuild.
|
||||
- **Notifications**: same topics, texts, hysteresis and `notification_state`
|
||||
keys as `src/notify.py`, so switching sends nothing twice. One change: for
|
||||
the stations whose ThaiWater twin is the same sensor, a reading newer than
|
||||
RID's (RID's report lags 2-10 h at night) drives the level alerts, marked
|
||||
"ThaiWater telemetry; the RID report is not in yet". "Gauge feed stale" now
|
||||
means no source has a reading for 3 h; RID alone lagging is reported once
|
||||
after `RID_STALE_ALERT_HOURS`.
|
||||
- Memory: 70 MB peak with all 15 models loaded (24-byte tree nodes).
|
||||
fails while it differs from Python; a change needs a rebuild.
|
||||
- **ntfy** topics, texts and hysteresis: [docs/NOTIFICATIONS.md](../docs/NOTIFICATIONS.md).
|
||||
For the stations whose ThaiWater twin is the same sensor, a reading newer
|
||||
than RID's (RID lags 2-10 h at night) drives the level alerts, marked as
|
||||
ThaiWater telemetry. "Gauge feed stale" means no source has a reading for
|
||||
3 h; RID alone lagging is reported once after `RID_STALE_ALERT_HOURS`.
|
||||
- **Matrix** (`MATRIX_*`): the room gets the same transitions (per station,
|
||||
outlook, feed; not ntfy's per-topic summaries), each once, tracked under
|
||||
its own `matrix:` keys in `notification_state`, so a Matrix outage is
|
||||
retried on Matrix only. [docs/MATRIX_QUICK_START.md](../docs/MATRIX_QUICK_START.md).
|
||||
- Memory: under 100 MB resident in production, all 15 models loaded
|
||||
(24-byte tree nodes); the unit allows 512 MB.
|
||||
|
||||
Verified:
|
||||
### Website and API
|
||||
|
||||
- `forecast/tests/parity.rs` replays fixtures from the Python production path
|
||||
(`forecast/testdata/gen_forecast_fixtures.py`: the 2024 flood rising and at
|
||||
its peak with RID lagging behind HII, a quiet window with dam data, a
|
||||
stale-feed window, moved thresholds). HII fill, every feature and every
|
||||
level are bit-identical; with all 15 local models 377 of 384 probabilities
|
||||
are bit-identical and the rest within 2 ulp (`exp`). CI checks the P.1 model.
|
||||
- `scripts/forecast_parity.py` does the same on the live database: Python and
|
||||
`ping-collector forecast --json` at the same moment with the same rain
|
||||
series. The install script runs it before `--cutover-forecast`.
|
||||
- `collector/testdata/gen_notify_cases.py`: 652 hourly steps through
|
||||
`src/notify.py` (thresholds, capacity guard, outlook, feed gaps, ntfy
|
||||
outages); the Rust notifier sends the same 751 messages word for word.
|
||||
|
||||
## Website and API (phase 3)
|
||||
|
||||
`src/web` serves what the FastAPI app served: the dashboard
|
||||
(`STATIC_DIR`/dashboard.html, default `src/static`, loaded at start), the
|
||||
crawler files, `/static`, `/docs` + `/redoc` + `/openapi.json`, and every API
|
||||
route with the same parameters, JSON (field order included), status codes,
|
||||
`{"detail": ...}` errors, Cache-Control, gzip, CORS (`CORS_ALLOW_ORIGINS`),
|
||||
Umami API tracking (`UMAMI_*`) and response caches (`*_CACHE_TTL_SECONDS`).
|
||||
Admin routes need `ADMIN_API_KEY`. With `WEB_LISTEN=0.0.0.0:8000` it runs
|
||||
inside `ping-collector run` (one process collects, forecasts, notifies and
|
||||
serves; `/scraping/status` and `POST /scrape/trigger` then talk to the
|
||||
collection loop directly); `serve` runs it alone.
|
||||
`collector/src/web` (axum) serves the dashboard (`STATIC_DIR`/dashboard.html,
|
||||
default `src/static`, loaded at start), `/static`, the crawler files,
|
||||
`/docs` + `/redoc` + `/openapi.json`, and the read API, with gzip,
|
||||
Cache-Control per path, CORS (`CORS_ALLOW_ORIGINS`), Umami API tracking
|
||||
(`UMAMI_*`) and stale-while-revalidate response caches with byte budgets.
|
||||
Station editing, `/config` and `POST /scrape/trigger` need `ADMIN_API_KEY`
|
||||
(`X-API-Key` header).
|
||||
|
||||
- Parameters are declared once per route (`web/params.rs`): the same list
|
||||
validates the request (pydantic-style 422s) and builds the OpenAPI
|
||||
document (`web/openapi.rs`), so `/docs` cannot drift from what is accepted.
|
||||
The component schemas are FastAPI's (`web/openapi_components.json`).
|
||||
- Python's output conventions are kept where they show: `isoformat()`
|
||||
timestamps, `round()`, numpy's pairwise sums for the catchment means
|
||||
(`web/pyfmt.rs`), the skill metrics' summation order.
|
||||
- `scripts/api_parity.py --python URL --rust URL` compares the two live on
|
||||
one database; `install_collector.sh --check-api` / `--cutover-api` run it
|
||||
on the server against `ping-collector serve` on port 8001.
|
||||
- The response formats are the retired FastAPI app's, kept so clients did
|
||||
not notice the switch: field order, `isoformat()` timestamps, Python's
|
||||
`round()`, numpy's pairwise sums for the catchment means (`web/pyfmt.rs`),
|
||||
`{"detail": ...}` errors.
|
||||
- An old stored forecast is served marked `X-Data-Stale: true`; `/health`
|
||||
counts current ThaiWater telemetry of the same gauges while RID lags.
|
||||
|
||||
Matrix (`MATRIX_HOMESERVER`, `MATRIX_ACCESS_TOKEN`, `MATRIX_ROOM_ID`): the
|
||||
notifier sends the room the same transitions as ntfy (per station, outlook,
|
||||
feed; not ntfy's per-topic summaries), each once, with its own `matrix:` keys
|
||||
in `notification_state` (seeded from the ntfy ones on first use, so nothing
|
||||
already announced is repeated). It replaces the `run.py --alert-check` cron
|
||||
job, which the install script comments out.
|
||||
|
||||
### Develop
|
||||
## Develop
|
||||
|
||||
```bash
|
||||
cd rust
|
||||
cargo fmt --all && cargo clippy --all-targets -- -D warnings
|
||||
docker run -d --name ping-pg -e POSTGRES_PASSWORD=ping -e POSTGRES_DB=ping -p 55432:5432 postgres:16
|
||||
TEST_DATABASE_URL=postgres://postgres:ping@localhost:55432/ping cargo test
|
||||
CARGO_TARGET_DIR=target/msrv cargo +1.88 check --all-targets --locked # the minimum version
|
||||
```
|
||||
|
||||
Without `TEST_DATABASE_URL` the database tests skip. CI
|
||||
(`.gitea/workflows/rust.yml`) runs them against a PostgreSQL service and
|
||||
uploads the Linux binary as the `ping-collector-linux-x86_64` artifact.
|
||||
|
||||
- The forecast's parity with the Python reference: `forecast/tests/parity.rs`
|
||||
replays fixtures from the Python production path
|
||||
(`forecast/testdata/gen_forecast_fixtures.py`; regenerate after a change to
|
||||
`src/ml/features.py`, `predict.py`, the HII fill or the dam frame). Features
|
||||
and levels must stay bit-identical, probabilities within a few ulp.
|
||||
`scripts/forecast_parity.py` compares the two on a live database.
|
||||
- The notifier's texts are pinned by `collector/testdata/notify_python_cases.json.gz`
|
||||
(751 messages recorded from the Python notifier before it was retired).
|
||||
|
||||
HTTP uses native-tls (OpenSSL on Linux), not rustls: app.rid.go.th (dams)
|
||||
only offers TLS 1.2 CBC cipher suites, which rustls refuses. Building on
|
||||
Linux needs `pkg-config` and `libssl-dev`.
|
||||
|
||||
### Deploy
|
||||
## Deploy
|
||||
|
||||
Needs Rust >= 1.88. Distro `cargo` packages are older (`resolver 3 is not
|
||||
valid` / `edition 2024` errors), so install the toolchain once with rustup:
|
||||
@@ -169,21 +131,15 @@ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profil
|
||||
cargo --version # >= 1.88
|
||||
```
|
||||
|
||||
Then on the server, after `git pull` (the Python side of the switch ships in
|
||||
the same commit):
|
||||
Then on the server, in the checkout:
|
||||
|
||||
```bash
|
||||
git pull
|
||||
cargo build --release --locked --manifest-path rust/Cargo.toml # or BIN=<CI artifact>
|
||||
sudo bash scripts/install_collector.sh # install + checks, nothing switched
|
||||
sudo bash scripts/install_collector.sh --cutover # Rust collects from now on
|
||||
sudo bash scripts/install_collector.sh --cutover-forecast # Rust also forecasts and notifies
|
||||
sudo bash scripts/install_collector.sh --check-api # compare the Rust API with FastAPI
|
||||
sudo bash scripts/install_collector.sh --cutover-api # Rust serves the website on :8000
|
||||
sudo bash scripts/install_collector.sh --rollback-api # FastAPI serves the website again
|
||||
sudo bash scripts/install_collector.sh --rollback-forecast # forecasts + ntfy back to Python
|
||||
sudo bash scripts/install_collector.sh --rollback # Python does everything again
|
||||
sudo bash scripts/install.sh
|
||||
journalctl -u ping-collector -f
|
||||
```
|
||||
|
||||
Every install exports the models and runs `scripts/forecast_parity.py`;
|
||||
`--cutover-forecast` refuses to switch if it fails (`FORCE=1` overrides).
|
||||
`scripts/install.sh` installs the binary and the units, syncs the Python
|
||||
training environment, runs a dry run of every source and the forecast check,
|
||||
restarts the service and waits for the website (details in its header).
|
||||
@@ -1,5 +1,7 @@
|
||||
//! Configuration from the same environment / `.env` file as the Python app,
|
||||
//! so both services can share `/opt/thailand-water-monitor/.env`.
|
||||
//! Configuration from the environment / `.env` file
|
||||
//! (`/opt/thailand-water-monitor/.env`), which the Python training tools
|
||||
//! (src/config.py) read too. Names and defaults are the ones the Python app
|
||||
//! used.
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use std::env;
|
||||
@@ -33,10 +35,10 @@ pub struct Config {
|
||||
/// Days of RID holes re-fetched at startup (0 = off).
|
||||
pub gap_fill_days_on_start: u32,
|
||||
pub station_config_path: PathBuf,
|
||||
/// Status JSON read by the Python API's /scraping/status.
|
||||
/// Status JSON for /scraping/status when the website runs in a separate
|
||||
/// process (`ping-collector serve`).
|
||||
pub status_file: Option<PathBuf>,
|
||||
/// FORECAST_IN_COLLECTOR: forecast and notify here; the Python API then
|
||||
/// serves the stored forecast and does neither.
|
||||
/// FORECAST_IN_COLLECTOR: forecast and notify (off: collect only).
|
||||
pub forecast_enabled: bool,
|
||||
/// Exported bundles (`flood_<station>.json.gz`, src/ml/export.py).
|
||||
pub models_dir: PathBuf,
|
||||
@@ -62,7 +64,7 @@ pub struct MatrixConfig {
|
||||
pub room_id: String,
|
||||
}
|
||||
|
||||
/// The website and read API (src/web_api.py's settings, same names).
|
||||
/// The website and read API (setting names as the retired Python app had them).
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct WebConfig {
|
||||
/// WEB_LISTEN: `run` also serves the website on this address
|
||||
|
||||
@@ -30,8 +30,9 @@ pub async fn connect(url: &str, max_connections: u32) -> Result<PgPool> {
|
||||
.context("connecting to PostgreSQL")
|
||||
}
|
||||
|
||||
/// Idempotent DDL, identical to what the Python app creates (so either side
|
||||
/// can start first against an empty database).
|
||||
/// Idempotent DDL, identical to what the Python app created and the Python
|
||||
/// backfill stores (src/hii_collector.py, src/rid_reservoir.py) still create,
|
||||
/// so either can start first against an empty database.
|
||||
const SCHEMA: &[&str] = &[
|
||||
"CREATE TABLE IF NOT EXISTS stations (
|
||||
id SERIAL PRIMARY KEY,
|
||||
@@ -145,12 +146,12 @@ const SCHEMA: &[&str] = &[
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (as_of, station_code, horizon_hours))",
|
||||
"CREATE INDEX IF NOT EXISTS idx_forecast_history_station ON forecast_history(station_code, as_of)",
|
||||
// src/notify.py
|
||||
// notify.rs
|
||||
"CREATE TABLE IF NOT EXISTS notification_state (
|
||||
key VARCHAR(64) PRIMARY KEY, state VARCHAR(16) NOT NULL,
|
||||
value NUMERIC(8,3), updated_at TIMESTAMP NOT NULL)",
|
||||
// The current /forecast payload, read by the Python API when
|
||||
// FORECAST_IN_COLLECTOR=true (one row, id 'all').
|
||||
// The current /forecast payload (one row, id 'all'), served by
|
||||
// `ping-collector serve` when the website runs as its own process.
|
||||
"CREATE TABLE IF NOT EXISTS forecast_current (
|
||||
id VARCHAR(16) PRIMARY KEY,
|
||||
computed_at TIMESTAMP NOT NULL,
|
||||
@@ -727,7 +728,7 @@ pub async fn save_forecast_history(
|
||||
Ok(rows.len())
|
||||
}
|
||||
|
||||
/// The payload the Python API serves as /forecast.
|
||||
/// The payload /forecast serves (see `forecast_current` above).
|
||||
pub async fn save_forecast_current(
|
||||
pool: &PgPool,
|
||||
computed_at: NaiveDateTime,
|
||||
|
||||
@@ -2,7 +2,8 @@
|
||||
//! interpolate `${VAR}` / `${VAR:-default}`, never override variables that
|
||||
//! are already set).
|
||||
//!
|
||||
//! The Python app reads the same file, so the two must agree on every value.
|
||||
//! The Python tools (src/config.py) read the same file, so the two must agree
|
||||
//! on every value.
|
||||
//! dotenvy (the usual crate) is stricter: a stray `'` or `"` inside an
|
||||
//! unquoted value makes it swallow the rest of the file and abort, and its
|
||||
//! error message echoes the line, password included. Here an unparsable
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
//! Forecasting and notifications in the collector (FORECAST_IN_COLLECTOR=true),
|
||||
//! replacing the Python API's forecast_notify_task.
|
||||
//! Forecasting and notifications in the collector (FORECAST_IN_COLLECTOR=true);
|
||||
//! the port of the retired Python app's forecast_notify_task.
|
||||
//!
|
||||
//! After each collection cycle that brought newer RID or HII readings, and at
|
||||
//! least once per clock hour: load the last 336 h (RID plus the HII mirror
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
//! ping-collector: the Ping River Monitor's data collection service.
|
||||
//! ping-collector: the Ping River Monitor service.
|
||||
//!
|
||||
//! Replaces the collection half of the Python web app (RID hourly gauges,
|
||||
//! HII/ThaiWater rain + water level, RID dams, Open-Meteo rain) and, with
|
||||
//! FORECAST_IN_COLLECTOR=true, the flood forecast (rust/forecast) and the
|
||||
//! ntfy/Matrix notifications, and with WEB_LISTEN the website and read API
|
||||
//! (the port of the FastAPI app). Writes the same PostgreSQL tables as the
|
||||
//! Python app did. See docs/RUST_REWRITE.md.
|
||||
//! Collects RID hourly gauges, HII/ThaiWater rain + water level, RID dams
|
||||
//! and Open-Meteo rain into PostgreSQL; with FORECAST_IN_COLLECTOR=true runs
|
||||
//! the flood forecast (rust/forecast, models trained in Python) and the
|
||||
//! ntfy/Matrix notifications; with WEB_LISTEN serves the website and API.
|
||||
//! It replaced the Python app in 2026-10 (same tables, routes and messages);
|
||||
//! see docs/RUST_REWRITE.md.
|
||||
|
||||
mod config;
|
||||
mod dams;
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
//! Public flood notifications over ntfy, and the same transitions to a
|
||||
//! Matrix room: the port of src/notify.py, run by the collector when
|
||||
//! FORECAST_IN_COLLECTOR=true. (Matrix replaces the old `run.py
|
||||
//! --alert-check` cron job, which repeated an alert on every run while a
|
||||
//! level stayed high.)
|
||||
//! Matrix room, run by the collector when FORECAST_IN_COLLECTOR=true. A port
|
||||
//! of the retired Python notifier (src/notify.py, removed after the 2026-10
|
||||
//! cutover); its settings still come from src/ml/export.py via
|
||||
//! flood_config.json. (Matrix replaced the old `run.py --alert-check` cron
|
||||
//! job, which repeated an alert on every run while a level stayed high.)
|
||||
//!
|
||||
//! Same topics, texts, hysteresis, capacity guard, outlook and persisted
|
||||
//! state keys (`notification_state`) as Python, so a cutover neither repeats
|
||||
//! nor drops a message. One change: for stations whose HII/ThaiWater twin is
|
||||
//! state keys (`notification_state`) as Python had, so the cutover neither
|
||||
//! repeated nor dropped a message. One change: for stations whose HII/ThaiWater twin is
|
||||
//! the same sensor, a reading newer than RID's (RID's report often lags 2-10 h
|
||||
//! at night) is used for the level alerts and the feed check. The feed is
|
||||
//! "stale" only when neither source has a reading for 3 h; RID lagging alone
|
||||
@@ -1011,8 +1012,10 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// 652 hourly steps from src/notify.py itself (testdata/gen_notify_cases.py):
|
||||
/// the same notifications, word for word, in the same order.
|
||||
/// 652 hourly steps recorded from the retired Python notifier (generator:
|
||||
/// testdata/gen_notify_cases.py, in git history before the cleanup): the
|
||||
/// same notifications, word for word, in the same order. A deliberate
|
||||
/// text change updates the fixture with the new expected messages.
|
||||
#[test]
|
||||
fn matches_python_notify_on_generated_sequences() {
|
||||
use std::io::Read;
|
||||
|
||||
@@ -403,7 +403,7 @@ impl Collector {
|
||||
pub async fn notify(&self, dry_run: bool) -> Result<()> {
|
||||
if !dry_run && !self.cfg.forecast_enabled {
|
||||
bail!(
|
||||
"the Python app notifies while FORECAST_IN_COLLECTOR is not true; \
|
||||
"notifications are off while FORECAST_IN_COLLECTOR is not true; \
|
||||
use --dry-run"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
//! Stale-while-revalidate response cache, as `_cached_swr` in src/web_api.py:
|
||||
//! Stale-while-revalidate response cache, as the retired FastAPI app's
|
||||
//! `_cached_swr`:
|
||||
//!
|
||||
//! - fresh entry: served;
|
||||
//! - expired entry (no older than `max_stale`): served at once, marked stale,
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
//! The website and read API: the port of the FastAPI app (src/web_api.py).
|
||||
//! The website and read API: the port of the retired FastAPI app.
|
||||
//!
|
||||
//! Same routes, parameters, JSON fields, status codes, cache headers and
|
||||
//! response caches as the Python app, checked route by route against it by
|
||||
//! scripts/api_parity.py. It runs inside `ping-collector run` (WEB_LISTEN)
|
||||
//! response caches as the Python app had; before the cutover (2026-10-18) a
|
||||
//! route-by-route comparison of the two on production data found no
|
||||
//! difference. It runs inside `ping-collector run` (WEB_LISTEN)
|
||||
//! or alone (`ping-collector serve`). /docs is the same Swagger UI, fed by an
|
||||
//! OpenAPI document built from the route table that also validates the
|
||||
//! parameters (openapi.rs, params.rs).
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
//! Route tests: the router as the server runs it, against a throwaway
|
||||
//! schema (TEST_DATABASE_URL; see db_tests.rs) or, for the routes that do
|
||||
//! not touch the database, a pool that never connects. Output formats are
|
||||
//! the FastAPI app's (scripts/api_parity.py compares the two live).
|
||||
//! the retired FastAPI app's (field order, isoformat, errors).
|
||||
|
||||
use super::{AppState, router};
|
||||
use crate::config::Config;
|
||||
|
||||
-127
@@ -1,127 +0,0 @@
|
||||
"""Notification cases for rust/collector/src/notify.rs, from src/notify.py itself.
|
||||
|
||||
uv run python rust/collector/testdata/gen_notify_cases.py
|
||||
|
||||
Random hourly sequences around the thresholds (capacity guard, hysteresis,
|
||||
danger escalation, the P.1 outlook, feed gaps, ntfy outages for some topics)
|
||||
run through notify.evaluate with an in-memory state; every delivered
|
||||
notification is recorded. The Rust test replays the same steps (RID readings
|
||||
only, so the ThaiWater additions stay out of play) and must deliver exactly
|
||||
the same messages.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import gzip
|
||||
import json
|
||||
import random
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE.parents[2]))
|
||||
|
||||
from src import notify # noqa: E402
|
||||
|
||||
|
||||
class Publisher(notify.NtfyPublisher):
|
||||
def __init__(self):
|
||||
super().__init__("http://ntfy.invalid", prefix="ping")
|
||||
self.down = set()
|
||||
|
||||
def publish(self, n):
|
||||
return n.topic not in self.down
|
||||
|
||||
|
||||
STATIONS = { # around (warning, danger)
|
||||
"P.1": (3.7, 4.2),
|
||||
"P.77": (4.3, 4.9),
|
||||
"P.21": (3.2, 3.6),
|
||||
"P.20": (2.35, 2.8),
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
rng = random.Random(20261018)
|
||||
sequences = []
|
||||
for _ in range(40):
|
||||
state = notify.InMemoryState()
|
||||
pub = Publisher()
|
||||
t = datetime.datetime(2026, 9, 1) + datetime.timedelta(
|
||||
hours=rng.randrange(2000)
|
||||
)
|
||||
codes = rng.sample(sorted(STATIONS), rng.randint(1, 4))
|
||||
levels = {c: STATIONS[c][0] + rng.uniform(-0.6, 0.6) for c in codes}
|
||||
p_out = rng.random()
|
||||
steps = []
|
||||
for _ in range(rng.randint(8, 25)):
|
||||
gap = rng.choice([1, 1, 1, 1, 2, 4, 6])
|
||||
t += datetime.timedelta(hours=gap)
|
||||
# the clock sometimes runs ahead of the data (a stale feed)
|
||||
now = t + datetime.timedelta(minutes=rng.choice([5, 5, 30, 200, 300]))
|
||||
readings = []
|
||||
for c in codes:
|
||||
levels[c] += rng.uniform(-0.25, 0.25)
|
||||
pct = rng.choice([None, rng.uniform(5, 120)])
|
||||
level = round(levels[c], 2) if rng.random() > 0.05 else None
|
||||
readings.append(
|
||||
{
|
||||
"station_code": c,
|
||||
"timestamp": t,
|
||||
"water_level": level,
|
||||
"discharge_percent": None if pct is None else round(pct, 2),
|
||||
}
|
||||
)
|
||||
p_out = min(1.0, max(0.0, p_out + rng.uniform(-0.3, 0.3)))
|
||||
forecasts = [
|
||||
{
|
||||
"station_code": "P.1",
|
||||
"horizon_hours": rng.choice([24, 24, 24, 12]),
|
||||
"p_warning": p_out,
|
||||
"predicted_max_level": round(rng.uniform(2.5, 5.0), 3),
|
||||
"source": rng.choice(["model", "model", "model", "heuristic"]),
|
||||
}
|
||||
]
|
||||
pub.down = set(
|
||||
rng.sample(
|
||||
[
|
||||
"ping-p1-warning",
|
||||
"ping-warning",
|
||||
"ping-status",
|
||||
"ping-p1-outlook",
|
||||
],
|
||||
rng.choice([0, 0, 0, 1]),
|
||||
)
|
||||
)
|
||||
sent = notify.evaluate(readings, forecasts, state, pub, now=now)
|
||||
steps.append(
|
||||
{
|
||||
"now": now.isoformat(),
|
||||
"readings": [
|
||||
{**r, "timestamp": r["timestamp"].isoformat()} for r in readings
|
||||
],
|
||||
"forecasts": forecasts,
|
||||
"down": sorted(pub.down),
|
||||
"sent": [
|
||||
{
|
||||
"topic": n.topic,
|
||||
"title": n.title,
|
||||
"message": n.message,
|
||||
"priority": n.priority,
|
||||
"tags": n.tags or [],
|
||||
}
|
||||
for n in sent
|
||||
],
|
||||
}
|
||||
)
|
||||
sequences.append(steps)
|
||||
out = HERE / "notify_python_cases.json.gz"
|
||||
with open(out, "wb") as raw, gzip.GzipFile(fileobj=raw, mode="wb", mtime=0) as gz:
|
||||
gz.write(json.dumps({"sequences": sequences}).encode())
|
||||
total = sum(len(s["sent"]) for seq in sequences for s in seq)
|
||||
print(
|
||||
f"{len(sequences)} sequences, {sum(map(len, sequences))} steps, {total} notifications"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,389 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare the Rust website/API with the FastAPI app, route by route.
|
||||
|
||||
Phase 3 of docs/RUST_REWRITE.md: before the Rust service takes over port
|
||||
8000, both answer the same requests against the same database and every
|
||||
response is compared: status, content type, Cache-Control and the JSON body
|
||||
(floats exactly; lists of database rows without ORDER BY as sets). Static
|
||||
files and the dashboard byte for byte.
|
||||
|
||||
# the FastAPI app on :8000, `ping-collector serve --listen 127.0.0.1:8001`
|
||||
uv run python scripts/api_parity.py \\
|
||||
--python http://127.0.0.1:8000 --rust http://127.0.0.1:8001
|
||||
|
||||
Query parameters vary per run (limits, hours) so the FastAPI app's response
|
||||
caches (up to 15 min) answer with fresh data, not an older copy. Data the
|
||||
collector writes during the run can still make a route differ once: a
|
||||
mismatch is retried after a pause before it counts.
|
||||
|
||||
Intended differences (the Rust side may answer differently):
|
||||
/health a lagging RID report with current ThaiWater telemetry for
|
||||
the same gauges is healthy (as the notifier, phase 2)
|
||||
/scraping/status extra fields (collector version, forecast run)
|
||||
/metrics process counters: compared by shape only
|
||||
/sensors/thaiwater live upstream: compared, a difference only warns
|
||||
/api/hii/rainfall/catchment correlation within 1e-9 (numpy uses BLAS)
|
||||
/api/stats newer counts on the Rust side are accepted (the FastAPI app
|
||||
caches this one for 5 minutes)
|
||||
*.geojson content type application/geo+json (FastAPI: octet-stream)
|
||||
|
||||
Exit status 0 when every hard check passed.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import random
|
||||
import sys
|
||||
import time
|
||||
from typing import Any, Callable, List, Optional, Tuple
|
||||
|
||||
import requests
|
||||
|
||||
SOFT = "soft" # a difference warns, does not fail
|
||||
|
||||
|
||||
def get(base: str, path: str, method: str = "GET", **kw) -> requests.Response:
|
||||
last = None
|
||||
for _ in range(3):
|
||||
try:
|
||||
return requests.request(method, base + path, timeout=120, **kw)
|
||||
except requests.RequestException as e: # connection reset etc.
|
||||
last = e
|
||||
time.sleep(1)
|
||||
raise last
|
||||
|
||||
|
||||
def first_diff(a: Any, b: Any, path: str = "$", rel: float = 0.0) -> Optional[str]:
|
||||
"""Where two decoded JSON values differ (None if equal)."""
|
||||
if isinstance(a, bool) or isinstance(b, bool):
|
||||
return None if a is b else f"{path}: {a!r} != {b!r}"
|
||||
if isinstance(a, (int, float)) and isinstance(b, (int, float)):
|
||||
if a == b:
|
||||
return None
|
||||
if rel and abs(a - b) <= rel * max(abs(a), abs(b)):
|
||||
return None
|
||||
return f"{path}: {a!r} != {b!r}"
|
||||
if type(a) is not type(b):
|
||||
return f"{path}: {type(a).__name__} {a!r} != {type(b).__name__} {b!r}"
|
||||
if isinstance(a, dict):
|
||||
if list(a.keys()) != list(b.keys()):
|
||||
if set(a.keys()) != set(b.keys()):
|
||||
return f"{path}: keys {sorted(set(a) ^ set(b))} only on one side"
|
||||
for k in a:
|
||||
d = first_diff(a[k], b[k], f"{path}.{k}", rel)
|
||||
if d:
|
||||
return d
|
||||
return None
|
||||
if isinstance(a, list):
|
||||
if len(a) != len(b):
|
||||
return f"{path}: {len(a)} items != {len(b)} items"
|
||||
for i, (x, y) in enumerate(zip(a, b)):
|
||||
d = first_diff(x, y, f"{path}[{i}]", rel)
|
||||
if d:
|
||||
return d
|
||||
return None
|
||||
return None if a == b else f"{path}: {a!r} != {b!r}"
|
||||
|
||||
|
||||
def media_type(r: requests.Response) -> str:
|
||||
return r.headers.get("content-type", "").split(";")[0].strip()
|
||||
|
||||
|
||||
class Case:
|
||||
def __init__(
|
||||
self,
|
||||
path: str,
|
||||
method: str = "GET",
|
||||
compare: Optional[Callable] = None,
|
||||
sort_key: Optional[str] = None,
|
||||
soft: bool = False,
|
||||
body: Any = None,
|
||||
headers: Optional[dict] = None,
|
||||
):
|
||||
self.path, self.method, self.compare = path, method, compare
|
||||
self.sort_key, self.soft, self.body = sort_key, soft, body
|
||||
self.headers = headers or {}
|
||||
|
||||
def run(self, py: str, rs: str) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""(hard difference, warning)"""
|
||||
kw = {"headers": self.headers}
|
||||
if self.body is not None:
|
||||
kw["json"] = self.body
|
||||
a = get(py, self.path, self.method, **kw)
|
||||
b = get(rs, self.path, self.method, **kw)
|
||||
if a.status_code != b.status_code:
|
||||
return self._fail(
|
||||
f"status {a.status_code} != {b.status_code} "
|
||||
f"({a.text[:120]!r} / {b.text[:120]!r})"
|
||||
)
|
||||
if media_type(a) != media_type(b):
|
||||
warn = f"content-type {media_type(a)!r} != {media_type(b)!r}"
|
||||
if media_type(a).startswith("application/json") or media_type(b).startswith(
|
||||
"application/json"
|
||||
):
|
||||
return self._fail(warn)
|
||||
return None, warn
|
||||
if self.method == "GET" and a.status_code == 200:
|
||||
ca, cb = a.headers.get("cache-control"), b.headers.get("cache-control")
|
||||
if ca != cb:
|
||||
return self._fail(f"Cache-Control {ca!r} != {cb!r}")
|
||||
if self.compare is not None:
|
||||
d = self.compare(a, b)
|
||||
elif media_type(a) == "application/json":
|
||||
ja, jb = a.json(), b.json()
|
||||
if a.status_code == 422:
|
||||
# pydantic's messages are approximated; type and location must agree
|
||||
ja = [(e["type"], e["loc"]) for e in ja["detail"]]
|
||||
jb = [(e["type"], e["loc"]) for e in jb["detail"]]
|
||||
if any(t.startswith("date") for t, _ in ja):
|
||||
ja, jb = [loc for _, loc in ja], [loc for _, loc in jb]
|
||||
elif self.sort_key and isinstance(ja, list):
|
||||
ja = sorted(ja, key=lambda r: str(r.get(self.sort_key)))
|
||||
jb = sorted(jb, key=lambda r: str(r.get(self.sort_key)))
|
||||
d = first_diff(ja, jb)
|
||||
else:
|
||||
d = (
|
||||
None
|
||||
if a.content == b.content
|
||||
else f"bodies differ ({len(a.content)} / {len(b.content)} bytes)"
|
||||
)
|
||||
return self._fail(d) if d else (None, None)
|
||||
|
||||
def _fail(self, d: str) -> Tuple[Optional[str], Optional[str]]:
|
||||
return (None, d) if self.soft else (d, None)
|
||||
|
||||
|
||||
def compare_health(a, b) -> Optional[str]:
|
||||
ja, jb = a.json(), b.json()
|
||||
if set(ja["checks"]) != set(jb["checks"]):
|
||||
return f"checks {sorted(ja['checks'])} != {sorted(jb['checks'])}"
|
||||
for name, ca in ja["checks"].items():
|
||||
cb = jb["checks"][name]
|
||||
if ca["status"] == cb["status"]:
|
||||
continue
|
||||
# the intended change: RID lagging, ThaiWater current
|
||||
if (
|
||||
name == "database"
|
||||
and ca["status"] == "degraded"
|
||||
and cb["status"] == "healthy"
|
||||
and "ThaiWater telemetry current" in cb["message"]
|
||||
):
|
||||
continue
|
||||
if name in ("api", "memory"):
|
||||
continue # probes of the RID API / own process, compared loosely
|
||||
return f"{name}: {ca} != {cb}"
|
||||
return None
|
||||
|
||||
|
||||
def compare_metrics(a, b) -> Optional[str]:
|
||||
ja, jb = a.json(), b.json()
|
||||
return None if set(ja) == set(jb) else f"keys {sorted(ja)} != {sorted(jb)}"
|
||||
|
||||
|
||||
def compare_status(a, b) -> Optional[str]:
|
||||
ja, jb = a.json(), b.json()
|
||||
for k, v in ja.items():
|
||||
if jb.get(k) != v:
|
||||
return f"{k}: {v!r} != {jb.get(k)!r}"
|
||||
return None
|
||||
|
||||
|
||||
def compare_skill(a, b) -> Optional[str]:
|
||||
ja, jb = a.json(), b.json()
|
||||
va, vb = ja.pop("verified_until"), jb.pop("verified_until")
|
||||
gap = abs(
|
||||
time.mktime(time.strptime(va[:19], "%Y-%m-%dT%H:%M:%S"))
|
||||
- time.mktime(time.strptime(vb[:19], "%Y-%m-%dT%H:%M:%S"))
|
||||
)
|
||||
if gap > 300:
|
||||
return f"verified_until {va} != {vb}"
|
||||
return first_diff(ja, jb)
|
||||
|
||||
|
||||
def compare_stats(a, b) -> Optional[str]:
|
||||
"""Exact, except that the FastAPI app may answer from its 5-minute cache
|
||||
while the collector has written newer rows since."""
|
||||
ja, jb = a.json(), b.json()
|
||||
if ja == jb:
|
||||
return None
|
||||
for k, v in ja.items():
|
||||
w = jb.get(k)
|
||||
if k.endswith("_measurements") or k == "days_spanned":
|
||||
if not isinstance(w, int) or w < v:
|
||||
return f"{k}: {v!r} > {w!r}"
|
||||
elif k == "last_timestamp":
|
||||
if not isinstance(w, str) or w < v:
|
||||
return f"{k}: {v!r} > {w!r}"
|
||||
elif k == "coverage_percent":
|
||||
if not isinstance(w, (int, float)) or abs(w - v) > 0.2:
|
||||
return f"{k}: {v!r} != {w!r}"
|
||||
elif v != w:
|
||||
return f"{k}: {v!r} != {w!r}"
|
||||
return None
|
||||
|
||||
|
||||
def compare_catchment(a, b) -> Optional[str]:
|
||||
return first_diff(a.json(), b.json(), rel=1e-9)
|
||||
|
||||
|
||||
def compare_openapi(a, b) -> Optional[str]:
|
||||
ja, jb = a.json(), b.json()
|
||||
if ja["info"]["title"] != jb["info"]["title"]:
|
||||
return "title differs"
|
||||
ops = lambda spec: { # noqa: E731
|
||||
(m, p): op for p, item in spec["paths"].items() for m, op in item.items()
|
||||
}
|
||||
oa, ob = ops(ja), ops(jb)
|
||||
if set(oa) != set(ob):
|
||||
return f"operations only on one side: {sorted(set(oa) ^ set(ob))}"
|
||||
for k, op in oa.items():
|
||||
other = ob[k]
|
||||
for field in ("operationId", "summary", "parameters", "requestBody"):
|
||||
d = first_diff(op.get(field), other.get(field), f"{k}.{field}")
|
||||
if d:
|
||||
return d
|
||||
if set(op["responses"]) != set(other["responses"]):
|
||||
return f"{k}: responses {sorted(op['responses'])} != {sorted(other['responses'])}"
|
||||
for name in ja["components"]["schemas"]:
|
||||
if name not in jb["components"]["schemas"]:
|
||||
return f"schema {name} missing"
|
||||
return None
|
||||
|
||||
|
||||
def compare_html_docs(a, b) -> Optional[str]:
|
||||
return (
|
||||
None
|
||||
if "swagger-ui" in b.text and "/openapi.json" in b.text
|
||||
else "no Swagger UI"
|
||||
)
|
||||
|
||||
|
||||
def cases(admin_key: Optional[str]) -> List[Case]:
|
||||
r = random.Random(time.time())
|
||||
hours = r.randint(150, 190) # fresh keys for the Python app's caches
|
||||
limit = r.randint(40000, 49999)
|
||||
feed_hours = r.randint(20, 30)
|
||||
skill_station = r.choice(
|
||||
["P.67", "P.103", "P.20", "P.75", "P.92", "P.21", "P.5", "P.82"]
|
||||
)
|
||||
day = time.strftime("%Y-%m-%d", time.localtime(time.time() - 86400 * 3))
|
||||
day2 = time.strftime("%Y-%m-%d", time.localtime(time.time() - 86400))
|
||||
out = [
|
||||
Case("/"),
|
||||
Case("/robots.txt"),
|
||||
Case("/llms.txt"),
|
||||
Case("/sitemap.xml"),
|
||||
Case("/static/ping-river-network.geojson", soft=True),
|
||||
Case("/static/custom-markers.json"),
|
||||
Case("/static/flood-zones-p1.geojson", soft=True),
|
||||
Case("/static/flood-2024-all.json"),
|
||||
Case("/static/nope.json"),
|
||||
Case("/openapi.json", compare=compare_openapi),
|
||||
Case("/docs", compare=compare_html_docs),
|
||||
Case("/health", compare=compare_health),
|
||||
Case("/metrics", compare=compare_metrics),
|
||||
Case("/stations"),
|
||||
Case("/stations/8"),
|
||||
Case("/stations/1"),
|
||||
Case("/stations/999"),
|
||||
Case("/stations/abc"),
|
||||
Case(f"/measurements/latest?limit={r.randint(200, 499)}"),
|
||||
Case(f"/measurements/latest?limit={r.randint(1, 15)}"),
|
||||
Case(f"/measurements/history/P.1?hours={hours}&limit={limit}"),
|
||||
Case(f"/measurements/history/P.67?hours={hours + 1}"),
|
||||
Case(f"/measurements/history/P.1?start={day}&end={day2}&limit={limit - 1}"),
|
||||
Case(f"/measurements/history/P.1?start={day}&limit={r.randint(5, 50)}"),
|
||||
Case(f"/measurements/history/P.1?start={day2}&end={day}"),
|
||||
Case(f"/measurements/history/NOPE?hours={hours}"),
|
||||
Case("/measurements/history/P.1?hours=0"),
|
||||
Case("/measurements/history/P.1?limit=100001"),
|
||||
Case("/measurements/history/P.1?start=yesterday"),
|
||||
Case(f"/measurements/station/P.1?hours={r.randint(20, 48)}"),
|
||||
Case("/measurements/station/P.103?hours=6&limit=2"),
|
||||
Case(f"/api/hii/rainfall/latest?hours={feed_hours}", sort_key="station_id"),
|
||||
Case(
|
||||
f"/api/hii/waterlevel/latest?hours={feed_hours + 1}", sort_key="station_id"
|
||||
),
|
||||
Case("/api/hii/waterlevel/latest?hours=500"),
|
||||
Case(
|
||||
f"/api/hii/rainfall/catchment?days={r.randint(15, 30)}",
|
||||
compare=compare_catchment,
|
||||
),
|
||||
Case("/sensors/thaiwater", sort_key="id", soft=True),
|
||||
Case("/forecast"),
|
||||
Case(f"/api/forecast/history/P.1?hours={hours}&horizon=24"),
|
||||
Case(f"/api/forecast/history/P.103?hours={feed_hours}"),
|
||||
Case(f"/api/forecast/history/P.1?start={day}&end={day2}&horizon=6"),
|
||||
Case("/api/forecast/history/P.1?horizon=99"),
|
||||
Case("/api/notifications"),
|
||||
# a station/horizon pair the dashboard does not ask for: not in the
|
||||
# FastAPI app's 15-minute cache; the dashboard's own one may be
|
||||
Case(
|
||||
f"/api/forecast/skill?station_code={skill_station}"
|
||||
f"&horizon={r.randint(1, 48)}",
|
||||
compare=compare_skill,
|
||||
),
|
||||
Case(
|
||||
"/api/forecast/skill?station_code=P.1&horizon=24",
|
||||
compare=compare_skill,
|
||||
soft=True,
|
||||
),
|
||||
Case("/api/forecast/skill?horizon=0"),
|
||||
Case("/api/stats", compare=compare_stats),
|
||||
Case("/scraping/status", compare=compare_status),
|
||||
Case("/nope"),
|
||||
Case("/forecast", method="DELETE"),
|
||||
Case("/config"),
|
||||
Case("/scrape/trigger", method="POST"),
|
||||
Case("/stations", method="POST", body={"station_code": "X"}),
|
||||
Case("/stations/8", method="DELETE"),
|
||||
]
|
||||
if admin_key:
|
||||
key = {"X-API-Key": admin_key}
|
||||
out += [
|
||||
Case("/config", headers={"X-API-Key": "wrong"}),
|
||||
Case("/stations", method="POST", body={"thai_name": 1}, headers=key),
|
||||
Case("/stations/999", method="PUT", body={}, headers=key),
|
||||
]
|
||||
return out
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
|
||||
ap.add_argument("--python", default="http://127.0.0.1:8000")
|
||||
ap.add_argument("--rust", default="http://127.0.0.1:8001")
|
||||
ap.add_argument(
|
||||
"--admin-key", help="ADMIN_API_KEY, to compare the admin errors too"
|
||||
)
|
||||
ap.add_argument("--retries", type=int, default=2)
|
||||
ap.add_argument("--pause", type=float, default=30.0, help="seconds before a retry")
|
||||
args = ap.parse_args()
|
||||
|
||||
failed, warned, passed = [], [], 0
|
||||
for case in cases(args.admin_key):
|
||||
label = f"{case.method} {case.path}"
|
||||
for attempt in range(args.retries + 1):
|
||||
try:
|
||||
hard, warn = case.run(args.python, args.rust)
|
||||
except Exception as e: # noqa: BLE001 - report and go on
|
||||
hard, warn = f"{type(e).__name__}: {e}", None
|
||||
if not hard or attempt == args.retries:
|
||||
break
|
||||
print(f" retry {label}: {hard}", flush=True)
|
||||
time.sleep(args.pause)
|
||||
if hard:
|
||||
failed.append((label, hard))
|
||||
print(f"FAIL {label}: {hard}", flush=True)
|
||||
elif warn:
|
||||
warned.append((label, warn))
|
||||
print(f"warn {label}: {warn}", flush=True)
|
||||
else:
|
||||
passed += 1
|
||||
print(
|
||||
f"{passed} identical, {len(warned)} warnings, {len(failed)} different "
|
||||
f"({args.python} vs {args.rust})"
|
||||
)
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -2,7 +2,7 @@
|
||||
"""Backfill the openmeteo_rain table with the full 2021+ catchment history.
|
||||
|
||||
Usage:
|
||||
uv run scripts/backfill_rain_db.py # DB from Config/.env
|
||||
uv run scripts/backfill_rain_db.py # DB from POSTGRES_* / .env
|
||||
uv run scripts/backfill_rain_db.py --db-url postgresql://...
|
||||
"""
|
||||
|
||||
@@ -13,7 +13,7 @@ import sys
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
|
||||
|
||||
from src.config import Config
|
||||
from src.config import database_url
|
||||
from src.ml.rain import backfill_db
|
||||
|
||||
|
||||
@@ -25,21 +25,17 @@ def main(argv=None) -> int:
|
||||
logging.basicConfig(
|
||||
level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s"
|
||||
)
|
||||
if args.db_url:
|
||||
# postgresql+psycopg2://... -> postgresql (driver suffix is not a dialect)
|
||||
connection_string = args.db_url
|
||||
db_type = args.db_url.split(":", 1)[0].split("+", 1)[0]
|
||||
else:
|
||||
cfg = Config.get_database_config()
|
||||
if cfg["type"] not in ("sqlite", "postgresql", "mysql"):
|
||||
print(f"requires a SQL database, got {cfg['type']}", file=sys.stderr)
|
||||
return 1
|
||||
connection_string, db_type = cfg["connection_string"], cfg["type"]
|
||||
connection_string = args.db_url or database_url()
|
||||
if not connection_string:
|
||||
print(
|
||||
"no database: set POSTGRES_CONNECTION_STRING or POSTGRES_*", file=sys.stderr
|
||||
)
|
||||
return 1
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
|
||||
engine = create_engine(connection_string, pool_pre_ping=True)
|
||||
saved = backfill_db(engine, db_type)
|
||||
saved = backfill_db(engine)
|
||||
print(f"backfilled {saved} hourly rows into openmeteo_rain")
|
||||
return 0 if saved else 1
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
|
||||
Two paths, both idempotent and both skipping what is already stored, so a
|
||||
rerun repairs holes left by transient failures and is safe alongside the
|
||||
hourly live collector:
|
||||
live collector (ping-collector, today + yesterday):
|
||||
|
||||
--dam-id (default: Mae Ngat) one dam, whole range, via api/dam — a handful
|
||||
of requests for the entire 2009-today archive
|
||||
@@ -26,13 +26,8 @@ import sys
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
|
||||
|
||||
from src.config import Config
|
||||
from src.rid_reservoir import (
|
||||
MAE_NGAT_DAM_ID,
|
||||
RidReservoirStore,
|
||||
backfill,
|
||||
backfill_dam,
|
||||
)
|
||||
from src.config import database_url
|
||||
from src.rid_reservoir import MAE_NGAT_DAM_ID, RidReservoirStore, backfill, backfill_dam
|
||||
|
||||
DEFAULT_START = datetime.date(2018, 8, 1) # start of the water_measurements grid
|
||||
|
||||
@@ -71,18 +66,14 @@ def main(argv=None) -> int:
|
||||
logging.basicConfig(
|
||||
level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s"
|
||||
)
|
||||
if args.db_url:
|
||||
# postgresql+psycopg2://... -> postgresql (driver suffix is not a dialect)
|
||||
connection_string = args.db_url
|
||||
db_type = args.db_url.split(":", 1)[0].split("+", 1)[0]
|
||||
else:
|
||||
cfg = Config.get_database_config()
|
||||
if cfg["type"] not in ("sqlite", "postgresql", "mysql"):
|
||||
print(f"requires a SQL database, got {cfg['type']}", file=sys.stderr)
|
||||
return 1
|
||||
connection_string, db_type = cfg["connection_string"], cfg["type"]
|
||||
connection_string = args.db_url or database_url()
|
||||
if not connection_string:
|
||||
print(
|
||||
"no database: set POSTGRES_CONNECTION_STRING or POSTGRES_*", file=sys.stderr
|
||||
)
|
||||
return 1
|
||||
|
||||
store = RidReservoirStore(connection_string, db_type)
|
||||
store = RidReservoirStore(connection_string)
|
||||
if not store.connect():
|
||||
print(
|
||||
"database connection failed — check the connection string",
|
||||
|
||||
+188
-75
@@ -14,7 +14,7 @@ non-zero exit if the model gives less than 12 h of warning before the first
|
||||
3.70 m crossing of the 2024 event.
|
||||
|
||||
Usage:
|
||||
python scripts/backtest_render.py # uses FLOOD_ML_DB_URL/Config
|
||||
python scripts/backtest_render.py # uses FLOOD_ML_DB_URL / POSTGRES_*
|
||||
python scripts/backtest_render.py --db-url postgresql://...
|
||||
"""
|
||||
|
||||
@@ -60,7 +60,11 @@ def fit_backtest_model(df_long: pd.DataFrame, train_end: str, use_dam: bool = Fa
|
||||
rain_series = rain_mod.catchment_mean(rain_mod.load_history())
|
||||
dam_frame = dam_mod.load_history() if use_dam else None
|
||||
X, Y, _meta = features.build_matrix(
|
||||
df_long, STATION, (HORIZON,), stats_end=train_end, rain=rain_series,
|
||||
df_long,
|
||||
STATION,
|
||||
(HORIZON,),
|
||||
stats_end=train_end,
|
||||
rain=rain_series,
|
||||
dam=dam_frame,
|
||||
)
|
||||
train_mask = X.index <= pd.Timestamp(train_end)
|
||||
@@ -92,6 +96,7 @@ def event_series(df_long, X, reg, clf, window_start: str, window_end: str):
|
||||
# whichever is more alarmed. The classifier alone proved unreliable on
|
||||
# out-of-distribution extremes (silent on the 2024 record flood).
|
||||
import numpy as np
|
||||
|
||||
p_clf = clf.predict_proba(Xw)[:, 1]
|
||||
p_sig = 1.0 / (1.0 + np.exp(-(forecasts["pred_max"] - STAGE1) / 0.15))
|
||||
forecasts["p_flood"] = np.maximum(p_clf, p_sig)
|
||||
@@ -109,55 +114,107 @@ def _style_axes(ax):
|
||||
ax.set_axisbelow(True)
|
||||
|
||||
|
||||
def render(observed, forecasts, flood_start, first_alert, out_path, *,
|
||||
title, subtitle, detail=False, show_stage7=False, peak_note=None):
|
||||
def render(
|
||||
observed,
|
||||
forecasts,
|
||||
flood_start,
|
||||
first_alert,
|
||||
out_path,
|
||||
*,
|
||||
title,
|
||||
subtitle,
|
||||
detail=False,
|
||||
show_stage7=False,
|
||||
peak_note=None,
|
||||
):
|
||||
fig, (ax, axp) = plt.subplots(
|
||||
2, 1, figsize=(12.6, 7.6), sharex=True,
|
||||
2,
|
||||
1,
|
||||
figsize=(12.6, 7.6),
|
||||
sharex=True,
|
||||
gridspec_kw={"height_ratios": [2.2, 1], "hspace": 0.12},
|
||||
)
|
||||
fig.patch.set_facecolor("white")
|
||||
|
||||
marker = dict(marker="o", markersize=3) if detail else {}
|
||||
ax.plot(observed.index, observed.values, color=BLUE, linewidth=2.2,
|
||||
label="Observed level" + (" (hourly)" if detail else ""), **marker)
|
||||
ax.plot(
|
||||
observed.index,
|
||||
observed.values,
|
||||
color=BLUE,
|
||||
linewidth=2.2,
|
||||
label="Observed level" + (" (hourly)" if detail else ""),
|
||||
**marker,
|
||||
)
|
||||
marker = dict(marker="s", markersize=3) if detail else {}
|
||||
ax.plot(forecasts.index, forecasts["pred_max"], color=AMBER, linewidth=2,
|
||||
linestyle="--", label="Predicted 24 h peak (issued at that hour)", **marker)
|
||||
ax.plot(
|
||||
forecasts.index,
|
||||
forecasts["pred_max"],
|
||||
color=AMBER,
|
||||
linewidth=2,
|
||||
linestyle="--",
|
||||
label="Predicted 24 h peak (issued at that hour)",
|
||||
**marker,
|
||||
)
|
||||
|
||||
ax.axhline(STAGE1, color=RED, linewidth=1, alpha=0.65)
|
||||
ax.annotate(f"{STAGE1:.2f} m · stage 1 · flooding begins", xy=(0.06, STAGE1),
|
||||
xycoords=("axes fraction", "data"), xytext=(0, 5),
|
||||
textcoords="offset points", color=RED, fontsize=10.5)
|
||||
ax.annotate(
|
||||
f"{STAGE1:.2f} m · stage 1 · flooding begins",
|
||||
xy=(0.06, STAGE1),
|
||||
xycoords=("axes fraction", "data"),
|
||||
xytext=(0, 5),
|
||||
textcoords="offset points",
|
||||
color=RED,
|
||||
fontsize=10.5,
|
||||
)
|
||||
if show_stage7:
|
||||
ax.axhline(STAGE7, color=RED, linewidth=1, alpha=0.65)
|
||||
ax.annotate(f"{STAGE7:.2f} m · stage 7 · widespread", xy=(0.06, STAGE7),
|
||||
xycoords=("axes fraction", "data"), xytext=(0, 5),
|
||||
textcoords="offset points", color=RED, fontsize=10.5)
|
||||
ax.annotate(
|
||||
f"{STAGE7:.2f} m · stage 7 · widespread",
|
||||
xy=(0.06, STAGE7),
|
||||
xycoords=("axes fraction", "data"),
|
||||
xytext=(0, 5),
|
||||
textcoords="offset points",
|
||||
color=RED,
|
||||
fontsize=10.5,
|
||||
)
|
||||
|
||||
if peak_note:
|
||||
peak_ts = observed.idxmax()
|
||||
ax.annotate(peak_note, xy=(peak_ts, observed.max()),
|
||||
xytext=(12, 10), textcoords="offset points",
|
||||
color=BLUE, fontsize=11.5, fontweight="bold")
|
||||
ax.annotate(
|
||||
peak_note,
|
||||
xy=(peak_ts, observed.max()),
|
||||
xytext=(12, 10),
|
||||
textcoords="offset points",
|
||||
color=BLUE,
|
||||
fontsize=11.5,
|
||||
fontweight="bold",
|
||||
)
|
||||
|
||||
ax.set_ylabel("P.1 water level (m)", color=INK, fontsize=11.5)
|
||||
ax.legend(loc="upper left", frameon=False, fontsize=10.5)
|
||||
_style_axes(ax)
|
||||
|
||||
axp.plot(forecasts.index, forecasts["p_flood"], color=AMBER, linewidth=1.8)
|
||||
axp.fill_between(forecasts.index, 0, forecasts["p_flood"],
|
||||
color=AMBER, alpha=0.28)
|
||||
axp.fill_between(forecasts.index, 0, forecasts["p_flood"], color=AMBER, alpha=0.28)
|
||||
axp.axhline(0.5, color=INK, linewidth=0.9, linestyle=":", alpha=0.6)
|
||||
axp.set_ylim(-0.02, 1.1)
|
||||
axp.set_ylabel(f"P(flooding within {HORIZON} h)", color=INK, fontsize=11.5)
|
||||
_style_axes(axp)
|
||||
|
||||
if first_alert is not None:
|
||||
lead_h = None if flood_start is None else \
|
||||
int((flood_start - first_alert).total_seconds() // 3600)
|
||||
lead_txt = "" if lead_h is None else (
|
||||
f"\n({lead_h} h before flooding began)" if lead_h >= 0
|
||||
else f"\n({-lead_h} h after flooding began)"
|
||||
lead_h = (
|
||||
None
|
||||
if flood_start is None
|
||||
else int((flood_start - first_alert).total_seconds() // 3600)
|
||||
)
|
||||
lead_txt = (
|
||||
""
|
||||
if lead_h is None
|
||||
else (
|
||||
f"\n({lead_h} h before flooding began)"
|
||||
if lead_h >= 0
|
||||
else f"\n({-lead_h} h after flooding began)"
|
||||
)
|
||||
)
|
||||
if detail and flood_start is not None:
|
||||
for a in (ax, axp):
|
||||
@@ -166,34 +223,61 @@ def render(observed, forecasts, flood_start, first_alert, out_path, *,
|
||||
# Anchor labels away from each other in chronological order so a
|
||||
# late alert (alert AFTER crossing) cannot overprint the labels.
|
||||
events = sorted(
|
||||
[(first_alert, "model alert", AMBER), (flood_start, "flooding begins", BLUE)]
|
||||
[
|
||||
(first_alert, "model alert", AMBER),
|
||||
(flood_start, "flooding begins", BLUE),
|
||||
]
|
||||
)
|
||||
for (ts, label, color), (offset, align) in zip(events, ((-8, "right"), (8, "left"))):
|
||||
ax.annotate(f"{label}\n{ts:%d %b %H:%M}",
|
||||
xy=(ts, observed.min()), xytext=(offset, 18),
|
||||
textcoords="offset points", ha=align,
|
||||
color=color, fontsize=11, fontweight="bold")
|
||||
for (ts, label, color), (offset, align) in zip(
|
||||
events, ((-8, "right"), (8, "left"))
|
||||
):
|
||||
ax.annotate(
|
||||
f"{label}\n{ts:%d %b %H:%M}",
|
||||
xy=(ts, observed.min()),
|
||||
xytext=(offset, 18),
|
||||
textcoords="offset points",
|
||||
ha=align,
|
||||
color=color,
|
||||
fontsize=11,
|
||||
fontweight="bold",
|
||||
)
|
||||
mid_y = observed.min() + (observed.max() - observed.min()) * 0.28
|
||||
ax.annotate("", xy=(flood_start, mid_y), xytext=(first_alert, mid_y),
|
||||
arrowprops=dict(arrowstyle="<->", color=INK, lw=1.3))
|
||||
ax.annotate(
|
||||
"",
|
||||
xy=(flood_start, mid_y),
|
||||
xytext=(first_alert, mid_y),
|
||||
arrowprops=dict(arrowstyle="<->", color=INK, lw=1.3),
|
||||
)
|
||||
arrow_label = (
|
||||
f"{lead_h} h warning" if lead_h >= 0 else f"alert {-lead_h} h late"
|
||||
)
|
||||
ax.annotate(arrow_label,
|
||||
xy=(first_alert + (flood_start - first_alert) / 2, mid_y),
|
||||
xytext=(0, 8), textcoords="offset points", ha="center",
|
||||
color=INK, fontsize=11.5, fontweight="bold")
|
||||
ax.annotate(
|
||||
arrow_label,
|
||||
xy=(first_alert + (flood_start - first_alert) / 2, mid_y),
|
||||
xytext=(0, 8),
|
||||
textcoords="offset points",
|
||||
ha="center",
|
||||
color=INK,
|
||||
fontsize=11.5,
|
||||
fontweight="bold",
|
||||
)
|
||||
else:
|
||||
axp.annotate(f"first alert · {first_alert:%d %b %H:%M}{lead_txt}",
|
||||
xy=(first_alert, 0.62), xytext=(10, 0),
|
||||
textcoords="offset points", color=RED, fontsize=10.5,
|
||||
bbox=dict(facecolor="white", alpha=0.75, edgecolor="none"))
|
||||
axp.annotate(
|
||||
f"first alert · {first_alert:%d %b %H:%M}{lead_txt}",
|
||||
xy=(first_alert, 0.62),
|
||||
xytext=(10, 0),
|
||||
textcoords="offset points",
|
||||
color=RED,
|
||||
fontsize=10.5,
|
||||
bbox=dict(facecolor="white", alpha=0.75, edgecolor="none"),
|
||||
)
|
||||
|
||||
locator = mdates.DayLocator(interval=1 if detail else 3)
|
||||
axp.xaxis.set_major_locator(locator)
|
||||
axp.xaxis.set_major_formatter(mdates.DateFormatter("%d %b"))
|
||||
fig.suptitle(f"{title}\n{subtitle}", x=0.07, y=0.985, ha="left",
|
||||
fontsize=15, color=INK)
|
||||
fig.suptitle(
|
||||
f"{title}\n{subtitle}", x=0.07, y=0.985, ha="left", fontsize=15, color=INK
|
||||
)
|
||||
fig.subplots_adjust(top=0.885, left=0.07, right=0.97, bottom=0.07)
|
||||
fig.savefig(out_path, dpi=110)
|
||||
plt.close(fig)
|
||||
@@ -204,16 +288,20 @@ def main(argv=None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--db-url", default=None)
|
||||
parser.add_argument("--out-dir", default=os.path.join("docs", "img"))
|
||||
parser.add_argument("--dam", action="store_true",
|
||||
help="ablation: include Mae Ngat reservoir features "
|
||||
"(2026-08 result: costs 1-3 h of alert lead)")
|
||||
parser.add_argument("--no-hii-fill", action="store_true",
|
||||
help="ablation: load without the HII gap-fill merge")
|
||||
parser.add_argument(
|
||||
"--dam",
|
||||
action="store_true",
|
||||
help="ablation: include Mae Ngat reservoir features "
|
||||
"(2026-08 result: costs 1-3 h of alert lead)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-hii-fill",
|
||||
action="store_true",
|
||||
help="ablation: load without the HII gap-fill merge",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
df = data.load_measurements(
|
||||
db_url=args.db_url, hii_fill=not args.no_hii_fill
|
||||
)
|
||||
df = data.load_measurements(db_url=args.db_url, hii_fill=not args.no_hii_fill)
|
||||
if df.empty:
|
||||
print("no measurement data available", file=sys.stderr)
|
||||
return 1
|
||||
@@ -222,41 +310,66 @@ def main(argv=None) -> int:
|
||||
# --- October 2024 record flood: trained only on data before 1 Sep 2024 ---
|
||||
X, reg, clf = fit_backtest_model(df, "2024-08-31", use_dam=args.dam)
|
||||
obs, fc, flood_start, first_alert = event_series(
|
||||
df, X, reg, clf, "2024-09-10", "2024-10-14 23:00")
|
||||
df, X, reg, clf, "2024-09-10", "2024-10-14 23:00"
|
||||
)
|
||||
peak = float(obs.max())
|
||||
render(obs, fc, flood_start, first_alert,
|
||||
os.path.join(args.out_dir, "backtest-2024-p1.png"),
|
||||
title="October 2024 flood: what the model saw coming",
|
||||
subtitle="P.1 Nawarat Bridge — model trained only on data before 1 Sep 2024",
|
||||
show_stage7=True, peak_note=f"record peak {peak:.2f} m")
|
||||
render(
|
||||
obs,
|
||||
fc,
|
||||
flood_start,
|
||||
first_alert,
|
||||
os.path.join(args.out_dir, "backtest-2024-p1.png"),
|
||||
title="October 2024 flood: what the model saw coming",
|
||||
subtitle="P.1 Nawarat Bridge — model trained only on data before 1 Sep 2024",
|
||||
show_stage7=True,
|
||||
peak_note=f"record peak {peak:.2f} m",
|
||||
)
|
||||
|
||||
obs_d, fc_d, flood_d, alert_d = event_series(
|
||||
df, X, reg, clf, "2024-09-21 18:00", "2024-09-28 06:00")
|
||||
df, X, reg, clf, "2024-09-21 18:00", "2024-09-28 06:00"
|
||||
)
|
||||
lead_h = None
|
||||
if alert_d is not None and flood_d is not None:
|
||||
lead_h = int((flood_d - alert_d).total_seconds() // 3600)
|
||||
render(obs_d, fc_d, flood_d, alert_d,
|
||||
os.path.join(args.out_dir, "backtest-2024-p1-detail.png"),
|
||||
title="Detection in detail: 22–28 September 2024, hour by hour",
|
||||
subtitle=(
|
||||
f"the model alerts {lead_h} h before the river crosses the flooding line"
|
||||
if lead_h is not None and lead_h > 0
|
||||
else "model alert vs the river crossing the flooding line"
|
||||
),
|
||||
detail=True)
|
||||
render(
|
||||
obs_d,
|
||||
fc_d,
|
||||
flood_d,
|
||||
alert_d,
|
||||
os.path.join(args.out_dir, "backtest-2024-p1-detail.png"),
|
||||
title="Detection in detail: 22–28 September 2024, hour by hour",
|
||||
subtitle=(
|
||||
f"the model alerts {lead_h} h before the river crosses the flooding line"
|
||||
if lead_h is not None and lead_h > 0
|
||||
else "model alert vs the river crossing the flooding line"
|
||||
),
|
||||
detail=True,
|
||||
)
|
||||
|
||||
# --- September 2025 flood: the deployed configuration (trained <= 2024) ---
|
||||
X25, reg25, clf25 = fit_backtest_model(df, "2024-12-31", use_dam=args.dam)
|
||||
obs25, fc25, flood25, alert25 = event_series(
|
||||
df, X25, reg25, clf25, "2025-09-22", "2025-10-02 12:00")
|
||||
pred_at_alert = float(fc25.loc[alert25:, "pred_max"].iloc[:24].max()) if alert25 is not None else None
|
||||
df, X25, reg25, clf25, "2025-09-22", "2025-10-02 12:00"
|
||||
)
|
||||
pred_at_alert = (
|
||||
float(fc25.loc[alert25:, "pred_max"].iloc[:24].max())
|
||||
if alert25 is not None
|
||||
else None
|
||||
)
|
||||
note = f"peak {float(obs25.max()):.2f} m" + (
|
||||
f" (predicted {pred_at_alert:.2f} m)" if pred_at_alert is not None else "")
|
||||
render(obs25, fc25, flood25, alert25,
|
||||
os.path.join(args.out_dir, "backtest-2025-p1.png"),
|
||||
title="The September 2025 flood — as forecast by the deployed configuration",
|
||||
subtitle="model trained only on data through 2024; this event was never seen in training",
|
||||
detail=True, peak_note=note)
|
||||
f" (predicted {pred_at_alert:.2f} m)" if pred_at_alert is not None else ""
|
||||
)
|
||||
render(
|
||||
obs25,
|
||||
fc25,
|
||||
flood25,
|
||||
alert25,
|
||||
os.path.join(args.out_dir, "backtest-2025-p1.png"),
|
||||
title="The September 2025 flood — as forecast by the deployed configuration",
|
||||
subtitle="model trained only on data through 2024; this event was never seen in training",
|
||||
detail=True,
|
||||
peak_note=note,
|
||||
)
|
||||
|
||||
print(f"2024: flooding began {flood_start}, first alert {first_alert}")
|
||||
print(f"2025: flooding began {flood25}, first alert {alert25}")
|
||||
|
||||
@@ -1,150 +0,0 @@
|
||||
"""Live parity check: Python collectors vs the Rust ping-collector.
|
||||
|
||||
Runs one collection of every source with the Python code into PY_DB and with
|
||||
`ping-collector` into RS_DB (both should start empty), back to back against the
|
||||
real upstream APIs, then diffs all nine collector tables row by row (ids and
|
||||
created/updated timestamps ignored). Exit status 1 on any difference.
|
||||
|
||||
uv run python scripts/collector_parity.py \
|
||||
--py-db postgresql://postgres:ping@localhost:55432/py \
|
||||
--rs-db postgresql://postgres:ping@localhost:55432/rs \
|
||||
--collector rust/target/debug/ping-collector
|
||||
|
||||
Upstream snapshots can move between the two runs (a new HII hour, a revised
|
||||
RID value); rerun before treating a small difference as a bug.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(REPO))
|
||||
|
||||
TABLES = {
|
||||
"stations": (
|
||||
"id",
|
||||
"id, station_code, thai_name, english_name, latitude, longitude, geohash",
|
||||
),
|
||||
"water_measurements": (
|
||||
"timestamp, station_id",
|
||||
"timestamp, station_id, water_level, discharge, discharge_percent, status",
|
||||
),
|
||||
"hii_rain_stations": (
|
||||
"id",
|
||||
"id, oldcode, name_th, name_en, latitude, longitude, sub_basin_id, agency",
|
||||
),
|
||||
"hii_rainfall": (
|
||||
"station_id, timestamp",
|
||||
"station_id, timestamp, rain_1h, rain_24h",
|
||||
),
|
||||
"hii_wl_stations": (
|
||||
"id",
|
||||
"id, oldcode, rid_code, name_th, name_en, latitude, longitude, agency, "
|
||||
"river_name, offset_msl, ground_level_msl, min_bank_msl, "
|
||||
"critical_level_msl, critical_level_m, qmax, is_key_station",
|
||||
),
|
||||
"hii_waterlevel": (
|
||||
"station_id, timestamp",
|
||||
"station_id, timestamp, wl_msl, wl_m, discharge, flow_rate, "
|
||||
"storage_percent, situation_level, diff_wl_bank",
|
||||
),
|
||||
"rid_dams": (
|
||||
"dam_id",
|
||||
"dam_id, region, name_th, latitude, longitude, capacity_max_mcm, "
|
||||
"capacity_normal_mcm",
|
||||
),
|
||||
"rid_reservoir_daily": (
|
||||
"dam_id, date",
|
||||
"dam_id, date, storage_mcm, storage_pct, inflow_mcm, outflow_mcm, level_msl",
|
||||
),
|
||||
"openmeteo_rain": (
|
||||
"timestamp",
|
||||
"timestamp, chiang_dao, mae_taeng, mae_ngat, mae_rim, chiang_mai, "
|
||||
"catchment_mean",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def run_python(db_url: str, gap_days: int):
|
||||
os.environ.update(
|
||||
DB_TYPE="postgresql", POSTGRES_CONNECTION_STRING=db_url, LOG_LEVEL="WARNING"
|
||||
)
|
||||
from sqlalchemy import create_engine
|
||||
|
||||
from src.config import Config
|
||||
from src.hii_collector import HiiCollector
|
||||
from src.ml import rain
|
||||
from src.rid_reservoir import RidReservoirCollector
|
||||
from src.water_scraper_v3 import EnhancedWaterMonitorScraper
|
||||
|
||||
cfg = Config.get_database_config()
|
||||
scraper = EnhancedWaterMonitorScraper(cfg)
|
||||
scraper.fill_data_gaps(gap_days)
|
||||
scraper.run_scraping_cycle()
|
||||
HiiCollector(cfg).run_cycle()
|
||||
RidReservoirCollector(cfg).run_cycle()
|
||||
rain.save_to_db(rain.fetch_forecast(), create_engine(db_url), "postgresql")
|
||||
|
||||
|
||||
def run_rust(db_url: str, collector: str, gap_days: int):
|
||||
env = {**os.environ, "POSTGRES_CONNECTION_STRING": db_url, "DB_TYPE": "postgresql"}
|
||||
for args in (["fill-gaps", str(gap_days)], ["once"]):
|
||||
subprocess.run([collector, *args], env=env, check=True)
|
||||
|
||||
|
||||
def table_rows(db_url: str, table: str, key: str, cols: str) -> dict:
|
||||
import psycopg2
|
||||
|
||||
with psycopg2.connect(db_url) as conn:
|
||||
cur = conn.cursor()
|
||||
cur.execute(f"SELECT {cols} FROM {table} ORDER BY {key}")
|
||||
nkey = len(key.split(","))
|
||||
return {row[:nkey]: row for row in cur.fetchall()}
|
||||
|
||||
|
||||
def diff(py_db: str, rs_db: str) -> bool:
|
||||
identical = True
|
||||
for table, (key, cols) in TABLES.items():
|
||||
a = table_rows(py_db, table, key, cols)
|
||||
b = table_rows(rs_db, table, key, cols)
|
||||
only_py, only_rs = sorted(set(a) - set(b)), sorted(set(b) - set(a))
|
||||
differ = sorted(k for k in set(a) & set(b) if a[k] != b[k])
|
||||
same = not (only_py or only_rs or differ)
|
||||
identical &= same
|
||||
print(
|
||||
f"{table:<20} py {len(a):>6} rs {len(b):>6} only-py {len(only_py):>4} "
|
||||
f"only-rs {len(only_rs):>4} differing {len(differ):>4} "
|
||||
f"{'identical' if same else 'DIFFERENT'}"
|
||||
)
|
||||
for k in only_py[:3]:
|
||||
print(" only py:", a[k])
|
||||
for k in only_rs[:3]:
|
||||
print(" only rs:", b[k])
|
||||
for k in differ[:3]:
|
||||
print(" py:", a[k], "\n rs:", b[k])
|
||||
return identical
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.add_argument("--py-db", required=True)
|
||||
parser.add_argument("--rs-db", required=True)
|
||||
parser.add_argument(
|
||||
"--collector", default=str(REPO / "rust/target/debug/ping-collector")
|
||||
)
|
||||
parser.add_argument("--gap-days", type=int, default=2)
|
||||
parser.add_argument("--diff-only", action="store_true")
|
||||
args = parser.parse_args()
|
||||
if not args.diff_only:
|
||||
run_python(args.py_db, args.gap_days)
|
||||
run_rust(args.rs_db, args.collector, args.gap_days)
|
||||
ok = diff(args.py_db, args.rs_db)
|
||||
print("ALL IDENTICAL" if ok else "differences found")
|
||||
sys.exit(0 if ok else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+15
-4
@@ -25,18 +25,29 @@ class Handler(http.server.BaseHTTPRequestHandler):
|
||||
self._send(200, "application/json", Path(file).read_bytes())
|
||||
return
|
||||
if self.path.startswith("/static/"):
|
||||
f = STATIC / self.path[len("/static/"):].split("?")[0]
|
||||
f = STATIC / self.path[len("/static/") :].split("?")[0]
|
||||
if f.is_file():
|
||||
ctype = "application/json" if f.suffix in (".json", ".geojson") else "application/octet-stream"
|
||||
ctype = (
|
||||
"application/json"
|
||||
if f.suffix in (".json", ".geojson")
|
||||
else "application/octet-stream"
|
||||
)
|
||||
self._send(200, ctype, f.read_bytes())
|
||||
return
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
UPSTREAM + self.path,
|
||||
headers={"User-Agent": "Mozilla/5.0 (dev_proxy; +https://buildfor.life)", "Accept": "application/json"},
|
||||
headers={
|
||||
"User-Agent": "Mozilla/5.0 (dev_proxy; +https://buildfor.life)",
|
||||
"Accept": "application/json",
|
||||
},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=60) as r:
|
||||
self._send(r.status, r.headers.get("Content-Type", "application/json"), r.read())
|
||||
self._send(
|
||||
r.status,
|
||||
r.headers.get("Content-Type", "application/json"),
|
||||
r.read(),
|
||||
)
|
||||
except urllib.error.HTTPError as e:
|
||||
self._send(e.code, "application/json", e.read())
|
||||
|
||||
|
||||
@@ -1,132 +0,0 @@
|
||||
"""Drive the production notify path in-process: startup init -> seeded readings
|
||||
-> forecast cache -> _notify_transitions -> sqlite state -> real ntfy."""
|
||||
|
||||
import asyncio
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
import requests
|
||||
|
||||
os.environ.update(
|
||||
DB_TYPE="sqlite",
|
||||
WATER_DB_PATH=os.path.join(os.environ["LOCALAPPDATA"], "Temp", "smoke3.db"),
|
||||
NTFY_SERVER="http://127.0.0.1:2586",
|
||||
NTFY_TOKEN=os.environ.get("NTFY_TOKEN", ""),
|
||||
NTFY_TOPIC_PREFIX="ping",
|
||||
)
|
||||
for f in ("smoke3.db",):
|
||||
p = os.path.join(os.environ["LOCALAPPDATA"], "Temp", f)
|
||||
if os.path.exists(p):
|
||||
os.remove(p)
|
||||
|
||||
from src import web_api # noqa: E402
|
||||
from src.config import Config # noqa: E402
|
||||
|
||||
assert Config.NTFY_SERVER
|
||||
|
||||
|
||||
async def main():
|
||||
# what the lifespan does at startup, minus the scheduler
|
||||
from src import notify as notify_mod
|
||||
from src.forecast_history import ForecastHistoryStore
|
||||
from src.water_scraper_v3 import EnhancedWaterMonitorScraper
|
||||
|
||||
db_config = Config.get_database_config()
|
||||
web_api.app_state["scraper"] = EnhancedWaterMonitorScraper(db_config)
|
||||
store = ForecastHistoryStore(db_config["connection_string"], db_config["type"])
|
||||
store.connect()
|
||||
web_api.app_state["forecast_store"] = store
|
||||
state = notify_mod.NotificationState(store.engine, store.db_type)
|
||||
pub = notify_mod.NtfyPublisher(
|
||||
Config.NTFY_SERVER, prefix=Config.NTFY_TOPIC_PREFIX, token=Config.NTFY_TOKEN
|
||||
)
|
||||
web_api.app_state["notify"] = (pub, state)
|
||||
|
||||
scraper = web_api.app_state["scraper"]
|
||||
now = datetime.datetime.now().replace(minute=0, second=0, microsecond=0)
|
||||
|
||||
def seed(level_p1, level_p103, ts):
|
||||
rows = [
|
||||
{
|
||||
"station_code": "P.1",
|
||||
"station_id": 1,
|
||||
"timestamp": ts,
|
||||
"water_level": level_p1,
|
||||
"discharge": 400.0,
|
||||
"station_name_en": "Nawarat Bridge",
|
||||
"station_name_th": "สะพานนวรัฐ",
|
||||
"discharge_percent": 30.0,
|
||||
"status": "active",
|
||||
},
|
||||
{
|
||||
"station_code": "P.103",
|
||||
"station_id": 2,
|
||||
"timestamp": ts,
|
||||
"water_level": level_p103,
|
||||
"discharge": 300.0,
|
||||
"station_name_en": "Ring Road 3",
|
||||
"station_name_th": "วงแหวน 3",
|
||||
"discharge_percent": 20.0,
|
||||
"status": "active",
|
||||
},
|
||||
]
|
||||
scraper.db_adapter.save_measurements(rows)
|
||||
|
||||
def forecast(p):
|
||||
with web_api.FORECAST_CACHE_LOCK:
|
||||
web_api.FORECAST_CACHE["all"] = (
|
||||
0,
|
||||
[
|
||||
{
|
||||
"station_code": "P.1",
|
||||
"horizon_hours": 24,
|
||||
"p_warning": p,
|
||||
"predicted_max_level": 3.9,
|
||||
"source": "model",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
def poll(topic):
|
||||
out = []
|
||||
for line in (
|
||||
requests.get(f"{Config.NTFY_SERVER}/{topic}/json?poll=1", timeout=5)
|
||||
.text.strip()
|
||||
.splitlines()
|
||||
):
|
||||
m = json.loads(line)
|
||||
if m.get("event") == "message":
|
||||
out.append(m.get("title") or m.get("message", "")[:40])
|
||||
return out
|
||||
|
||||
# cycle 1: quiet
|
||||
seed(1.6, 3.2, now - datetime.timedelta(hours=2))
|
||||
forecast(0.02)
|
||||
await web_api._notify_transitions()
|
||||
# cycle 2: P.1 crosses warning, model outlook on
|
||||
seed(3.75, 3.3, now - datetime.timedelta(hours=1))
|
||||
forecast(0.7)
|
||||
await web_api._notify_transitions()
|
||||
# cycle 3: same state -> silence
|
||||
seed(3.80, 3.3, now)
|
||||
forecast(0.65)
|
||||
await web_api._notify_transitions()
|
||||
|
||||
print("ping-p1-warning:", poll("ping-p1-warning"))
|
||||
print("ping-warning: ", poll("ping-warning"))
|
||||
print("ping-p1-outlook:", poll("ping-p1-outlook"))
|
||||
print("ping-p103-warning:", poll("ping-p103-warning"))
|
||||
from sqlalchemy import text
|
||||
|
||||
with store.engine.connect() as c:
|
||||
print(
|
||||
"state table:",
|
||||
c.execute(
|
||||
text("SELECT key, state, value FROM notification_state ORDER BY key")
|
||||
).fetchall(),
|
||||
)
|
||||
|
||||
|
||||
asyncio.run(main())
|
||||
@@ -1,8 +1,10 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare the Python forecast with ping-collector's on the live database.
|
||||
"""Compare ping-collector's forecast with the Python reference on the live database.
|
||||
|
||||
Run before setting FORECAST_IN_COLLECTOR=true, and after any change to src/ml
|
||||
or rust/forecast. On the server, as the service user from the app dir:
|
||||
src/ml/predict.py is the reference implementation the Rust forecaster
|
||||
(rust/forecast) was ported from. Run after any change to src/ml or
|
||||
rust/forecast; scripts/install.sh runs it on every install. On the server, as
|
||||
the service user from the app dir:
|
||||
|
||||
sudo -u water-monitor .venv/bin/python scripts/forecast_parity.py
|
||||
|
||||
|
||||
+173
-98
@@ -1,127 +1,202 @@
|
||||
#!/usr/bin/env bash
|
||||
# Install or update the Ping River Monitor on a systemd host.
|
||||
#
|
||||
# Install the Thailand Water Level Monitor as a hardened systemd service.
|
||||
# sudo bash scripts/install.sh
|
||||
#
|
||||
# Creates a dedicated system user, deploys the code to /opt, builds a uv-managed
|
||||
# virtualenv, installs the systemd unit, and enables the service. Idempotent:
|
||||
# safe to re-run to update an existing install.
|
||||
#
|
||||
# Usage (as root, from a checkout of the repo):
|
||||
# sudo bash scripts/install.sh
|
||||
#
|
||||
# Override defaults via environment variables:
|
||||
# APP_DIR=/opt/thailand-water-monitor SERVICE_USER=water-monitor sudo -E bash scripts/install.sh
|
||||
# What runs:
|
||||
# ping-collector.service the service (rust/collector): RID/HII/dam/rain
|
||||
# collection, forecast, ntfy + Matrix
|
||||
# notifications, website + API on WEB_LISTEN
|
||||
# water-monitor-retrain.timer monthly model retrain (Python, scripts/retrain.sh)
|
||||
#
|
||||
# Re-run after every update (git pull + build). Steps, all idempotent:
|
||||
# 1. system user/group (water-monitor)
|
||||
# 2. code: run from a checkout other than APP_DIR, it is rsynced there (the
|
||||
# trained models, .env, stations.json and bin/ are kept) and handed to
|
||||
# the service user. Production runs in place (APP_DIR is the checkout):
|
||||
# then nothing is copied and no ownership changes.
|
||||
# 3. the binary: BIN=/path/to/ping-collector (e.g. the CI artifact
|
||||
# "ping-collector-linux-x86_64") or rust/target/release/ping-collector,
|
||||
# built beforehand as a normal user:
|
||||
# cargo build --release --locked --manifest-path rust/Cargo.toml
|
||||
# (Rust >= 1.88 from rustup, plus pkg-config and libssl-dev)
|
||||
# 4. the Python training environment (.venv, `uv sync --frozen`): in place
|
||||
# as the user running sudo (who pulled and built), otherwise as root;
|
||||
# SKIP_UV=1 skips it
|
||||
# 5. .env: created from .env.example on a fresh install (edit, then re-run);
|
||||
# WEB_LISTEN and FORECAST_IN_COLLECTOR default to the production values
|
||||
# when absent
|
||||
# 6. systemd units; the retired Python web app (water-monitor.service) and
|
||||
# the old `run.py --alert-check` Matrix cron job are removed
|
||||
# 7. checks (SKIP_CHECKS=1 skips them): a dry run of every source against
|
||||
# the database, and the forecast compared with the Python reference on
|
||||
# the live data (scripts/forecast_parity.py)
|
||||
# 8. (re)start, then wait for the website to answer
|
||||
set -euo pipefail
|
||||
|
||||
APP_DIR="${APP_DIR:-/opt/thailand-water-monitor}"
|
||||
SERVICE_USER="${SERVICE_USER:-water-monitor}"
|
||||
SERVICE_USER="${SERVICE_USER:-water-monitor}" # as in the units
|
||||
SERVICE_GROUP="${SERVICE_GROUP:-${SERVICE_USER}}"
|
||||
SERVICE_NAME="water-monitor.service"
|
||||
RETRAIN_NAME="water-monitor-retrain"
|
||||
|
||||
# Resolve the repo root (parent of this scripts/ directory).
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
ENV_FILE="${APP_DIR}/.env"
|
||||
PYTHON="${APP_DIR}/.venv/bin/python"
|
||||
REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
RETRAIN="water-monitor-retrain"
|
||||
|
||||
log() { printf '\033[1;32m==>\033[0m %s\n' "$*"; }
|
||||
warn() { printf '\033[1;33m[warn]\033[0m %s\n' "$*"; }
|
||||
warn() { printf '\033[1;33m[warn]\033[0m %s\n' "$*" >&2; }
|
||||
die() { printf '\033[1;31m[error]\033[0m %s\n' "$*" >&2; exit 1; }
|
||||
|
||||
[ "$(id -u)" -eq 0 ] || die "This script must be run as root (use sudo)."
|
||||
[ "$(id -u)" -eq 0 ] || die "run as root (sudo)"
|
||||
|
||||
# 1. Dedicated system user/group (no login, no home) --------------------------
|
||||
if ! getent group "${SERVICE_GROUP}" >/dev/null; then
|
||||
log "Creating group ${SERVICE_GROUP}"
|
||||
groupadd --system "${SERVICE_GROUP}"
|
||||
fi
|
||||
if ! id "${SERVICE_USER}" >/dev/null 2>&1; then
|
||||
log "Creating system user ${SERVICE_USER}"
|
||||
set_default() { # key value: append when the key is not set at all
|
||||
grep -q "^$1=" "$ENV_FILE" || printf '%s=%s\n' "$1" "$2" >>"$ENV_FILE"
|
||||
}
|
||||
|
||||
# A command the way the services run it: same user, working directory and
|
||||
# EnvironmentFile parsing (systemd's).
|
||||
as_service() {
|
||||
if systemd-run --version >/dev/null 2>&1; then
|
||||
systemd-run --quiet --pipe --wait --collect \
|
||||
-p User="$SERVICE_USER" -p Group="$SERVICE_GROUP" \
|
||||
-p WorkingDirectory="$APP_DIR" -p EnvironmentFile="$ENV_FILE" "$@"
|
||||
else
|
||||
(cd "$APP_DIR" && runuser -u "$SERVICE_USER" -- "$@")
|
||||
fi
|
||||
}
|
||||
|
||||
wait_http() { # url seconds
|
||||
local i
|
||||
for i in $(seq "$2"); do
|
||||
curl -fs -o /dev/null --max-time 5 "$1" 2>/dev/null && return 0
|
||||
sleep 1
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# 1. user ---------------------------------------------------------------------
|
||||
getent group "$SERVICE_GROUP" >/dev/null || groupadd --system "$SERVICE_GROUP"
|
||||
if ! id "$SERVICE_USER" >/dev/null 2>&1; then
|
||||
log "Creating system user $SERVICE_USER"
|
||||
useradd --system --no-create-home --shell /usr/sbin/nologin \
|
||||
--gid "${SERVICE_GROUP}" "${SERVICE_USER}"
|
||||
--gid "$SERVICE_GROUP" "$SERVICE_USER"
|
||||
fi
|
||||
|
||||
# 2. Deploy code to APP_DIR ----------------------------------------------------
|
||||
log "Deploying code to ${APP_DIR}"
|
||||
mkdir -p "${APP_DIR}"
|
||||
if command -v rsync >/dev/null 2>&1; then
|
||||
# 2. code ---------------------------------------------------------------------
|
||||
in_place=0
|
||||
[ -d "$APP_DIR" ] && [ "$(cd "$APP_DIR" && pwd -P)" = "$(cd "$REPO_DIR" && pwd -P)" ] && in_place=1
|
||||
if [ "$in_place" = 0 ]; then
|
||||
command -v rsync >/dev/null || die "rsync is needed to copy $REPO_DIR to $APP_DIR"
|
||||
log "Copying $REPO_DIR to $APP_DIR"
|
||||
mkdir -p "$APP_DIR"
|
||||
rsync -a --delete \
|
||||
--exclude '.git' --exclude '.venv' --exclude 'venv' \
|
||||
--exclude '__pycache__' --exclude '*.pyc' \
|
||||
--exclude '*.db' --exclude '.env' --exclude 'stations.json' \
|
||||
"${REPO_DIR}/" "${APP_DIR}/"
|
||||
else
|
||||
warn "rsync not found; falling back to cp (will not prune deleted files)"
|
||||
cp -r "${REPO_DIR}/." "${APP_DIR}/"
|
||||
--exclude '.git' --exclude '.venv' --exclude '__pycache__' --exclude '/rust/target' \
|
||||
--exclude '/.env' --exclude '/stations.json' --exclude '/bin' \
|
||||
--exclude '/collector-status.json' --exclude '/crontab-*.bak' \
|
||||
--exclude '/models/flood_*' --exclude '/models/metrics.json' \
|
||||
--exclude '/models/cache' --exclude '/models/.staging' --exclude '/models/.previous' \
|
||||
"$REPO_DIR/" "$APP_DIR/"
|
||||
fi
|
||||
|
||||
# 3. Build the uv-managed virtualenv ------------------------------------------
|
||||
# Prefer an already-installed uv. For stricter supply-chain control install uv
|
||||
# ahead of time via your distro / package manager; this script only fetches the
|
||||
# upstream installer (piped to a root shell) when AUTO_INSTALL_UV=1 is set, and
|
||||
# pins the version so the fetched script is reproducible.
|
||||
UV_VERSION="${UV_VERSION:-0.5.11}"
|
||||
if ! command -v uv >/dev/null 2>&1; then
|
||||
if [ "${AUTO_INSTALL_UV:-0}" = "1" ]; then
|
||||
warn "uv not found; installing pinned uv ${UV_VERSION} from astral.sh (runs as root)"
|
||||
curl -LsSf "https://astral.sh/uv/${UV_VERSION}/install.sh" \
|
||||
| env UV_INSTALL_DIR=/usr/local/bin sh
|
||||
# 3. binary -------------------------------------------------------------------
|
||||
BIN="${BIN:-${REPO_DIR}/rust/target/release/ping-collector}"
|
||||
[ -x "$BIN" ] || die "no binary at $BIN: build it (see the header) or pass BIN=/path/to/ping-collector"
|
||||
log "Installing $BIN -> ${APP_DIR}/bin/ping-collector ($("$BIN" --version))"
|
||||
install -D -m 0755 -o root -g root "$BIN" "${APP_DIR}/bin/ping-collector"
|
||||
|
||||
# 4. Python training environment ------------------------------------------------
|
||||
if [ "${SKIP_UV:-0}" != 1 ]; then
|
||||
uv_user=root # copy mode: handed to the service user below
|
||||
[ "$in_place" = 1 ] && uv_user="${SUDO_USER:-root}"
|
||||
log "Syncing the Python training environment (${APP_DIR}/.venv) as $uv_user"
|
||||
if [ "$uv_user" = root ]; then
|
||||
(cd "$APP_DIR" && uv sync --python 3.11 --frozen) ||
|
||||
warn "uv sync failed: the monthly retrain needs ${APP_DIR}/.venv (uv sync --python 3.11 --frozen)"
|
||||
else
|
||||
die "uv not found. Install it (e.g. your package manager, or 'pipx install uv'),
|
||||
or re-run with AUTO_INSTALL_UV=1 to fetch the pinned upstream installer."
|
||||
fi
|
||||
fi
|
||||
UV="$(command -v uv)"
|
||||
|
||||
log "Syncing uv-managed virtualenv at ${APP_DIR}/.venv"
|
||||
cd "${APP_DIR}"
|
||||
# ONE environment: uv sync owns .venv/ (from pyproject.toml + uv.lock, so the
|
||||
# ML extras such as scikit-learn/joblib are present) and both systemd units
|
||||
# run its interpreter directly. Never create a second env by another name --
|
||||
# a stale 'venv/' once coexisted here and broke manual retrains with
|
||||
# ModuleNotFoundError while the service itself ran fine.
|
||||
"${UV}" sync --python 3.11 --frozen
|
||||
if [ -d "${APP_DIR}/venv" ]; then
|
||||
warn "Removing stale ${APP_DIR}/venv (superseded by .venv)"
|
||||
rm -rf "${APP_DIR}/venv"
|
||||
fi
|
||||
|
||||
# 4. Environment file ----------------------------------------------------------
|
||||
if [ ! -f "${APP_DIR}/.env" ]; then
|
||||
if [ -f "${REPO_DIR}/.env" ]; then
|
||||
log "Copying .env from checkout"
|
||||
cp "${REPO_DIR}/.env" "${APP_DIR}/.env"
|
||||
else
|
||||
warn "No .env found. Copy .env.example to ${APP_DIR}/.env and fill in"
|
||||
warn "MATRIX_ACCESS_TOKEN / MATRIX_ROOM_ID and DB settings before starting."
|
||||
runuser -u "$uv_user" -- bash -lc "cd '$APP_DIR' && uv sync --python 3.11 --frozen" ||
|
||||
warn "uv sync as $uv_user failed: run 'uv sync --python 3.11 --frozen' in $APP_DIR as the owner of .venv"
|
||||
fi
|
||||
fi
|
||||
|
||||
# 5. Ownership and permissions -------------------------------------------------
|
||||
# Service user needs write access for logs / stations.json.
|
||||
log "Setting ownership to ${SERVICE_USER}:${SERVICE_GROUP}"
|
||||
chown -R "${SERVICE_USER}:${SERVICE_GROUP}" "${APP_DIR}"
|
||||
# Restrict traversal to root + the service user, and lock down the secrets file
|
||||
# (contains the Matrix token and DB credentials).
|
||||
chmod 0750 "${APP_DIR}"
|
||||
if [ -f "${APP_DIR}/.env" ]; then
|
||||
chmod 0600 "${APP_DIR}/.env"
|
||||
# 5. .env ---------------------------------------------------------------------
|
||||
fresh_env=0
|
||||
if [ ! -f "$ENV_FILE" ]; then
|
||||
cp "${APP_DIR}/.env.example" "$ENV_FILE"
|
||||
chmod 0600 "$ENV_FILE"
|
||||
fresh_env=1
|
||||
fi
|
||||
set_default WEB_LISTEN 0.0.0.0:8000
|
||||
set_default FORECAST_IN_COLLECTOR true
|
||||
|
||||
if [ "$in_place" = 0 ]; then
|
||||
# a copy belongs to the service (it writes models/, stations.json and the
|
||||
# status file); an in-place checkout keeps its owner (git pull, builds)
|
||||
chown -R "${SERVICE_USER}:${SERVICE_GROUP}" "$APP_DIR"
|
||||
chown root:root "${APP_DIR}/bin/ping-collector"
|
||||
chmod 0750 "$APP_DIR"
|
||||
fi
|
||||
|
||||
# 6. Install and enable the systemd units -------------------------------------
|
||||
# 6. units --------------------------------------------------------------------
|
||||
log "Installing systemd units"
|
||||
install -m 0644 "${SCRIPT_DIR}/${SERVICE_NAME}" "/etc/systemd/system/${SERVICE_NAME}"
|
||||
install -m 0644 "${SCRIPT_DIR}/${RETRAIN_NAME}.service" "/etc/systemd/system/${RETRAIN_NAME}.service"
|
||||
install -m 0644 "${SCRIPT_DIR}/${RETRAIN_NAME}.timer" "/etc/systemd/system/${RETRAIN_NAME}.timer"
|
||||
install -m 0644 "${APP_DIR}/scripts/ping-collector.service" /etc/systemd/system/ping-collector.service
|
||||
install -m 0644 "${APP_DIR}/scripts/${RETRAIN}.service" "/etc/systemd/system/${RETRAIN}.service"
|
||||
install -m 0644 "${APP_DIR}/scripts/${RETRAIN}.timer" "/etc/systemd/system/${RETRAIN}.timer"
|
||||
if [ -f /etc/systemd/system/water-monitor.service ]; then
|
||||
log "Removing the retired Python web app (water-monitor.service)"
|
||||
systemctl disable --now water-monitor.service 2>/dev/null || true
|
||||
rm -f /etc/systemd/system/water-monitor.service
|
||||
fi
|
||||
systemctl daemon-reload
|
||||
systemctl enable "${SERVICE_NAME}"
|
||||
systemctl enable --now "${RETRAIN_NAME}.timer"
|
||||
systemctl enable ping-collector.service
|
||||
systemctl enable --now "${RETRAIN}.timer"
|
||||
|
||||
log "Done."
|
||||
# The old Matrix cron job (`run.py --alert-check`, repeating an alert every
|
||||
# 15 minutes while a level stayed high): ping-collector sends Matrix now.
|
||||
for user in root "$SERVICE_USER"; do
|
||||
crontab -l -u "$user" 2>/dev/null | grep -v '^\s*#' | grep -q -- '--alert-check' || continue
|
||||
backup="${APP_DIR}/crontab-${user}.$(date +%Y%m%d%H%M%S).bak"
|
||||
crontab -l -u "$user" >"$backup"
|
||||
sed '/--alert-check/ s|^\([^#]\)|# retired: Matrix is sent by ping-collector # \1|' "$backup" |
|
||||
crontab -u "$user" -
|
||||
log "Commented out the old --alert-check cron job of $user (backup: $backup)"
|
||||
done
|
||||
if grep -rlqs -- '--alert-check' /etc/cron.d /etc/crontab 2>/dev/null; then
|
||||
warn "an --alert-check job is still in $(grep -rls -- '--alert-check' /etc/cron.d /etc/crontab | tr '\n' ' '): remove it"
|
||||
fi
|
||||
|
||||
if [ "$fresh_env" = 1 ]; then
|
||||
warn "created $ENV_FILE from .env.example: set the database (POSTGRES_*), NTFY_* and MATRIX_*, then re-run"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# 7. checks -------------------------------------------------------------------
|
||||
if [ "${SKIP_CHECKS:-0}" != 1 ]; then
|
||||
log "Dry run of every source against the configured database (writes nothing)"
|
||||
as_service "${APP_DIR}/bin/ping-collector" once --dry-run
|
||||
if compgen -G "${APP_DIR}/models/flood_*.joblib" >/dev/null && [ -x "$PYTHON" ]; then
|
||||
log "Exporting trained models for the forecaster (models/flood_*.json.gz)"
|
||||
as_service "$PYTHON" -m src.ml.export
|
||||
log "Forecast check: ping-collector against the Python reference, same data, moment and rain"
|
||||
as_service "$PYTHON" scripts/forecast_parity.py ||
|
||||
warn "the forecast check found differences (see above); the service runs regardless"
|
||||
else
|
||||
warn "no trained models or no ${PYTHON}: forecast check skipped (the forecast falls back to the heuristic)"
|
||||
fi
|
||||
fi
|
||||
|
||||
# 8. (re)start ----------------------------------------------------------------
|
||||
systemctl restart ping-collector.service
|
||||
listen="$(grep '^WEB_LISTEN=' "$ENV_FILE" | tail -1 | cut -d= -f2- | tr -d '"'"'"' ')"
|
||||
if [ -n "$listen" ]; then
|
||||
port="${listen##*:}"
|
||||
# readiness: a route that answers 200 whatever the data (/health may be 503)
|
||||
if wait_http "http://127.0.0.1:${port}/api/notifications" 60; then
|
||||
log "Website and API answer on :${port} ($(curl -s --max-time 5 "http://127.0.0.1:${port}/scraping/status" | grep -o '"collector":"[^"]*"'))"
|
||||
else
|
||||
warn "no answer on port ${port} yet: journalctl -u ping-collector"
|
||||
fi
|
||||
fi
|
||||
sleep 3
|
||||
systemctl --no-pager --lines=0 status ping-collector.service || true
|
||||
journalctl -u ping-collector -n 25 --no-pager || true
|
||||
echo
|
||||
echo "Next steps:"
|
||||
echo " sudo systemctl start ${SERVICE_NAME}"
|
||||
echo " systemctl status ${SERVICE_NAME}"
|
||||
echo " sudo journalctl -u ${SERVICE_NAME} -f"
|
||||
echo " systemctl list-timers ${RETRAIN_NAME}.timer # monthly flood-model retrain"
|
||||
echo " sudo systemctl start ${RETRAIN_NAME}.service # retrain now"
|
||||
log "Done. Logs: journalctl -u ping-collector -f; retrain: systemctl list-timers ${RETRAIN}.timer"
|
||||
@@ -1,286 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Install the Rust service (rust/collector, `ping-collector`) and move work
|
||||
# from the Python app to it, or back.
|
||||
#
|
||||
# sudo bash scripts/install_collector.sh # install binary + unit; checks only
|
||||
# sudo bash scripts/install_collector.sh --cutover # Rust collects (COLLECTION_ENABLED=false)
|
||||
# sudo bash scripts/install_collector.sh --cutover-forecast # Rust also forecasts and notifies:
|
||||
# # ntfy + Matrix (FORECAST_IN_COLLECTOR=true)
|
||||
# sudo bash scripts/install_collector.sh --check-api # compare the Rust website/API with the
|
||||
# # Python app's, change nothing
|
||||
# sudo bash scripts/install_collector.sh --cutover-api # Rust serves the website and API on
|
||||
# # port 8000 (WEB_LISTEN); the Python app stops
|
||||
# sudo bash scripts/install_collector.sh --rollback-api # the Python app serves the website again
|
||||
# sudo bash scripts/install_collector.sh --rollback-forecast # forecasts + notifications back to Python
|
||||
# sudo bash scripts/install_collector.sh --rollback # Python does everything again
|
||||
#
|
||||
# Every install also exports the trained models for the Rust forecaster and
|
||||
# compares its forecast with Python's on the live data
|
||||
# (scripts/forecast_parity.py); --cutover-forecast refuses to switch when they
|
||||
# differ. --cutover-api first runs `ping-collector serve` next to the Python
|
||||
# app and compares every route (scripts/api_parity.py); it refuses to switch
|
||||
# when they differ. FORCE=1 overrides either check.
|
||||
#
|
||||
# Once ping-collector sends the Matrix messages (FORECAST_IN_COLLECTOR=true
|
||||
# and MATRIX_* set), the old `run.py --alert-check` cron job is commented out
|
||||
# (backup next to the app); it repeated an alert on every run.
|
||||
#
|
||||
# The binary comes from BIN=/path/to/ping-collector (e.g. the "ping-collector-
|
||||
# linux-x86_64" CI artifact) or from rust/target/release/ping-collector, built
|
||||
# beforehand as a normal user:
|
||||
# cargo build --release --locked --manifest-path rust/Cargo.toml
|
||||
# (needs Rust >= 1.88 from rustup -- distro cargo is too old -- plus pkg-config
|
||||
# and libssl-dev; see rust/README.md).
|
||||
set -euo pipefail
|
||||
|
||||
APP_DIR="${APP_DIR:-/opt/thailand-water-monitor}"
|
||||
SERVICE_USER="${SERVICE_USER:-water-monitor}" # as in scripts/ping-collector.service
|
||||
ENV_FILE="${APP_DIR}/.env"
|
||||
STATUS_FILE="${APP_DIR}/collector-status.json"
|
||||
PYTHON="${APP_DIR}/.venv/bin/python"
|
||||
REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
WEB_ADDR="${WEB_ADDR:-0.0.0.0:8000}" # where the Python app listens today
|
||||
CHECK_PORT="${CHECK_PORT:-8001}" # the Rust side during --check-api
|
||||
MODE="${1:-install}"
|
||||
|
||||
log() { printf '\033[1;32m==>\033[0m %s\n' "$*"; }
|
||||
warn() { printf '\033[1;33m[warn]\033[0m %s\n' "$*" >&2; }
|
||||
die() { printf '\033[1;31m[error]\033[0m %s\n' "$*" >&2; exit 1; }
|
||||
|
||||
[ "$(id -u)" -eq 0 ] || die "run as root (sudo)"
|
||||
[ -f "$ENV_FILE" ] || die "$ENV_FILE not found"
|
||||
|
||||
set_env() {
|
||||
local key="$1" value="$2"
|
||||
if grep -q "^${key}=" "$ENV_FILE"; then
|
||||
sed -i "s|^${key}=.*|${key}=${value}|" "$ENV_FILE"
|
||||
else
|
||||
printf '%s=%s\n' "$key" "$value" >>"$ENV_FILE"
|
||||
fi
|
||||
}
|
||||
|
||||
# Run a command the way the services run: same user, working directory and
|
||||
# EnvironmentFile parsing (systemd's, not a .env library's).
|
||||
as_service() {
|
||||
if systemd-run --version >/dev/null 2>&1; then
|
||||
systemd-run --quiet --pipe --wait --collect \
|
||||
-p User="$SERVICE_USER" -p Group="$(id -gn "$SERVICE_USER")" \
|
||||
-p WorkingDirectory="$APP_DIR" -p EnvironmentFile="$ENV_FILE" "$@"
|
||||
else
|
||||
(cd "$APP_DIR" && runuser -u "$SERVICE_USER" -- "$@")
|
||||
fi
|
||||
}
|
||||
|
||||
show() {
|
||||
sleep "$1"
|
||||
systemctl --no-pager --lines=0 status ping-collector.service || true
|
||||
journalctl -u ping-collector -n 25 --no-pager || true
|
||||
}
|
||||
|
||||
env_is() { grep -q "^$1=$2\s*$" "$ENV_FILE"; }
|
||||
|
||||
wait_http() { # url seconds
|
||||
local i
|
||||
for i in $(seq "$2"); do
|
||||
curl -fs -o /dev/null --max-time 5 "$1" 2>/dev/null && return 0
|
||||
sleep 1
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# The Matrix messages come from ping-collector once it notifies; the old
|
||||
# cron job (`run.py --alert-check`, every 15 min, repeating while a level
|
||||
# stays high) is commented out, with a backup of the crontab.
|
||||
retire_matrix_cron() {
|
||||
env_is FORECAST_IN_COLLECTOR true || return 0
|
||||
grep -q '^MATRIX_ACCESS_TOKEN=..*' "$ENV_FILE" || return 0
|
||||
local user backup
|
||||
for user in root "$SERVICE_USER"; do
|
||||
crontab -l -u "$user" 2>/dev/null | grep -v '^\s*#' | grep -q -- '--alert-check' || continue
|
||||
backup="${APP_DIR}/crontab-${user}.$(date +%Y%m%d%H%M%S).bak"
|
||||
crontab -l -u "$user" >"$backup"
|
||||
sed '/--alert-check/ s|^\([^#]\)|# retired: Matrix is sent by ping-collector # \1|' "$backup" |
|
||||
crontab -u "$user" -
|
||||
log "Matrix: commented out the old --alert-check cron job of $user (backup: $backup)"
|
||||
done
|
||||
if grep -rlqs -- '--alert-check' /etc/cron.d /etc/crontab 2>/dev/null; then
|
||||
warn "an --alert-check job is still in $(grep -rls -- '--alert-check' /etc/cron.d /etc/crontab | tr '\n' ' '): remove it, ping-collector sends Matrix now"
|
||||
fi
|
||||
if systemctl list-timers --all --no-pager 2>/dev/null | grep -qi alert; then
|
||||
warn "a systemd timer mentions 'alert' (systemctl list-timers --all): if it runs --alert-check, disable it"
|
||||
fi
|
||||
}
|
||||
|
||||
# ping-collector serve next to the Python app; every route compared.
|
||||
api_check() {
|
||||
local unit=ping-api-check ok=0
|
||||
systemctl stop "$unit" 2>/dev/null || true
|
||||
systemd-run --quiet --unit="$unit" --collect \
|
||||
-p User="$SERVICE_USER" -p Group="$(id -gn "$SERVICE_USER")" \
|
||||
-p WorkingDirectory="$APP_DIR" -p EnvironmentFile="$ENV_FILE" \
|
||||
"${APP_DIR}/bin/ping-collector" serve --listen "127.0.0.1:${CHECK_PORT}" --no-analytics
|
||||
if wait_http "http://127.0.0.1:${CHECK_PORT}/api/notifications" 30; then
|
||||
as_service "$PYTHON" scripts/api_parity.py \
|
||||
--python "http://127.0.0.1:${WEB_ADDR##*:}" --rust "http://127.0.0.1:${CHECK_PORT}" && ok=1
|
||||
else
|
||||
warn "ping-collector serve did not answer on port ${CHECK_PORT}:"
|
||||
journalctl -u "$unit" -n 20 --no-pager || true
|
||||
fi
|
||||
systemctl stop "$unit" 2>/dev/null || true
|
||||
[ "$ok" = 1 ]
|
||||
}
|
||||
|
||||
rollback_api() {
|
||||
if env_is WEB_LISTEN '..*'; then
|
||||
log "The Python app serves the website and API again"
|
||||
set_env WEB_LISTEN ""
|
||||
systemctl restart ping-collector.service
|
||||
fi
|
||||
systemctl enable water-monitor.service
|
||||
systemctl restart water-monitor.service
|
||||
}
|
||||
|
||||
matrix_backups() {
|
||||
ls "${APP_DIR}"/crontab-*.bak 2>/dev/null | sed 's/^/ /' || true
|
||||
}
|
||||
|
||||
case "$MODE" in
|
||||
--rollback)
|
||||
log "Stopping ping-collector; the Python app collects, serves, forecasts and notifies again"
|
||||
systemctl disable --now ping-collector.service 2>/dev/null || true
|
||||
set_env WEB_LISTEN ""
|
||||
set_env FORECAST_IN_COLLECTOR false
|
||||
set_env COLLECTION_ENABLED true
|
||||
systemctl enable water-monitor.service
|
||||
systemctl restart water-monitor.service
|
||||
if [ -n "$(matrix_backups)" ]; then
|
||||
warn "Matrix alerts came from ping-collector; to restore the old cron job: crontab -u USER BACKUP"
|
||||
matrix_backups
|
||||
fi
|
||||
log "Done. Check: journalctl -u water-monitor -n 50"
|
||||
exit 0
|
||||
;;
|
||||
--rollback-api)
|
||||
rollback_api
|
||||
log "Done. Check: journalctl -u water-monitor -n 50; curl -s http://127.0.0.1:${WEB_ADDR##*:}/health"
|
||||
exit 0
|
||||
;;
|
||||
--rollback-forecast)
|
||||
# the Python app forecasts in its own process: it must serve again
|
||||
rollback_api
|
||||
log "Forecasts and notifications back to the Python app; ping-collector keeps collecting"
|
||||
set_env FORECAST_IN_COLLECTOR false
|
||||
systemctl restart ping-collector.service water-monitor.service
|
||||
if [ -n "$(matrix_backups)" ]; then
|
||||
warn "Matrix alerts stop with this (the Python app sends ntfy only); old cron job: crontab -u USER BACKUP"
|
||||
matrix_backups
|
||||
fi
|
||||
log "Done. Check: journalctl -u water-monitor -n 50"
|
||||
exit 0
|
||||
;;
|
||||
--check-api)
|
||||
[ -x "${APP_DIR}/bin/ping-collector" ] || die "install first"
|
||||
if env_is WEB_LISTEN '..*'; then
|
||||
die "ping-collector already serves the website (WEB_LISTEN is set)"
|
||||
fi
|
||||
api_check && log "The Rust API answers as the Python app" || die "the API check found differences"
|
||||
exit 0
|
||||
;;
|
||||
install | --cutover | --cutover-forecast | --cutover-api) ;;
|
||||
*) die "unknown option: $MODE" ;;
|
||||
esac
|
||||
|
||||
BIN="${BIN:-${REPO_DIR}/rust/target/release/ping-collector}"
|
||||
[ -x "$BIN" ] || die "no binary at $BIN: build it (see header) or pass BIN=/path/to/ping-collector"
|
||||
|
||||
log "Installing $BIN -> ${APP_DIR}/bin/ping-collector ($("$BIN" --version))"
|
||||
install -D -m 0755 -o root -g root "$BIN" "${APP_DIR}/bin/ping-collector"
|
||||
install -m 0644 "${REPO_DIR}/scripts/ping-collector.service" /etc/systemd/system/ping-collector.service
|
||||
systemctl daemon-reload
|
||||
|
||||
# Both services read and write /scraping/status through this file (inside
|
||||
# the app dir: the units use PrivateTmp).
|
||||
set_env SCRAPING_STATUS_FILE "$STATUS_FILE"
|
||||
touch "$STATUS_FILE"
|
||||
chown "${SERVICE_USER}:$(id -gn "$SERVICE_USER")" "$STATUS_FILE"
|
||||
|
||||
log "Dry run against the configured database (writes nothing)"
|
||||
as_service "${APP_DIR}/bin/ping-collector" once --dry-run
|
||||
|
||||
forecast_ok=1
|
||||
if compgen -G "${APP_DIR}/models/flood_*.joblib" >/dev/null; then
|
||||
log "Exporting the trained models for the Rust forecaster (models/flood_*.json.gz)"
|
||||
as_service "$PYTHON" -m src.ml.export
|
||||
log "Forecast check: Python and ping-collector on the same data, moment and rain"
|
||||
as_service "$PYTHON" scripts/forecast_parity.py || forecast_ok=0
|
||||
else
|
||||
warn "no trained models in ${APP_DIR}/models: forecast check skipped"
|
||||
forecast_ok=0
|
||||
fi
|
||||
|
||||
case "$MODE" in
|
||||
--cutover)
|
||||
log "Cutover: ping-collector collects; the Python app forecasts and notifies"
|
||||
set_env COLLECTION_ENABLED false
|
||||
systemctl enable ping-collector.service
|
||||
systemctl restart ping-collector.service water-monitor.service
|
||||
show 20
|
||||
log "Next step: --cutover-forecast. Rollback: sudo bash scripts/install_collector.sh --rollback"
|
||||
;;
|
||||
--cutover-forecast)
|
||||
grep -q '^COLLECTION_ENABLED=false' "$ENV_FILE" ||
|
||||
die "ping-collector must collect first: run --cutover"
|
||||
if [ "$forecast_ok" != 1 ] && [ "${FORCE:-0}" != 1 ]; then
|
||||
die "the forecast check did not pass; not switching (FORCE=1 overrides)"
|
||||
fi
|
||||
log "Cutover: ping-collector also forecasts and notifies; the Python app serves its forecast"
|
||||
set_env FORECAST_IN_COLLECTOR true
|
||||
retire_matrix_cron
|
||||
systemctl enable ping-collector.service
|
||||
systemctl restart ping-collector.service water-monitor.service
|
||||
show 30
|
||||
log "Next step: --cutover-api. Rollback: sudo bash scripts/install_collector.sh --rollback-forecast"
|
||||
;;
|
||||
--cutover-api)
|
||||
env_is FORECAST_IN_COLLECTOR true || die "ping-collector must forecast first: run --cutover-forecast"
|
||||
if env_is WEB_LISTEN '..*'; then
|
||||
log "ping-collector already serves the website; restarting it on the new binary"
|
||||
retire_matrix_cron
|
||||
systemctl restart ping-collector.service
|
||||
show 10
|
||||
exit 0
|
||||
fi
|
||||
api_ok=1
|
||||
if [ "$(systemctl is-active water-monitor.service 2>/dev/null)" = active ]; then
|
||||
log "API check: ping-collector serve on :${CHECK_PORT} next to the Python app, every route compared"
|
||||
api_check || api_ok=0
|
||||
else
|
||||
warn "the Python app is not running: nothing to compare the API with"
|
||||
api_ok=0
|
||||
fi
|
||||
if [ "$api_ok" != 1 ] && [ "${FORCE:-0}" != 1 ]; then
|
||||
die "the API check did not pass; not switching (FORCE=1 overrides)"
|
||||
fi
|
||||
log "Cutover: ping-collector serves the website and API on ${WEB_ADDR}; the Python app stops"
|
||||
set_env WEB_LISTEN "$WEB_ADDR"
|
||||
retire_matrix_cron
|
||||
systemctl disable --now water-monitor.service
|
||||
systemctl restart ping-collector.service
|
||||
if wait_http "http://127.0.0.1:${WEB_ADDR##*:}/health" 60; then
|
||||
log "The website answers from ping-collector: $(curl -s --max-time 5 "http://127.0.0.1:${WEB_ADDR##*:}/scraping/status" | grep -o '"collector":"[^"]*"')"
|
||||
else
|
||||
warn "no answer on port ${WEB_ADDR##*:} yet; check journalctl -u ping-collector"
|
||||
fi
|
||||
show 5
|
||||
log "Rollback if needed: sudo bash scripts/install_collector.sh --rollback-api"
|
||||
;;
|
||||
*)
|
||||
retire_matrix_cron
|
||||
if [ "$(systemctl is-active ping-collector.service 2>/dev/null)" = active ]; then
|
||||
systemctl restart ping-collector.service
|
||||
log "Installed; ping-collector restarted on the new binary"
|
||||
else
|
||||
log "Installed, not started. Switch over with: sudo bash scripts/install_collector.sh --cutover"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
@@ -92,7 +92,7 @@ if [[ -f "$env_file" ]] && ! grep -q '^NTFY_SERVER=' "$env_file"; then
|
||||
echo "NTFY_TOPIC_PREFIX=ping"
|
||||
echo "NTFY_TOKEN=${token}"
|
||||
} >> "$env_file"
|
||||
echo "wrote NTFY_* to ${env_file}; restart water-monitor to enable"
|
||||
echo "wrote NTFY_* to ${env_file}; restart ping-collector to enable"
|
||||
else
|
||||
echo "NTFY_TOKEN=${token}"
|
||||
fi
|
||||
|
||||
@@ -9,7 +9,7 @@ and errors per stage plus the slowest endpoints.
|
||||
Run against a LOCAL instance for full stress (never full-stress production —
|
||||
it hosts live flood monitoring):
|
||||
|
||||
python -m uvicorn src.web_api:app --port 8125 # separate shell
|
||||
rust/target/release/ping-collector serve --listen 127.0.0.1:8125 --no-analytics # separate shell
|
||||
python scripts/load_test.py http://localhost:8125
|
||||
|
||||
A gentle production baseline (low, fixed concurrency):
|
||||
@@ -98,9 +98,7 @@ def run_stage(base, clients, seconds):
|
||||
by_endpoint = {}
|
||||
for path, elapsed in results:
|
||||
by_endpoint.setdefault(path, []).append(elapsed)
|
||||
slowest = sorted(
|
||||
by_endpoint.items(), key=lambda kv: -statistics.median(kv[1])
|
||||
)[:4]
|
||||
slowest = sorted(by_endpoint.items(), key=lambda kv: -statistics.median(kv[1]))[:4]
|
||||
for path, values in slowest:
|
||||
print(
|
||||
f" slow: {path:45} n={len(values):5} "
|
||||
|
||||
@@ -7,7 +7,8 @@ the page makes, polling like the auto-refresh does) and API consumers
|
||||
Full stress against a LOCAL instance (never full-stress production — it hosts
|
||||
live flood monitoring):
|
||||
|
||||
# separate shell: python -m uvicorn src.web_api:app --port 8125
|
||||
# separate shell:
|
||||
# rust/target/release/ping-collector serve --listen 127.0.0.1:8125 --no-analytics
|
||||
.venv/Scripts/python.exe -m locust -f scripts/locustfile.py \
|
||||
--host http://localhost:8125 --headless \
|
||||
--users 200 --spawn-rate 20 --run-time 2m \
|
||||
@@ -77,9 +78,11 @@ class ApiConsumer(FastHttpUser):
|
||||
|
||||
@task(3)
|
||||
def hii_feeds(self):
|
||||
self.client.get(random.choice(
|
||||
["/api/hii/waterlevel/latest", "/api/hii/rainfall/latest"]
|
||||
), headers=GZIP, name="/api/hii/[feed]/latest")
|
||||
self.client.get(
|
||||
random.choice(["/api/hii/waterlevel/latest", "/api/hii/rainfall/latest"]),
|
||||
headers=GZIP,
|
||||
name="/api/hii/[feed]/latest",
|
||||
)
|
||||
|
||||
@task(2)
|
||||
def forecast(self):
|
||||
|
||||
@@ -1,619 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
SQLite to PostgreSQL Migration Tool
|
||||
Migrates all data from SQLite database to PostgreSQL
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
import sqlite3
|
||||
from datetime import datetime, timezone
|
||||
from typing import Dict, List, Optional, Tuple, Any
|
||||
from dataclasses import dataclass
|
||||
|
||||
# Add src to path for imports
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src'))
|
||||
|
||||
@dataclass
|
||||
class MigrationStats:
|
||||
stations_migrated: int = 0
|
||||
measurements_migrated: int = 0
|
||||
errors: List[str] = None
|
||||
start_time: Optional[datetime] = None
|
||||
end_time: Optional[datetime] = None
|
||||
|
||||
def __post_init__(self):
|
||||
if self.errors is None:
|
||||
self.errors = []
|
||||
|
||||
class SQLiteToPostgresMigrator:
|
||||
def __init__(self, sqlite_path: str, postgres_config: Dict[str, Any]):
|
||||
self.sqlite_path = sqlite_path
|
||||
self.postgres_config = postgres_config
|
||||
self.sqlite_conn = None
|
||||
self.postgres_adapter = None
|
||||
self.stats = MigrationStats()
|
||||
|
||||
# Setup logging with UTF-8 encoding
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(levelname)s - %(message)s',
|
||||
handlers=[
|
||||
logging.StreamHandler(),
|
||||
logging.FileHandler('migration.log', encoding='utf-8')
|
||||
]
|
||||
)
|
||||
self.logger = logging.getLogger(__name__)
|
||||
|
||||
def connect_databases(self) -> bool:
|
||||
"""Connect to both SQLite and PostgreSQL databases"""
|
||||
try:
|
||||
# Connect to SQLite
|
||||
if not os.path.exists(self.sqlite_path):
|
||||
self.logger.error(f"SQLite database not found: {self.sqlite_path}")
|
||||
return False
|
||||
|
||||
self.sqlite_conn = sqlite3.connect(self.sqlite_path)
|
||||
self.sqlite_conn.row_factory = sqlite3.Row # For dict-like access
|
||||
self.logger.info(f"Connected to SQLite database: {self.sqlite_path}")
|
||||
|
||||
# Connect to PostgreSQL
|
||||
from database_adapters import create_database_adapter
|
||||
self.postgres_adapter = create_database_adapter(
|
||||
self.postgres_config['type'],
|
||||
connection_string=self.postgres_config['connection_string']
|
||||
)
|
||||
|
||||
if not self.postgres_adapter.connect():
|
||||
self.logger.error("Failed to connect to PostgreSQL")
|
||||
return False
|
||||
|
||||
self.logger.info("Connected to PostgreSQL database")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Database connection error: {e}")
|
||||
return False
|
||||
|
||||
def analyze_sqlite_schema(self) -> Dict[str, List[str]]:
|
||||
"""Analyze SQLite database structure"""
|
||||
try:
|
||||
cursor = self.sqlite_conn.cursor()
|
||||
|
||||
# Get all tables
|
||||
cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'")
|
||||
tables = [row[0] for row in cursor.fetchall()]
|
||||
|
||||
schema_info = {}
|
||||
for table in tables:
|
||||
cursor.execute(f"PRAGMA table_info({table})")
|
||||
columns = [row[1] for row in cursor.fetchall()]
|
||||
schema_info[table] = columns
|
||||
|
||||
# Get row count
|
||||
cursor.execute(f"SELECT COUNT(*) FROM {table}")
|
||||
count = cursor.fetchone()[0]
|
||||
self.logger.info(f"Table '{table}': {len(columns)} columns, {count} rows")
|
||||
|
||||
return schema_info
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Schema analysis error: {e}")
|
||||
return {}
|
||||
|
||||
def migrate_stations(self) -> bool:
|
||||
"""Migrate station data"""
|
||||
try:
|
||||
cursor = self.sqlite_conn.cursor()
|
||||
|
||||
# Try different possible table names and structures
|
||||
station_queries = [
|
||||
# Modern structure
|
||||
"""SELECT id, station_code, station_name_th as thai_name, station_name_en as english_name,
|
||||
latitude, longitude, geohash, created_at, updated_at
|
||||
FROM stations""",
|
||||
|
||||
# Alternative structure 1
|
||||
"""SELECT id, station_code, thai_name, english_name,
|
||||
latitude, longitude, geohash, created_at, updated_at
|
||||
FROM stations""",
|
||||
|
||||
# Legacy structure
|
||||
"""SELECT station_id as id, station_code, station_name as thai_name,
|
||||
station_name as english_name, lat as latitude, lon as longitude,
|
||||
NULL as geohash, datetime('now') as created_at, datetime('now') as updated_at
|
||||
FROM water_stations""",
|
||||
|
||||
# Simple structure
|
||||
"""SELECT rowid as id, station_code, name as thai_name, name as english_name,
|
||||
NULL as latitude, NULL as longitude, NULL as geohash,
|
||||
datetime('now') as created_at, datetime('now') as updated_at
|
||||
FROM stations""",
|
||||
]
|
||||
|
||||
stations_data = []
|
||||
|
||||
for query in station_queries:
|
||||
try:
|
||||
cursor.execute(query)
|
||||
rows = cursor.fetchall()
|
||||
if rows:
|
||||
self.logger.info(f"Found {len(rows)} stations using query variant")
|
||||
|
||||
for row in rows:
|
||||
station = {
|
||||
'station_id': row[0],
|
||||
'station_code': row[1] or f"STATION_{row[0]}",
|
||||
'station_name_th': row[2] or f"Station {row[0]}",
|
||||
'station_name_en': row[3] or f"Station {row[0]}",
|
||||
'latitude': row[4],
|
||||
'longitude': row[5],
|
||||
'geohash': row[6],
|
||||
'status': 'active'
|
||||
}
|
||||
stations_data.append(station)
|
||||
break
|
||||
|
||||
except sqlite3.OperationalError as e:
|
||||
if "no such table" in str(e).lower() or "no such column" in str(e).lower():
|
||||
continue
|
||||
else:
|
||||
raise
|
||||
|
||||
if not stations_data:
|
||||
self.logger.warning("No stations found in SQLite database")
|
||||
return True
|
||||
|
||||
# Insert stations into PostgreSQL using raw SQL
|
||||
# Since the adapter is designed for measurements, we'll use direct SQL
|
||||
try:
|
||||
from sqlalchemy import create_engine, text
|
||||
engine = create_engine(self.postgres_config['connection_string'])
|
||||
|
||||
# Process stations individually to avoid transaction rollback issues
|
||||
for station in stations_data:
|
||||
try:
|
||||
with engine.begin() as conn:
|
||||
# Use PostgreSQL UPSERT syntax with correct column names
|
||||
station_sql = """
|
||||
INSERT INTO stations (id, station_code, thai_name, english_name, latitude, longitude, geohash)
|
||||
VALUES (:station_id, :station_code, :thai_name, :english_name, :latitude, :longitude, :geohash)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
thai_name = EXCLUDED.thai_name,
|
||||
english_name = EXCLUDED.english_name,
|
||||
latitude = EXCLUDED.latitude,
|
||||
longitude = EXCLUDED.longitude,
|
||||
geohash = EXCLUDED.geohash,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
"""
|
||||
|
||||
conn.execute(text(station_sql), {
|
||||
'station_id': station['station_id'],
|
||||
'station_code': station['station_code'],
|
||||
'thai_name': station['station_name_th'],
|
||||
'english_name': station['station_name_en'],
|
||||
'latitude': station.get('latitude'),
|
||||
'longitude': station.get('longitude'),
|
||||
'geohash': station.get('geohash')
|
||||
})
|
||||
|
||||
self.stats.stations_migrated += 1
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Error migrating station {station.get('station_code', 'unknown')}: {str(e)[:100]}..."
|
||||
self.logger.warning(error_msg)
|
||||
self.stats.errors.append(error_msg)
|
||||
|
||||
self.logger.info(f"Migrated {self.stats.stations_migrated} stations")
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Station migration failed: {e}")
|
||||
return False
|
||||
|
||||
self.logger.info(f"Migrated {self.stats.stations_migrated} stations")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Station migration error: {e}")
|
||||
return False
|
||||
|
||||
def migrate_measurements(self, batch_size: int = 5000) -> bool:
|
||||
"""Migrate measurement data in batches"""
|
||||
try:
|
||||
cursor = self.sqlite_conn.cursor()
|
||||
|
||||
# Try different possible measurement table structures
|
||||
measurement_queries = [
|
||||
# Modern structure
|
||||
"""SELECT w.timestamp, w.station_id, s.station_code, s.station_name_th, s.station_name_en,
|
||||
w.water_level, w.discharge, w.discharge_percent, w.status
|
||||
FROM water_measurements w
|
||||
JOIN stations s ON w.station_id = s.id
|
||||
ORDER BY w.timestamp""",
|
||||
|
||||
# Alternative with different join
|
||||
"""SELECT w.timestamp, w.station_id, s.station_code, s.thai_name, s.english_name,
|
||||
w.water_level, w.discharge, w.discharge_percent, 'active' as status
|
||||
FROM water_measurements w
|
||||
JOIN stations s ON w.station_id = s.id
|
||||
ORDER BY w.timestamp""",
|
||||
|
||||
# Legacy structure
|
||||
"""SELECT timestamp, station_id, station_code, station_name, station_name,
|
||||
water_level, discharge, discharge_percent, 'active' as status
|
||||
FROM measurements
|
||||
ORDER BY timestamp""",
|
||||
|
||||
# Simple structure without joins
|
||||
"""SELECT timestamp, station_id, 'UNKNOWN' as station_code, 'Unknown' as station_name_th, 'Unknown' as station_name_en,
|
||||
water_level, discharge, discharge_percent, 'active' as status
|
||||
FROM water_measurements
|
||||
ORDER BY timestamp""",
|
||||
]
|
||||
|
||||
measurements_processed = 0
|
||||
|
||||
for query in measurement_queries:
|
||||
try:
|
||||
# Get total count first
|
||||
count_query = query.replace("SELECT", "SELECT COUNT(*) FROM (SELECT").replace("ORDER BY w.timestamp", "") + ")"
|
||||
cursor.execute(count_query)
|
||||
total_measurements = cursor.fetchone()[0]
|
||||
|
||||
if total_measurements == 0:
|
||||
continue
|
||||
|
||||
self.logger.info(f"Found {total_measurements} measurements to migrate")
|
||||
|
||||
# Process in batches
|
||||
offset = 0
|
||||
while True:
|
||||
batch_query = f"{query} LIMIT {batch_size} OFFSET {offset}"
|
||||
cursor.execute(batch_query)
|
||||
rows = cursor.fetchall()
|
||||
|
||||
if not rows:
|
||||
break
|
||||
|
||||
# Convert to measurement format
|
||||
measurements = []
|
||||
for row in rows:
|
||||
try:
|
||||
# Parse timestamp
|
||||
timestamp_str = row[0]
|
||||
if isinstance(timestamp_str, str):
|
||||
try:
|
||||
timestamp = datetime.fromisoformat(timestamp_str.replace('Z', '+00:00'))
|
||||
except:
|
||||
# Try other common formats
|
||||
for fmt in ['%Y-%m-%d %H:%M:%S', '%Y-%m-%d %H:%M:%S.%f', '%Y-%m-%dT%H:%M:%S']:
|
||||
try:
|
||||
timestamp = datetime.strptime(timestamp_str, fmt)
|
||||
break
|
||||
except:
|
||||
continue
|
||||
else:
|
||||
timestamp = datetime.now()
|
||||
else:
|
||||
timestamp = timestamp_str
|
||||
|
||||
measurement = {
|
||||
'timestamp': timestamp,
|
||||
'station_id': row[1] or 999,
|
||||
'station_code': row[2] or 'UNKNOWN',
|
||||
'station_name_th': row[3] or 'Unknown',
|
||||
'station_name_en': row[4] or 'Unknown',
|
||||
'water_level': float(row[5]) if row[5] is not None else None,
|
||||
'discharge': float(row[6]) if row[6] is not None else None,
|
||||
'discharge_percent': float(row[7]) if row[7] is not None else None,
|
||||
'status': row[8] or 'active'
|
||||
}
|
||||
measurements.append(measurement)
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Error processing measurement row: {e}"
|
||||
self.logger.warning(error_msg)
|
||||
continue
|
||||
|
||||
# Save batch to PostgreSQL using fast bulk insert
|
||||
if measurements:
|
||||
try:
|
||||
self._fast_bulk_insert(measurements)
|
||||
measurements_processed += len(measurements)
|
||||
self.stats.measurements_migrated += len(measurements)
|
||||
self.logger.info(f"Migrated {measurements_processed}/{total_measurements} measurements")
|
||||
except Exception as e:
|
||||
error_msg = f"Error saving measurement batch: {e}"
|
||||
self.logger.error(error_msg)
|
||||
self.stats.errors.append(error_msg)
|
||||
|
||||
offset += batch_size
|
||||
|
||||
# If we processed measurements, we're done
|
||||
if measurements_processed > 0:
|
||||
break
|
||||
|
||||
except sqlite3.OperationalError as e:
|
||||
if "no such table" in str(e).lower() or "no such column" in str(e).lower():
|
||||
continue
|
||||
else:
|
||||
raise
|
||||
|
||||
if measurements_processed == 0:
|
||||
self.logger.warning("No measurements found in SQLite database")
|
||||
else:
|
||||
self.logger.info(f"Successfully migrated {measurements_processed} measurements")
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Measurement migration error: {e}")
|
||||
return False
|
||||
|
||||
def _fast_bulk_insert(self, measurements: List[Dict]) -> bool:
|
||||
"""Super fast bulk insert using PostgreSQL COPY or VALUES clause"""
|
||||
try:
|
||||
import psycopg2
|
||||
from urllib.parse import urlparse
|
||||
import io
|
||||
|
||||
# Parse connection string for direct psycopg2 connection
|
||||
parsed = urlparse(self.postgres_config['connection_string'])
|
||||
|
||||
# Try super fast COPY method first
|
||||
try:
|
||||
conn = psycopg2.connect(
|
||||
host=parsed.hostname,
|
||||
port=parsed.port or 5432,
|
||||
database=parsed.path[1:],
|
||||
user=parsed.username,
|
||||
password=parsed.password
|
||||
)
|
||||
|
||||
with conn:
|
||||
with conn.cursor() as cur:
|
||||
# Prepare data for COPY
|
||||
data_buffer = io.StringIO()
|
||||
null_val = '\\N'
|
||||
for m in measurements:
|
||||
data_buffer.write(f"{m['timestamp']}\t{m['station_id']}\t{m['water_level'] or null_val}\t{m['discharge'] or null_val}\t{m['discharge_percent'] or null_val}\t{m['status']}\n")
|
||||
|
||||
data_buffer.seek(0)
|
||||
|
||||
# Use COPY for maximum speed
|
||||
cur.copy_from(
|
||||
data_buffer,
|
||||
'water_measurements',
|
||||
columns=('timestamp', 'station_id', 'water_level', 'discharge', 'discharge_percent', 'status'),
|
||||
sep='\t'
|
||||
)
|
||||
|
||||
conn.close()
|
||||
return True
|
||||
|
||||
except Exception as copy_error:
|
||||
# Fallback to SQLAlchemy bulk insert
|
||||
self.logger.debug(f"COPY failed, using bulk VALUES: {copy_error}")
|
||||
|
||||
from sqlalchemy import create_engine, text
|
||||
engine = create_engine(self.postgres_config['connection_string'])
|
||||
|
||||
with engine.begin() as conn:
|
||||
# Use PostgreSQL's fast bulk insert with ON CONFLICT
|
||||
values_list = []
|
||||
for m in measurements:
|
||||
timestamp = m['timestamp'].isoformat() if hasattr(m['timestamp'], 'isoformat') else str(m['timestamp'])
|
||||
values_list.append(
|
||||
f"('{timestamp}', {m['station_id']}, {m['water_level'] or 'NULL'}, "
|
||||
f"{m['discharge'] or 'NULL'}, {m['discharge_percent'] or 'NULL'}, '{m['status']}')"
|
||||
)
|
||||
|
||||
# Build bulk insert query with ON CONFLICT handling
|
||||
bulk_sql = f"""
|
||||
INSERT INTO water_measurements (timestamp, station_id, water_level, discharge, discharge_percent, status)
|
||||
VALUES {','.join(values_list)}
|
||||
ON CONFLICT (timestamp, station_id) DO UPDATE SET
|
||||
water_level = EXCLUDED.water_level,
|
||||
discharge = EXCLUDED.discharge,
|
||||
discharge_percent = EXCLUDED.discharge_percent,
|
||||
status = EXCLUDED.status
|
||||
"""
|
||||
|
||||
conn.execute(text(bulk_sql))
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Fast bulk insert failed: {e}")
|
||||
# Final fallback to original method
|
||||
try:
|
||||
success = self.postgres_adapter.save_measurements(measurements)
|
||||
return success
|
||||
except Exception as fallback_e:
|
||||
self.logger.error(f"All insert methods failed: {fallback_e}")
|
||||
return False
|
||||
|
||||
def verify_migration(self) -> bool:
|
||||
"""Verify the migration by comparing counts"""
|
||||
try:
|
||||
# Get SQLite counts
|
||||
cursor = self.sqlite_conn.cursor()
|
||||
|
||||
sqlite_stations = 0
|
||||
sqlite_measurements = 0
|
||||
|
||||
# Try to get station count
|
||||
for table in ['stations', 'water_stations']:
|
||||
try:
|
||||
cursor.execute(f"SELECT COUNT(*) FROM {table}")
|
||||
sqlite_stations = cursor.fetchone()[0]
|
||||
break
|
||||
except:
|
||||
continue
|
||||
|
||||
# Try to get measurement count
|
||||
for table in ['water_measurements', 'measurements']:
|
||||
try:
|
||||
cursor.execute(f"SELECT COUNT(*) FROM {table}")
|
||||
sqlite_measurements = cursor.fetchone()[0]
|
||||
break
|
||||
except:
|
||||
continue
|
||||
|
||||
# Get PostgreSQL counts
|
||||
postgres_measurements = self.postgres_adapter.get_latest_measurements(limit=999999)
|
||||
postgres_count = len(postgres_measurements)
|
||||
|
||||
self.logger.info("Migration Verification:")
|
||||
self.logger.info(f"SQLite stations: {sqlite_stations}")
|
||||
self.logger.info(f"SQLite measurements: {sqlite_measurements}")
|
||||
self.logger.info(f"PostgreSQL measurements retrieved: {postgres_count}")
|
||||
self.logger.info(f"Migrated stations: {self.stats.stations_migrated}")
|
||||
self.logger.info(f"Migrated measurements: {self.stats.measurements_migrated}")
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Verification error: {e}")
|
||||
return False
|
||||
|
||||
def run_migration(self, sqlite_path: str = None) -> bool:
|
||||
"""Run the complete migration process"""
|
||||
self.stats.start_time = datetime.now()
|
||||
|
||||
if sqlite_path:
|
||||
self.sqlite_path = sqlite_path
|
||||
|
||||
self.logger.info("=" * 60)
|
||||
self.logger.info("SQLite to PostgreSQL Migration Tool")
|
||||
self.logger.info("=" * 60)
|
||||
self.logger.info(f"SQLite database: {self.sqlite_path}")
|
||||
self.logger.info(f"PostgreSQL: {self.postgres_config['type']}")
|
||||
|
||||
try:
|
||||
# Step 1: Connect to databases
|
||||
self.logger.info("Step 1: Connecting to databases...")
|
||||
if not self.connect_databases():
|
||||
return False
|
||||
|
||||
# Step 2: Analyze SQLite schema
|
||||
self.logger.info("Step 2: Analyzing SQLite database structure...")
|
||||
schema_info = self.analyze_sqlite_schema()
|
||||
if not schema_info:
|
||||
self.logger.error("Could not analyze SQLite database structure")
|
||||
return False
|
||||
|
||||
# Step 3: Migrate stations
|
||||
self.logger.info("Step 3: Migrating station data...")
|
||||
if not self.migrate_stations():
|
||||
self.logger.error("Station migration failed")
|
||||
return False
|
||||
|
||||
# Step 4: Migrate measurements
|
||||
self.logger.info("Step 4: Migrating measurement data...")
|
||||
if not self.migrate_measurements():
|
||||
self.logger.error("Measurement migration failed")
|
||||
return False
|
||||
|
||||
# Step 5: Verify migration
|
||||
self.logger.info("Step 5: Verifying migration...")
|
||||
self.verify_migration()
|
||||
|
||||
self.stats.end_time = datetime.now()
|
||||
duration = self.stats.end_time - self.stats.start_time
|
||||
|
||||
# Final report
|
||||
self.logger.info("=" * 60)
|
||||
self.logger.info("MIGRATION COMPLETED")
|
||||
self.logger.info("=" * 60)
|
||||
self.logger.info(f"Duration: {duration}")
|
||||
self.logger.info(f"Stations migrated: {self.stats.stations_migrated}")
|
||||
self.logger.info(f"Measurements migrated: {self.stats.measurements_migrated}")
|
||||
|
||||
if self.stats.errors:
|
||||
self.logger.warning(f"Errors encountered: {len(self.stats.errors)}")
|
||||
for error in self.stats.errors[:10]: # Show first 10 errors
|
||||
self.logger.warning(f" - {error}")
|
||||
if len(self.stats.errors) > 10:
|
||||
self.logger.warning(f" ... and {len(self.stats.errors) - 10} more errors")
|
||||
else:
|
||||
self.logger.info("No errors encountered")
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Migration failed: {e}")
|
||||
return False
|
||||
|
||||
finally:
|
||||
# Cleanup
|
||||
if self.sqlite_conn:
|
||||
self.sqlite_conn.close()
|
||||
|
||||
def main():
|
||||
"""Main entry point"""
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description="Migrate SQLite data to PostgreSQL")
|
||||
parser.add_argument("sqlite_path", nargs="?", help="Path to SQLite database file")
|
||||
parser.add_argument("--batch-size", type=int, default=5000, help="Batch size for processing measurements")
|
||||
parser.add_argument("--fast", action="store_true", help="Use maximum speed mode (batch-size 10000)")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Analyze only, don't migrate")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Set fast mode
|
||||
if args.fast:
|
||||
args.batch_size = 10000
|
||||
|
||||
# Get SQLite path
|
||||
sqlite_path = args.sqlite_path
|
||||
if not sqlite_path:
|
||||
# Try to find common SQLite database files
|
||||
possible_paths = [
|
||||
"water_levels.db",
|
||||
"water_monitoring.db",
|
||||
"database.db",
|
||||
"../water_levels.db"
|
||||
]
|
||||
|
||||
for path in possible_paths:
|
||||
if os.path.exists(path):
|
||||
sqlite_path = path
|
||||
break
|
||||
|
||||
if not sqlite_path:
|
||||
print("SQLite database file not found. Please specify the path:")
|
||||
print(" python migrate_sqlite_to_postgres.py /path/to/database.db")
|
||||
return False
|
||||
|
||||
# Get PostgreSQL configuration
|
||||
try:
|
||||
from config import Config
|
||||
postgres_config = Config.get_database_config()
|
||||
|
||||
if postgres_config['type'] != 'postgresql':
|
||||
print("Error: PostgreSQL not configured. Set DB_TYPE=postgresql in your .env file")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error loading PostgreSQL configuration: {e}")
|
||||
return False
|
||||
|
||||
# Run migration
|
||||
migrator = SQLiteToPostgresMigrator(sqlite_path, postgres_config)
|
||||
|
||||
if args.dry_run:
|
||||
print("DRY RUN MODE - Analyzing SQLite database structure only")
|
||||
if migrator.connect_databases():
|
||||
schema_info = migrator.analyze_sqlite_schema()
|
||||
print("\nSQLite database structure analysis complete.")
|
||||
print("Run without --dry-run to perform the actual migration.")
|
||||
return True
|
||||
|
||||
success = migrator.run_migration()
|
||||
return success
|
||||
|
||||
if __name__ == "__main__":
|
||||
success = main()
|
||||
sys.exit(0 if success else 1)
|
||||
@@ -14,10 +14,9 @@ Restart=always
|
||||
RestartSec=30
|
||||
TimeoutStopSec=20
|
||||
|
||||
# Same .env as water-monitor.service (POSTGRES_*, RID_*, GAP_FILL_*,
|
||||
# SCRAPING_STATUS_FILE, FORECAST_IN_COLLECTOR, NTFY_*). The Python app must run
|
||||
# with COLLECTION_ENABLED=false. Models are read from models/ (relative to
|
||||
# WorkingDirectory).
|
||||
# Every key with its default: .env.example (POSTGRES_*, WEB_LISTEN,
|
||||
# FORECAST_IN_COLLECTOR, NTFY_*, MATRIX_*, ...). Models are read from models/
|
||||
# (relative to WorkingDirectory); installed by scripts/install.sh.
|
||||
EnvironmentFile=/opt/thailand-water-monitor/.env
|
||||
|
||||
# Security settings
|
||||
|
||||
+7
-7
@@ -5,15 +5,15 @@
|
||||
#
|
||||
# Why a script rather than ExecStart=train_flood_model.py:
|
||||
# * train.py writes each station's bundle straight into models/ over ~12 min,
|
||||
# and the API's hourly precompute reloads bundles by mtime. Training into
|
||||
# a staging dir and mv-ing (atomic on one filesystem) means the API never
|
||||
# sees a half-written joblib file or a mixed old/new set.
|
||||
# and ping-collector reloads bundles by mtime on every forecast run.
|
||||
# Training into a staging dir and mv-ing (atomic on one filesystem) means
|
||||
# it never sees a half-written bundle or a mixed old/new set.
|
||||
# * A run that produced gauge-only (v2) bundles, or trained too few stations,
|
||||
# must NOT replace the deployed models. train.py already aborts on a
|
||||
# missing rain series; this script re-checks the written metrics anyway.
|
||||
# * No restart is needed: predict.py (Python) and ping-collector (Rust,
|
||||
# FORECAST_IN_COLLECTOR=true) reload changed bundles on their next
|
||||
# forecast run, so the new models are live within an hour.
|
||||
# * No restart is needed: ping-collector reloads changed bundles
|
||||
# (models/flood_*.json.gz, written by train.py next to each .joblib) on
|
||||
# its next forecast run, so the new models are live within an hour.
|
||||
#
|
||||
# Exit codes: 0 ok, 2 training refused (see log), 3 verification failed.
|
||||
set -euo pipefail
|
||||
@@ -88,4 +88,4 @@ for f in "${STAGE_DIR}"/flood_*.joblib "${STAGE_DIR}"/flood_*.json.gz "${STAGE_D
|
||||
mv -f "${f}" "${MODELS_DIR}/${name}"
|
||||
done
|
||||
rm -rf "${STAGE_DIR}"
|
||||
log "deployed ${VERSION} (${TRAINED} stations); previous generation in models/.previous. The API picks it up on its next hourly precompute."
|
||||
log "deployed ${VERSION} (${TRAINED} stations); previous generation in models/.previous. ping-collector loads it on its next forecast run."
|
||||
@@ -1,175 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
PostgreSQL setup script for Northern Thailand Ping River Monitor
|
||||
This script helps you configure and test your PostgreSQL connection
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
from typing import Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
def setup_logging():
|
||||
logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
|
||||
|
||||
def test_postgres_connection(connection_string: str) -> bool:
|
||||
"""Test connection to PostgreSQL database"""
|
||||
try:
|
||||
from sqlalchemy import create_engine, text
|
||||
|
||||
# Test connection
|
||||
engine = create_engine(connection_string, pool_pre_ping=True)
|
||||
with engine.connect() as conn:
|
||||
result = conn.execute(text("SELECT version()"))
|
||||
version = result.fetchone()[0]
|
||||
logging.info(f"✅ Connected to PostgreSQL successfully!")
|
||||
logging.info(f"Database version: {version}")
|
||||
return True
|
||||
|
||||
except ImportError:
|
||||
logging.error("❌ psycopg2-binary not installed. Run: uv add psycopg2-binary")
|
||||
return False
|
||||
except Exception as e:
|
||||
logging.error(f"❌ Connection failed: {e}")
|
||||
return False
|
||||
|
||||
def parse_connection_string(connection_string: str) -> dict:
|
||||
"""Parse PostgreSQL connection string into components"""
|
||||
try:
|
||||
parsed = urlparse(connection_string)
|
||||
return {
|
||||
'host': parsed.hostname,
|
||||
'port': parsed.port or 5432,
|
||||
'database': parsed.path[1:] if parsed.path else None,
|
||||
'username': parsed.username,
|
||||
'password': parsed.password,
|
||||
}
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to parse connection string: {e}")
|
||||
return {}
|
||||
|
||||
def create_database_if_not_exists(connection_string: str, database_name: str) -> bool:
|
||||
"""Create database if it doesn't exist"""
|
||||
try:
|
||||
from sqlalchemy import create_engine, text
|
||||
|
||||
# Connect to default postgres database to create our database
|
||||
parsed = urlparse(connection_string)
|
||||
admin_connection = connection_string.replace(f"/{parsed.path[1:]}", "/postgres")
|
||||
|
||||
engine = create_engine(admin_connection, pool_pre_ping=True)
|
||||
|
||||
with engine.connect() as conn:
|
||||
# Check if database exists
|
||||
result = conn.execute(text(
|
||||
"SELECT 1 FROM pg_database WHERE datname = :db_name"
|
||||
), {"db_name": database_name})
|
||||
|
||||
if result.fetchone():
|
||||
logging.info(f"✅ Database '{database_name}' already exists")
|
||||
return True
|
||||
else:
|
||||
# Create database
|
||||
conn.execute(text("COMMIT")) # End transaction
|
||||
conn.execute(text(f'CREATE DATABASE "{database_name}"'))
|
||||
logging.info(f"✅ Created database '{database_name}'")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"❌ Failed to create database: {e}")
|
||||
return False
|
||||
|
||||
def initialize_tables(connection_string: str) -> bool:
|
||||
"""Initialize database tables"""
|
||||
try:
|
||||
# Import the database adapter to create tables
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src'))
|
||||
from database_adapters import SQLAdapter
|
||||
|
||||
adapter = SQLAdapter(connection_string=connection_string, db_type='postgresql')
|
||||
if adapter.connect():
|
||||
logging.info("✅ Database tables initialized successfully")
|
||||
return True
|
||||
else:
|
||||
logging.error("❌ Failed to initialize tables")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"❌ Failed to initialize tables: {e}")
|
||||
return False
|
||||
|
||||
def interactive_setup():
|
||||
"""Interactive setup wizard"""
|
||||
print("🐘 PostgreSQL Setup Wizard for Ping River Monitor")
|
||||
print("=" * 50)
|
||||
|
||||
# Get connection details
|
||||
host = input("PostgreSQL host (e.g., 192.168.1.100): ").strip()
|
||||
port = input("PostgreSQL port [5432]: ").strip() or "5432"
|
||||
database = input("Database name [water_monitoring]: ").strip() or "water_monitoring"
|
||||
username = input("Username: ").strip()
|
||||
password = input("Password: ").strip()
|
||||
|
||||
# Optional SSL
|
||||
use_ssl = input("Use SSL connection? (y/N): ").strip().lower() == 'y'
|
||||
ssl_params = "?sslmode=require" if use_ssl else ""
|
||||
|
||||
connection_string = f"postgresql://{username}:{password}@{host}:{port}/{database}{ssl_params}"
|
||||
|
||||
print(f"\nGenerated connection string:")
|
||||
print(f"POSTGRES_CONNECTION_STRING={connection_string}")
|
||||
|
||||
return connection_string
|
||||
|
||||
def main():
|
||||
setup_logging()
|
||||
|
||||
print("🚀 Northern Thailand Ping River Monitor - PostgreSQL Setup")
|
||||
print("=" * 60)
|
||||
|
||||
# Check if connection string is provided via environment
|
||||
connection_string = os.getenv('POSTGRES_CONNECTION_STRING')
|
||||
|
||||
if not connection_string:
|
||||
print("No POSTGRES_CONNECTION_STRING found in environment.")
|
||||
print("Starting interactive setup...\n")
|
||||
connection_string = interactive_setup()
|
||||
|
||||
# Suggest adding to .env file
|
||||
print(f"\n💡 Add this to your .env file:")
|
||||
print(f"DB_TYPE=postgresql")
|
||||
print(f"POSTGRES_CONNECTION_STRING={connection_string}")
|
||||
|
||||
# Parse connection details
|
||||
config = parse_connection_string(connection_string)
|
||||
if not config.get('host'):
|
||||
logging.error("Invalid connection string format")
|
||||
return False
|
||||
|
||||
print(f"\n🔗 Connecting to PostgreSQL at {config['host']}:{config['port']}")
|
||||
|
||||
# Test connection
|
||||
if not test_postgres_connection(connection_string):
|
||||
return False
|
||||
|
||||
# Try to create database
|
||||
database_name = config.get('database', 'water_monitoring')
|
||||
if database_name:
|
||||
create_database_if_not_exists(connection_string, database_name)
|
||||
|
||||
# Initialize tables
|
||||
if not initialize_tables(connection_string):
|
||||
return False
|
||||
|
||||
print("\n🎉 PostgreSQL setup completed successfully!")
|
||||
print("\nNext steps:")
|
||||
print("1. Update your .env file with the connection string")
|
||||
print("2. Run: make run-test")
|
||||
print("3. Run: make run-api")
|
||||
|
||||
return True
|
||||
|
||||
if __name__ == "__main__":
|
||||
success = main()
|
||||
sys.exit(0 if success else 1)
|
||||
@@ -1,48 +0,0 @@
|
||||
@echo off
|
||||
REM Setup script for uv-based development environment on Windows
|
||||
|
||||
echo 🚀 Setting up Northern Thailand Ping River Monitor with uv...
|
||||
|
||||
REM Check if uv is installed
|
||||
uv --version >nul 2>&1
|
||||
if %errorlevel% neq 0 (
|
||||
echo ❌ uv is not installed. Please install it first:
|
||||
echo powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
echo ✅ uv found
|
||||
uv --version
|
||||
|
||||
REM Initialize uv project if not already initialized
|
||||
if not exist "uv.lock" (
|
||||
echo 🔧 Initializing uv project...
|
||||
uv sync
|
||||
) else (
|
||||
echo 📦 Syncing dependencies with uv...
|
||||
uv sync
|
||||
)
|
||||
|
||||
REM Install pre-commit hooks
|
||||
echo 🎣 Installing pre-commit hooks...
|
||||
uv run pre-commit install
|
||||
|
||||
REM Create .env file if it doesn't exist
|
||||
if not exist ".env" (
|
||||
if exist ".env.example" (
|
||||
echo 📝 Creating .env file from template...
|
||||
copy .env.example .env
|
||||
echo ⚠️ Please edit .env file with your configuration
|
||||
)
|
||||
)
|
||||
|
||||
echo ✅ Setup complete!
|
||||
echo.
|
||||
echo 📚 Quick start commands:
|
||||
echo make install-dev # Install all dependencies
|
||||
echo make run-test # Run a test cycle
|
||||
echo make run-api # Start the web API
|
||||
echo make test # Run tests
|
||||
echo make lint # Check code quality
|
||||
echo.
|
||||
echo 🎉 Happy monitoring!
|
||||
@@ -1,46 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Setup script for uv-based development environment
|
||||
|
||||
set -e
|
||||
|
||||
echo "🚀 Setting up Northern Thailand Ping River Monitor with uv..."
|
||||
|
||||
# Check if uv is installed
|
||||
if ! command -v uv &> /dev/null; then
|
||||
echo "❌ uv is not installed. Please install it first:"
|
||||
echo " curl -LsSf https://astral.sh/uv/install.sh | sh"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✅ uv found: $(uv --version)"
|
||||
|
||||
# Initialize uv project if not already initialized
|
||||
if [ ! -f "uv.lock" ]; then
|
||||
echo "🔧 Initializing uv project..."
|
||||
uv sync
|
||||
else
|
||||
echo "📦 Syncing dependencies with uv..."
|
||||
uv sync
|
||||
fi
|
||||
|
||||
# Install pre-commit hooks
|
||||
echo "🎣 Installing pre-commit hooks..."
|
||||
uv run pre-commit install
|
||||
|
||||
# Create .env file if it doesn't exist
|
||||
if [ ! -f ".env" ] && [ -f ".env.example" ]; then
|
||||
echo "📝 Creating .env file from template..."
|
||||
cp .env.example .env
|
||||
echo "⚠️ Please edit .env file with your configuration"
|
||||
fi
|
||||
|
||||
echo "✅ Setup complete!"
|
||||
echo ""
|
||||
echo "📚 Quick start commands:"
|
||||
echo " make install-dev # Install all dependencies"
|
||||
echo " make run-test # Run a test cycle"
|
||||
echo " make run-api # Start the web API"
|
||||
echo " make test # Run tests"
|
||||
echo " make lint # Check code quality"
|
||||
echo ""
|
||||
echo "🎉 Happy monitoring!"
|
||||
@@ -25,7 +25,8 @@ def summarize(paths):
|
||||
for fold in station["folds"]:
|
||||
for name, m in fold["variants"].items():
|
||||
a = agg.setdefault(
|
||||
name, {"mae": [], "mae_hi": [], "brier": [], "fa": 0, "events": []}
|
||||
name,
|
||||
{"mae": [], "mae_hi": [], "brier": [], "fa": 0, "events": []},
|
||||
)
|
||||
a["mae"].append(m["mae"])
|
||||
if m.get("mae_above_2p5") is not None:
|
||||
@@ -39,8 +40,18 @@ def summarize(paths):
|
||||
if e["peak_pred_24h_before"] is None
|
||||
else e["peak_pred_24h_before"] - e["peak_level"]
|
||||
)
|
||||
a["events"].append((fold["year"], e["crossing"][:10], e["lead_h"], e["peak_level"], err))
|
||||
print(f"{'variant':22} {'MAE':>6} {'MAE_hi':>7} {'Brier':>7} {'FA':>3} events: year crossing lead_h peak(err24h)")
|
||||
a["events"].append(
|
||||
(
|
||||
fold["year"],
|
||||
e["crossing"][:10],
|
||||
e["lead_h"],
|
||||
e["peak_level"],
|
||||
err,
|
||||
)
|
||||
)
|
||||
print(
|
||||
f"{'variant':22} {'MAE':>6} {'MAE_hi':>7} {'Brier':>7} {'FA':>3} events: year crossing lead_h peak(err24h)"
|
||||
)
|
||||
for name, a in agg.items():
|
||||
ev = " ".join(
|
||||
f"{y} {d} {'—' if l is None else format(l, '+.0f')}h {p:.2f}({'—' if err is None else format(err, '+.2f')})"
|
||||
@@ -54,8 +65,10 @@ def summarize(paths):
|
||||
f"{a['fa']:>3} {ev}"
|
||||
)
|
||||
if leads:
|
||||
print(f"{'':22} lead: mean {statistics.mean(leads):+.1f} h, min {min(leads):+.0f} h, "
|
||||
f"missed {sum(1 for *_, l, _, _ in a['events'] if l is None)}/{len(a['events'])}")
|
||||
print(
|
||||
f"{'':22} lead: mean {statistics.mean(leads):+.1f} h, min {min(leads):+.0f} h, "
|
||||
f"missed {sum(1 for *_, l, _, _ in a['events'] if l is None)}/{len(a['events'])}"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -10,11 +10,11 @@ User=water-monitor
|
||||
Group=water-monitor
|
||||
WorkingDirectory=/opt/thailand-water-monitor
|
||||
EnvironmentFile=/opt/thailand-water-monitor/.env
|
||||
# Same interpreter as water-monitor.service -- the uv-managed .venv.
|
||||
# The uv-managed .venv (Python training stack; scripts/install.sh syncs it).
|
||||
# scripts/retrain.sh trains into models/.staging, refuses to promote anything
|
||||
# that is not a rain-enabled (hgb-v3) set covering the expected stations, then
|
||||
# renames the bundles into place. The API reloads them on its next hourly
|
||||
# precompute; no restart, so a failed run leaves the old models serving.
|
||||
# renames the bundles into place. ping-collector reloads them on its next
|
||||
# forecast run; no restart, so a failed run leaves the old models serving.
|
||||
ExecStart=/bin/bash /opt/thailand-water-monitor/scripts/retrain.sh
|
||||
# HistGradientBoosting is CPU-bound; cap threads so training cannot starve
|
||||
# the API (docs/FLOOD_FORECASTING.md section 6 measured 4 as the sweet spot).
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
[Unit]
|
||||
Description=Thailand Water Level Monitor
|
||||
Documentation=https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor
|
||||
After=network.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=water-monitor
|
||||
Group=water-monitor
|
||||
WorkingDirectory=/opt/thailand-water-monitor
|
||||
# The uv-managed env (uv sync -> .venv). Same interpreter for water-monitor-retrain.service.
|
||||
ExecStart=/opt/thailand-water-monitor/.venv/bin/python run.py --web-api
|
||||
ExecReload=/bin/kill -HUP $MAINPID
|
||||
Restart=always
|
||||
RestartSec=60
|
||||
TimeoutStopSec=30
|
||||
|
||||
# DB_TYPE / POSTGRES_CONNECTION_STRING / MATRIX_* come from the .env file.
|
||||
EnvironmentFile=/opt/thailand-water-monitor/.env
|
||||
Environment=PYTHONPATH=/opt/thailand-water-monitor
|
||||
# Serving path is latency-bound; single-threaded BLAS is 2.6x faster per call
|
||||
# (docs/FLOOD_FORECASTING.md section 6). Training sets its own value.
|
||||
Environment=OMP_NUM_THREADS=1
|
||||
Environment=PYTHONUNBUFFERED=1
|
||||
|
||||
# Security settings
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
ReadWritePaths=/opt/thailand-water-monitor
|
||||
CapabilityBoundingSet=
|
||||
|
||||
# Logging
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=water-monitor
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,2 +0,0 @@
|
||||
target/
|
||||
data/
|
||||
Generated
-132
@@ -1,132 +0,0 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "1.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
||||
|
||||
[[package]]
|
||||
name = "chrono"
|
||||
version = "0.4.45"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327"
|
||||
dependencies = [
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
||||
|
||||
[[package]]
|
||||
name = "num-traits"
|
||||
version = "0.2.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ping-forecast-spike"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.107"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.47"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.229"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_core"
|
||||
version = "1.0.229"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.229"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.154"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"memchr",
|
||||
"serde",
|
||||
"serde_core",
|
||||
"zmij",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "3.0.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
|
||||
|
||||
[[package]]
|
||||
name = "zmij"
|
||||
version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
||||
@@ -1,15 +0,0 @@
|
||||
[package]
|
||||
name = "ping-forecast-spike"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
publish = false
|
||||
|
||||
[dependencies]
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
# float_roundtrip: exact float parsing. Tree thresholds often equal real data
|
||||
# values, so a 1-ulp parse error flips `x <= threshold` (default parser is not exact).
|
||||
serde_json = { version = "1", features = ["float_roundtrip"] }
|
||||
chrono = { version = "0.4", default-features = false, features = ["std"] }
|
||||
|
||||
[profile.release]
|
||||
debug = false
|
||||
@@ -1,43 +0,0 @@
|
||||
# Rust forecast spike
|
||||
|
||||
Answers one question: can the flood forecast be served from Rust without
|
||||
train/serve skew, while training stays in Python (scikit-learn)?
|
||||
Assessment and migration plan: [docs/RUST_REWRITE.md](../../docs/RUST_REWRITE.md).
|
||||
|
||||
- `export/export_fixture.py`: dumps `models/flood_P.1.joblib` (all 9 HGB heads) as
|
||||
flat tree arrays, plus two fixtures from `models/cache` (Aug-Oct 2024 record
|
||||
flood, Jun-Aug 2026): the observed hourly grid of P.1 and its 7 upstream
|
||||
gauges, rain, the pandas feature matrix and every head's sklearn prediction.
|
||||
- `src/features.rs`: port of `src/ml/features.py::build_features` (P.1 path:
|
||||
lags, rises, rolling stats, upstream leads, season, rain).
|
||||
- `src/hgb.rs`: HistGradientBoosting inference (~90 lines).
|
||||
- `src/main.rs`: compares bit-for-bit and times serving-shaped work.
|
||||
|
||||
```bash
|
||||
cd spikes/rust-forecast/export
|
||||
uv run --project ../../.. python export_fixture.py # writes ../data/*.json (gitignored)
|
||||
cd .. && cargo run --release
|
||||
uv run --project ../.. python export/time_python.py # Python timing for comparison
|
||||
```
|
||||
|
||||
Result (model hgb-v3+cbb3bf7, 3,600 hourly rows):
|
||||
|
||||
| | Result |
|
||||
| --- | --- |
|
||||
| Features, 68 columns x 3,600 rows | 244,800 / 244,800 cells bit-identical |
|
||||
| Regression heads (max_6/12/24) | bit-identical on every row |
|
||||
| Classifier heads (warn/danger) | equal to the last ulp (<= 2.2e-16, `exp` vs scipy `expit`) |
|
||||
| One station: 336 h window -> features + 9 heads | Rust 0.25 ms, Python 158 ms |
|
||||
| Model load | Rust 16 ms (JSON), Python 199 ms (joblib) |
|
||||
|
||||
Two traps the spike had to get past, both of which flip tree branches because
|
||||
HGB thresholds often equal real data values exactly:
|
||||
|
||||
1. `serde_json` parses floats best-effort (can be 1 ulp off) unless the
|
||||
`float_roundtrip` feature is on.
|
||||
2. pandas `rolling().sum()/.mean()` are incremental with Kahan compensation,
|
||||
a "run of identical values" shortcut and a sign clamp. A plain window sum
|
||||
differs in the last bits; `rolling_kahan()` replicates pandas exactly.
|
||||
|
||||
Not covered yet: dam features (Mae Ngat), HII gap fill, Open-Meteo catchment
|
||||
mean, the heuristic fallback, stage probabilities, the other 14 stations.
|
||||
@@ -1,132 +0,0 @@
|
||||
"""Export a trained flood bundle + a feature/prediction fixture for the Rust spike.
|
||||
|
||||
Run from the monitor repo's environment:
|
||||
uv run --project ../../.. python export_fixture.py (from this directory)
|
||||
|
||||
Writes ../data/model_<station>.json (all heads as flat tree arrays) and
|
||||
../data/fixture_<name>.json (observed hourly grid + rain, the Python feature
|
||||
matrix, and every head's Python prediction for every row).
|
||||
"""
|
||||
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import joblib
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
REPO = Path(__file__).resolve().parents[3] # repo root
|
||||
sys.path.insert(0, str(REPO))
|
||||
|
||||
from src.ml import features # noqa: E402
|
||||
|
||||
OUT = Path(__file__).resolve().parent.parent / "data"
|
||||
OUT.mkdir(exist_ok=True)
|
||||
STATION = "P.1"
|
||||
WINDOWS = {
|
||||
"flood2024": ("2024-08-15", "2024-10-31"), # record flood, high-level paths
|
||||
"recent2026": ("2026-06-01", "2026-08-13"), # latest cached data
|
||||
}
|
||||
|
||||
|
||||
def clean(values):
|
||||
return [None if (v is None or (isinstance(v, float) and math.isnan(v))) else float(v) for v in values]
|
||||
|
||||
|
||||
def export_head(est):
|
||||
preds = est._predictors
|
||||
assert all(len(it) == 1 for it in preds), "multiclass not expected"
|
||||
trees = []
|
||||
for (tp,) in preds:
|
||||
n = tp.nodes
|
||||
assert not n["is_categorical"].any()
|
||||
assert np.isfinite(n["value"]).all() and not np.isneginf(n["num_threshold"]).any()
|
||||
assert not np.isnan(n["num_threshold"]).any()
|
||||
trees.append(
|
||||
{
|
||||
"feature": n["feature_idx"].astype(int).tolist(),
|
||||
# +inf thresholds (a "missing vs everything" split) are not
|
||||
# valid JSON: written as null, read back as +inf in Rust
|
||||
"threshold": [
|
||||
None if np.isposinf(t) else float(t) for t in n["num_threshold"]
|
||||
],
|
||||
"missing_left": n["missing_go_to_left"].astype(bool).tolist(),
|
||||
"left": n["left"].astype(int).tolist(),
|
||||
"right": n["right"].astype(int).tolist(),
|
||||
"leaf": n["is_leaf"].astype(bool).tolist(),
|
||||
"value": n["value"].astype(float).tolist(),
|
||||
}
|
||||
)
|
||||
kind = "classifier" if hasattr(est, "predict_proba") else "regressor"
|
||||
return {
|
||||
"kind": kind,
|
||||
"baseline": float(np.ravel(est._baseline_prediction)[0]),
|
||||
"trees": trees,
|
||||
}
|
||||
|
||||
|
||||
bundle = joblib.load(REPO / "models" / f"flood_{STATION}.joblib")
|
||||
model = {
|
||||
"station_code": STATION,
|
||||
"model_version": bundle["model_version"],
|
||||
"regression_target": bundle.get("regression_target"),
|
||||
"feature_names": bundle["feature_names"],
|
||||
"horizons": bundle["horizons"],
|
||||
"sigma": {str(k): float(v) for k, v in bundle["sigma"].items()},
|
||||
"thresholds": bundle["thresholds"],
|
||||
"heads": {name: export_head(est) for name, est in bundle["heads"].items()},
|
||||
}
|
||||
(OUT / f"model_{STATION}.json").write_text(json.dumps(model))
|
||||
n_nodes = sum(len(t["value"]) for h in model["heads"].values() for t in h["trees"])
|
||||
print(f"model {model['model_version']}: {len(model['heads'])} heads, {n_nodes} nodes")
|
||||
|
||||
stations = [STATION] + [u for u, _ in features.UPSTREAM_LEADS[STATION]]
|
||||
long = pd.concat(
|
||||
pd.read_csv(REPO / "models" / "cache" / f"{s}.csv.gz", parse_dates=["timestamp"])
|
||||
for s in stations
|
||||
)
|
||||
rain_all = pd.read_csv(REPO / "models" / "cache" / "rain_openmeteo.csv.gz")
|
||||
print("rain columns:", list(rain_all.columns))
|
||||
ts_col = rain_all.columns[0]
|
||||
rain_all[ts_col] = pd.to_datetime(rain_all[ts_col])
|
||||
rain_all = rain_all.set_index(ts_col)[rain_all.columns[1]].astype(float)
|
||||
|
||||
for name, (start, end) in WINDOWS.items():
|
||||
df = long[(long.timestamp >= start) & (long.timestamp < end)]
|
||||
grid = features.make_hourly_grid(df)
|
||||
idx = grid.observed.index
|
||||
rain = rain_all.reindex(idx)
|
||||
# punch deterministic holes so NaN handling of the rolling sums is exercised
|
||||
rain.iloc[::97] = np.nan
|
||||
feats = features.build_features(grid, STATION, rain=rain)[model["feature_names"]]
|
||||
|
||||
preds = {}
|
||||
for head_name, est in bundle["heads"].items():
|
||||
if hasattr(est, "predict_proba"):
|
||||
preds[head_name] = est.predict_proba(feats)[:, 1].tolist()
|
||||
else:
|
||||
preds[head_name] = est.predict(feats).tolist()
|
||||
|
||||
fixture = {
|
||||
"station": STATION,
|
||||
"timestamps": [t.strftime("%Y-%m-%dT%H:%M:%S") for t in idx],
|
||||
"observed": {
|
||||
s: {
|
||||
"water_level": clean(grid.observed[(s, "water_level")].tolist()),
|
||||
"discharge": clean(grid.observed[(s, "discharge")].tolist()),
|
||||
}
|
||||
for s in stations
|
||||
if (s, "water_level") in grid.observed.columns
|
||||
},
|
||||
"rain": clean(rain.tolist()),
|
||||
"feature_names": model["feature_names"],
|
||||
"features": [clean(row) for row in feats.to_numpy().tolist()],
|
||||
"predictions": preds,
|
||||
}
|
||||
(OUT / f"fixture_{name}.json").write_text(json.dumps(fixture))
|
||||
print(
|
||||
f"{name}: {len(idx)} hours, {len(fixture['observed'])} stations, "
|
||||
f"P.1 max {np.nanmax(grid.observed[(STATION, 'water_level')]):.2f} m"
|
||||
)
|
||||
@@ -1,38 +0,0 @@
|
||||
"""Python timing for the same serving-shaped work as the Rust spike."""
|
||||
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import joblib
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
REPO = Path(__file__).resolve().parents[3] # repo root
|
||||
sys.path.insert(0, str(REPO))
|
||||
from src.ml import features # noqa: E402
|
||||
|
||||
bundle = joblib.load(REPO / "models" / "flood_P.1.joblib")
|
||||
stations = ["P.1"] + [u for u, _ in features.UPSTREAM_LEADS["P.1"]]
|
||||
long = pd.concat(
|
||||
pd.read_csv(REPO / "models" / "cache" / f"{s}.csv.gz", parse_dates=["timestamp"])
|
||||
for s in stations
|
||||
)
|
||||
df = long[(long.timestamp >= "2026-07-30") & (long.timestamp < "2026-08-13")]
|
||||
rain_all = pd.read_csv(REPO / "models" / "cache" / "rain_openmeteo.csv.gz", parse_dates=["timestamp"])
|
||||
rain_all = rain_all.set_index("timestamp")["chiang_dao"].astype(float)
|
||||
|
||||
reps = 20
|
||||
t = time.perf_counter()
|
||||
for _ in range(reps):
|
||||
grid = features.make_hourly_grid(df)
|
||||
rain = rain_all.reindex(grid.observed.index)
|
||||
row = features.build_features(grid, "P.1", rain=rain)[bundle["feature_names"]].iloc[[-1]]
|
||||
out = {
|
||||
k: (h.predict_proba(row)[0, 1] if hasattr(h, "predict_proba") else h.predict(row)[0])
|
||||
for k, h in bundle["heads"].items()
|
||||
}
|
||||
print(f"python: grid + features + 9 heads for the last hour: {(time.perf_counter() - t) / reps * 1000:.1f} ms per station")
|
||||
t = time.perf_counter()
|
||||
joblib.load(REPO / "models" / "flood_P.1.joblib")
|
||||
print(f"python: bundle load {(time.perf_counter() - t) * 1000:.0f} ms")
|
||||
@@ -1,318 +0,0 @@
|
||||
//! Port of `src/ml/features.py::build_features` (pandas) to plain Rust.
|
||||
//!
|
||||
//! Pandas semantics reproduced: `ffill(limit)`, `shift(k)` (NaN-padded),
|
||||
//! `rolling(w, min_periods=1)` aggregations that skip NaN and yield NaN for an
|
||||
//! all-NaN window, NaN propagation through subtraction.
|
||||
|
||||
use chrono::{Datelike, NaiveDateTime};
|
||||
use std::collections::HashMap;
|
||||
use std::f64::consts::PI;
|
||||
|
||||
pub const FFILL_LIMIT_H: usize = 3;
|
||||
pub const BASIN_ANCHOR: &str = "P.1";
|
||||
const MONSOON_MONTHS: [u32; 5] = [6, 7, 8, 9, 10];
|
||||
|
||||
pub fn upstream_leads(station: &str) -> &'static [(&'static str, i64)] {
|
||||
match station {
|
||||
"P.1" => &[
|
||||
("P.103", 1),
|
||||
("P.67", 7),
|
||||
("P.21", 9),
|
||||
("P.75", 12),
|
||||
("P.4A", 12),
|
||||
("P.92", 15),
|
||||
("P.20", 17),
|
||||
],
|
||||
"P.21" => &[
|
||||
("P.67", 1),
|
||||
("P.75", 3),
|
||||
("P.4A", 3),
|
||||
("P.92", 6),
|
||||
("P.20", 8),
|
||||
],
|
||||
"P.67" => &[("P.75", 5), ("P.4A", 5), ("P.92", 8), ("P.20", 10)],
|
||||
"P.75" | "P.4A" => &[("P.92", 3), ("P.20", 5)],
|
||||
"P.92" => &[("P.20", 2)],
|
||||
"P.5" => &[("P.1", 12), ("P.103", 13)],
|
||||
"P.81" => &[("P.1", 4), ("P.103", 5)],
|
||||
_ => &[], // spike: P.103's table omitted
|
||||
}
|
||||
}
|
||||
|
||||
pub type Series = Vec<f64>;
|
||||
|
||||
pub struct Grid {
|
||||
pub times: Vec<NaiveDateTime>,
|
||||
/// station -> observed (unfilled) hourly series, NaN where missing
|
||||
pub level: HashMap<String, Series>,
|
||||
pub discharge: HashMap<String, Series>,
|
||||
}
|
||||
|
||||
fn nan_series(n: usize) -> Series {
|
||||
vec![f64::NAN; n]
|
||||
}
|
||||
|
||||
pub fn ffill(v: &[f64], limit: usize) -> Series {
|
||||
let mut out = v.to_vec();
|
||||
let mut last = f64::NAN;
|
||||
let mut run = 0;
|
||||
for x in out.iter_mut() {
|
||||
if x.is_nan() {
|
||||
run += 1;
|
||||
if !last.is_nan() && run <= limit {
|
||||
*x = last;
|
||||
}
|
||||
} else {
|
||||
last = *x;
|
||||
run = 0;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// pandas `shift(k)`: positive k looks back, negative k looks ahead.
|
||||
pub fn shift(v: &[f64], k: i64) -> Series {
|
||||
let n = v.len() as i64;
|
||||
(0..n)
|
||||
.map(|i| {
|
||||
let j = i - k;
|
||||
if (0..n).contains(&j) {
|
||||
v[j as usize]
|
||||
} else {
|
||||
f64::NAN
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn sub(a: &[f64], b: &[f64]) -> Series {
|
||||
a.iter().zip(b).map(|(x, y)| x - y).collect()
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, PartialEq)]
|
||||
enum Agg {
|
||||
Mean,
|
||||
Sum,
|
||||
Max,
|
||||
Min,
|
||||
}
|
||||
|
||||
/// pandas `rolling(w, min_periods=1).<agg>()` over the trailing window [i-w+1, i].
|
||||
///
|
||||
/// Sum/Mean replicate pandas' `roll_sum`/`roll_mean` (window/aggregations.pyx)
|
||||
/// operation for operation: an incremental window with separate Kahan
|
||||
/// compensations for adds and removes, the "run of identical values" shortcut
|
||||
/// and the sign clamp of the mean. A plain per-window sum differs in the last
|
||||
/// bits, and HGB thresholds often sit exactly on real data values, so last-bit
|
||||
/// differences flip tree branches. Max/Min involve no arithmetic: exact as is.
|
||||
fn rolling(v: &[f64], w: usize, agg: Agg) -> Series {
|
||||
match agg {
|
||||
Agg::Sum | Agg::Mean => rolling_kahan(v, w, agg == Agg::Mean),
|
||||
Agg::Max | Agg::Min => (0..v.len())
|
||||
.map(|i| {
|
||||
let lo = (i + 1).saturating_sub(w);
|
||||
v[lo..=i]
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|x| !x.is_nan())
|
||||
.fold(f64::NAN, |acc, x| {
|
||||
if acc.is_nan() {
|
||||
x
|
||||
} else if agg == Agg::Max {
|
||||
acc.max(x)
|
||||
} else {
|
||||
acc.min(x)
|
||||
}
|
||||
})
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn rolling_kahan(v: &[f64], w: usize, mean: bool) -> Series {
|
||||
let (mut sum, mut comp_add, mut comp_remove) = (0.0f64, 0.0f64, 0.0f64);
|
||||
let (mut nobs, mut neg_ct, mut same_run) = (0i64, 0i64, 0i64);
|
||||
let mut prev = if v.is_empty() { f64::NAN } else { v[0] };
|
||||
let mut out = Vec::with_capacity(v.len());
|
||||
for i in 0..v.len() {
|
||||
// window [s, e) for row i; previous window was [i-w, i) clipped
|
||||
if i >= w {
|
||||
let val = v[i - w];
|
||||
if !val.is_nan() {
|
||||
nobs -= 1;
|
||||
let y = -val - comp_remove;
|
||||
let t = sum + y;
|
||||
comp_remove = t - sum - y;
|
||||
sum = t;
|
||||
if val.is_sign_negative() {
|
||||
neg_ct -= 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
let val = v[i];
|
||||
if !val.is_nan() {
|
||||
nobs += 1;
|
||||
let y = val - comp_add;
|
||||
let t = sum + y;
|
||||
comp_add = t - sum - y;
|
||||
sum = t;
|
||||
if val.is_sign_negative() {
|
||||
neg_ct += 1;
|
||||
}
|
||||
same_run = if val == prev { same_run + 1 } else { 1 };
|
||||
prev = val;
|
||||
}
|
||||
out.push(if nobs < 1 {
|
||||
f64::NAN
|
||||
} else if mean {
|
||||
if same_run >= nobs {
|
||||
prev
|
||||
} else {
|
||||
let r = sum / nobs as f64;
|
||||
if neg_ct == 0 && r < 0.0 || neg_ct == nobs && r > 0.0 {
|
||||
0.0
|
||||
} else {
|
||||
r
|
||||
}
|
||||
}
|
||||
} else if same_run >= nobs {
|
||||
prev * nobs as f64
|
||||
} else {
|
||||
sum
|
||||
});
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Hours since the last observed value (0 when observed; NaN before the first).
|
||||
fn hours_since_observed(mask: &[bool]) -> Series {
|
||||
let mut last: Option<usize> = None;
|
||||
mask.iter()
|
||||
.enumerate()
|
||||
.map(|(i, &m)| {
|
||||
if m {
|
||||
last = Some(i);
|
||||
}
|
||||
last.map_or(f64::NAN, |l| (i - l) as f64)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Feature columns by name, in pandas' construction order.
|
||||
pub fn build_features(grid: &Grid, station: &str, rain: Option<&[f64]>) -> Vec<(String, Series)> {
|
||||
let n = grid.times.len();
|
||||
let filled = |s: &str| -> Series {
|
||||
grid.level
|
||||
.get(s)
|
||||
.map_or_else(|| nan_series(n), |v| ffill(v, FFILL_LIMIT_H))
|
||||
};
|
||||
let mut cols: Vec<(String, Series)> = Vec::new();
|
||||
let mut put = |name: String, s: Series| cols.push((name, s));
|
||||
|
||||
let level = filled(station);
|
||||
let discharge = grid
|
||||
.discharge
|
||||
.get(station)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| nan_series(n));
|
||||
let mask: Vec<bool> = grid.level.get(station).map_or_else(
|
||||
|| vec![false; n],
|
||||
|v| v.iter().map(|x| !x.is_nan()).collect(),
|
||||
);
|
||||
|
||||
put("level".into(), level.clone());
|
||||
for k in [1, 2, 3, 6, 12, 24, 48, 72] {
|
||||
put(format!("level_lag_{k}"), shift(&level, k));
|
||||
}
|
||||
for k in [1, 3, 6, 12, 24] {
|
||||
put(format!("rise_{k}"), sub(&level, &shift(&level, k)));
|
||||
}
|
||||
put("roll_mean_6".into(), rolling(&level, 6, Agg::Mean));
|
||||
put("roll_mean_24".into(), rolling(&level, 24, Agg::Mean));
|
||||
put("roll_max_6".into(), rolling(&level, 6, Agg::Max));
|
||||
put("roll_max_24".into(), rolling(&level, 24, Agg::Max));
|
||||
put("roll_max_72".into(), rolling(&level, 72, Agg::Max));
|
||||
put("roll_min_24".into(), rolling(&level, 24, Agg::Min));
|
||||
|
||||
put("discharge".into(), discharge.clone());
|
||||
put("discharge_lag_6".into(), shift(&discharge, 6));
|
||||
put("discharge_lag_24".into(), shift(&discharge, 24));
|
||||
put(
|
||||
"discharge_rise_6".into(),
|
||||
sub(&discharge, &shift(&discharge, 6)),
|
||||
);
|
||||
|
||||
let age = hours_since_observed(&mask);
|
||||
put(
|
||||
"obs_age_h".into(),
|
||||
age.iter()
|
||||
.map(|&a| {
|
||||
if a <= FFILL_LIMIT_H as f64 {
|
||||
a
|
||||
} else {
|
||||
f64::NAN
|
||||
}
|
||||
})
|
||||
.collect(),
|
||||
);
|
||||
let mask_f: Series = mask.iter().map(|&m| if m { 1.0 } else { 0.0 }).collect();
|
||||
put("cov_24h".into(), rolling(&mask_f, 24, Agg::Mean));
|
||||
|
||||
for &(up, lead) in upstream_leads(station) {
|
||||
let u = filled(up);
|
||||
let u_rise_6 = sub(&u, &shift(&u, 6));
|
||||
let u_rollmax_24 = rolling(&u, 24, Agg::Max);
|
||||
let near = (lead - 3).max(0);
|
||||
put(format!("{up}_level_lag_{near}"), shift(&u, near));
|
||||
put(format!("{up}_level_lag_{lead}"), shift(&u, lead));
|
||||
put(format!("{up}_level_lag_{}", lead + 3), shift(&u, lead + 3));
|
||||
put(format!("{up}_rise_6_lag_{lead}"), shift(&u_rise_6, lead));
|
||||
put(
|
||||
format!("{up}_rollmax_24_lag_{near}"),
|
||||
shift(&u_rollmax_24, near),
|
||||
);
|
||||
}
|
||||
|
||||
if station != BASIN_ANCHOR {
|
||||
let p1 = filled(BASIN_ANCHOR);
|
||||
put("P1_level".into(), p1.clone());
|
||||
put("P1_rollmax_24".into(), rolling(&p1, 24, Agg::Max));
|
||||
put("P1_rise_24".into(), sub(&p1, &shift(&p1, 24)));
|
||||
}
|
||||
|
||||
let doy: Vec<f64> = grid.times.iter().map(|t| t.ordinal() as f64).collect();
|
||||
put(
|
||||
"doy_sin".into(),
|
||||
doy.iter().map(|d| (2.0 * PI * d / 365.25).sin()).collect(),
|
||||
);
|
||||
put(
|
||||
"doy_cos".into(),
|
||||
doy.iter().map(|d| (2.0 * PI * d / 365.25).cos()).collect(),
|
||||
);
|
||||
put(
|
||||
"is_monsoon".into(),
|
||||
grid.times
|
||||
.iter()
|
||||
.map(|t| {
|
||||
if MONSOON_MONTHS.contains(&t.month()) {
|
||||
1.0
|
||||
} else {
|
||||
0.0
|
||||
}
|
||||
})
|
||||
.collect(),
|
||||
);
|
||||
|
||||
if let Some(r) = rain {
|
||||
put("rain_6h".into(), rolling(r, 6, Agg::Sum));
|
||||
put("rain_24h".into(), rolling(r, 24, Agg::Sum));
|
||||
put("rain_72h".into(), rolling(r, 72, Agg::Sum));
|
||||
// forward sum over (t, t+24]
|
||||
let mut ahead = shift(r, -1);
|
||||
ahead.reverse();
|
||||
let mut fc = rolling(&ahead, 24, Agg::Sum);
|
||||
fc.reverse();
|
||||
put("rain_fc24".into(), fc);
|
||||
}
|
||||
cols
|
||||
}
|
||||
@@ -1,85 +0,0 @@
|
||||
//! Inference for scikit-learn HistGradientBoosting models exported as flat
|
||||
//! tree arrays (see export/export_fixture.py). Mirrors sklearn's
|
||||
//! `_predictor.pyx`: NaN follows `missing_go_to_left`, otherwise
|
||||
//! `x <= threshold` goes left; raw = baseline + sum of leaf values, summed in
|
||||
//! tree order (same order as sklearn, so results are bit-identical).
|
||||
|
||||
use serde::Deserialize;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
fn thresholds<'de, D: serde::Deserializer<'de>>(d: D) -> Result<Vec<f64>, D::Error> {
|
||||
let raw: Vec<Option<f64>> = Deserialize::deserialize(d)?;
|
||||
Ok(raw
|
||||
.into_iter()
|
||||
.map(|t| t.unwrap_or(f64::INFINITY))
|
||||
.collect())
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct Tree {
|
||||
feature: Vec<usize>,
|
||||
#[serde(deserialize_with = "thresholds")]
|
||||
threshold: Vec<f64>,
|
||||
missing_left: Vec<bool>,
|
||||
left: Vec<usize>,
|
||||
right: Vec<usize>,
|
||||
leaf: Vec<bool>,
|
||||
value: Vec<f64>,
|
||||
}
|
||||
|
||||
impl Tree {
|
||||
#[inline]
|
||||
fn predict(&self, x: &[f64]) -> f64 {
|
||||
let mut n = 0;
|
||||
while !self.leaf[n] {
|
||||
let v = x[self.feature[n]];
|
||||
let go_left = if v.is_nan() {
|
||||
self.missing_left[n]
|
||||
} else {
|
||||
v <= self.threshold[n]
|
||||
};
|
||||
n = if go_left { self.left[n] } else { self.right[n] };
|
||||
}
|
||||
self.value[n]
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize, PartialEq, Clone, Copy)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum Kind {
|
||||
Regressor,
|
||||
Classifier,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct Head {
|
||||
pub kind: Kind,
|
||||
baseline: f64,
|
||||
trees: Vec<Tree>,
|
||||
}
|
||||
|
||||
impl Head {
|
||||
/// Regressor: the prediction. Classifier: P(class 1), like predict_proba[:, 1].
|
||||
pub fn predict(&self, x: &[f64]) -> f64 {
|
||||
let mut raw = self.baseline;
|
||||
for t in &self.trees {
|
||||
raw += t.predict(x);
|
||||
}
|
||||
match self.kind {
|
||||
Kind::Regressor => raw,
|
||||
Kind::Classifier => 1.0 / (1.0 + (-raw).exp()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct Bundle {
|
||||
pub station_code: String,
|
||||
pub model_version: String,
|
||||
pub regression_target: Option<String>,
|
||||
pub feature_names: Vec<String>,
|
||||
pub horizons: Vec<u32>,
|
||||
pub sigma: BTreeMap<String, f64>,
|
||||
pub thresholds: BTreeMap<String, f64>,
|
||||
pub heads: BTreeMap<String, Head>,
|
||||
}
|
||||
@@ -1,211 +0,0 @@
|
||||
//! Parity + timing check: Rust features and HGB inference vs the Python output
|
||||
//! recorded in data/fixture_*.json.
|
||||
|
||||
mod features;
|
||||
mod hgb;
|
||||
|
||||
use chrono::NaiveDateTime;
|
||||
use serde::Deserialize;
|
||||
use std::collections::HashMap;
|
||||
use std::time::Instant;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct StationObs {
|
||||
water_level: Vec<Option<f64>>,
|
||||
discharge: Vec<Option<f64>>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Fixture {
|
||||
station: String,
|
||||
timestamps: Vec<String>,
|
||||
observed: HashMap<String, StationObs>,
|
||||
rain: Vec<Option<f64>>,
|
||||
feature_names: Vec<String>,
|
||||
features: Vec<Vec<Option<f64>>>,
|
||||
predictions: HashMap<String, Vec<f64>>,
|
||||
}
|
||||
|
||||
fn nanify(v: &[Option<f64>]) -> Vec<f64> {
|
||||
v.iter().map(|x| x.unwrap_or(f64::NAN)).collect()
|
||||
}
|
||||
|
||||
fn same(a: f64, b: f64, tol: f64) -> bool {
|
||||
(a.is_nan() && b.is_nan()) || (a - b).abs() <= tol * (1.0 + b.abs())
|
||||
}
|
||||
|
||||
fn matrix(cols: &[(String, Vec<f64>)], names: &[String]) -> Vec<Vec<f64>> {
|
||||
let by_name: HashMap<&str, &Vec<f64>> = cols.iter().map(|(k, v)| (k.as_str(), v)).collect();
|
||||
let picked: Vec<&Vec<f64>> = names
|
||||
.iter()
|
||||
.map(|n| {
|
||||
*by_name
|
||||
.get(n.as_str())
|
||||
.unwrap_or_else(|| panic!("missing feature {n}"))
|
||||
})
|
||||
.collect();
|
||||
let rows = picked[0].len();
|
||||
(0..rows)
|
||||
.map(|i| picked.iter().map(|c| c[i]).collect())
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let data = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("data");
|
||||
let t = Instant::now();
|
||||
let bundle: hgb::Bundle =
|
||||
serde_json::from_slice(&std::fs::read(data.join("model_P.1.json")).unwrap()).unwrap();
|
||||
println!(
|
||||
"model {} ({}): {} heads, {} features, target={:?}, horizons {:?}, sigma {:?}, thresholds {:?}; loaded in {:?}",
|
||||
bundle.station_code,
|
||||
bundle.model_version,
|
||||
bundle.heads.len(),
|
||||
bundle.feature_names.len(),
|
||||
bundle.regression_target,
|
||||
bundle.horizons,
|
||||
bundle.sigma,
|
||||
bundle.thresholds,
|
||||
t.elapsed()
|
||||
);
|
||||
|
||||
for name in ["flood2024", "recent2026"] {
|
||||
let fx: Fixture = serde_json::from_slice(
|
||||
&std::fs::read(data.join(format!("fixture_{name}.json"))).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(fx.feature_names, bundle.feature_names);
|
||||
let grid = features::Grid {
|
||||
times: fx
|
||||
.timestamps
|
||||
.iter()
|
||||
.map(|s| NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S").unwrap())
|
||||
.collect(),
|
||||
level: fx
|
||||
.observed
|
||||
.iter()
|
||||
.map(|(k, v)| (k.clone(), nanify(&v.water_level)))
|
||||
.collect(),
|
||||
discharge: fx
|
||||
.observed
|
||||
.iter()
|
||||
.map(|(k, v)| (k.clone(), nanify(&v.discharge)))
|
||||
.collect(),
|
||||
};
|
||||
let rain = nanify(&fx.rain);
|
||||
|
||||
// 1. feature parity
|
||||
let t = Instant::now();
|
||||
let cols = features::build_features(&grid, &fx.station, Some(&rain));
|
||||
let feat_time = t.elapsed();
|
||||
let rust_x = matrix(&cols, &bundle.feature_names);
|
||||
let py_x: Vec<Vec<f64>> = fx.features.iter().map(|r| nanify(r)).collect();
|
||||
let (mut cells, mut exact, mut close, mut worst) =
|
||||
(0usize, 0usize, 0usize, (0.0f64, String::new()));
|
||||
for (r, p) in rust_x.iter().zip(&py_x) {
|
||||
for (j, (a, b)) in r.iter().zip(p).enumerate() {
|
||||
cells += 1;
|
||||
if a.to_bits() == b.to_bits() || (a.is_nan() && b.is_nan()) {
|
||||
exact += 1;
|
||||
} else if same(*a, *b, 1e-9) {
|
||||
close += 1;
|
||||
let d = (a - b).abs();
|
||||
if d > worst.0 {
|
||||
worst = (d, bundle.feature_names[j].clone());
|
||||
}
|
||||
} else {
|
||||
panic!(
|
||||
"{name}: feature {} differs: rust {a} vs python {b}",
|
||||
bundle.feature_names[j]
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
println!(
|
||||
"\n[{name}] {} rows x {} features: {exact} bit-identical, {close} within 1e-9 (max |diff| {:.1e} in {}), 0 mismatches; Rust feature build {:?}",
|
||||
rust_x.len(),
|
||||
bundle.feature_names.len(),
|
||||
worst.0,
|
||||
if worst.1.is_empty() { "-" } else { &worst.1 },
|
||||
feat_time
|
||||
);
|
||||
let _ = cells;
|
||||
|
||||
// 2. inference parity, on Python's features and on Rust's own features
|
||||
let t = Instant::now();
|
||||
let mut report = Vec::new();
|
||||
for (head_name, head) in &bundle.heads {
|
||||
let want = &fx.predictions[head_name];
|
||||
let (mut max_py, mut max_rs, mut ident) = (0.0f64, 0.0f64, 0usize);
|
||||
for i in 0..rust_x.len() {
|
||||
let on_py = head.predict(&py_x[i]);
|
||||
let on_rs = head.predict(&rust_x[i]);
|
||||
if on_py.to_bits() == want[i].to_bits() {
|
||||
ident += 1;
|
||||
}
|
||||
max_py = max_py.max((on_py - want[i]).abs());
|
||||
max_rs = max_rs.max((on_rs - want[i]).abs());
|
||||
}
|
||||
report.push(format!(
|
||||
" {head_name:<10} bit-identical {ident}/{} max|diff| on py-features {max_py:.1e}, on rust-features {max_rs:.1e}",
|
||||
rust_x.len()
|
||||
));
|
||||
}
|
||||
let infer_time = t.elapsed();
|
||||
println!("{}", report.join("\n"));
|
||||
println!(
|
||||
" {} heads x {} rows x 2 inference passes in {:?}",
|
||||
bundle.heads.len(),
|
||||
rust_x.len(),
|
||||
infer_time
|
||||
);
|
||||
|
||||
// 3. serving-shaped work: 14-day window, forecast for the last hour
|
||||
let tail = 336.min(grid.times.len());
|
||||
let start = grid.times.len() - tail;
|
||||
let window = features::Grid {
|
||||
times: grid.times[start..].to_vec(),
|
||||
level: grid
|
||||
.level
|
||||
.iter()
|
||||
.map(|(k, v)| (k.clone(), v[start..].to_vec()))
|
||||
.collect(),
|
||||
discharge: grid
|
||||
.discharge
|
||||
.iter()
|
||||
.map(|(k, v)| (k.clone(), v[start..].to_vec()))
|
||||
.collect(),
|
||||
};
|
||||
let w_rain = rain[start..].to_vec();
|
||||
let reps = 200;
|
||||
let t = Instant::now();
|
||||
let mut last = Vec::new();
|
||||
for _ in 0..reps {
|
||||
let cols = features::build_features(&window, &fx.station, Some(&w_rain));
|
||||
let x = matrix(&cols, &bundle.feature_names);
|
||||
let row = x.last().unwrap();
|
||||
last = bundle
|
||||
.heads
|
||||
.iter()
|
||||
.map(|(k, h)| (k.clone(), h.predict(row)))
|
||||
.collect();
|
||||
}
|
||||
let per = t.elapsed() / reps;
|
||||
let level_now = window.level[&fx.station]
|
||||
.iter()
|
||||
.rev()
|
||||
.find(|x| !x.is_nan())
|
||||
.copied()
|
||||
.unwrap();
|
||||
let rise24 = last.iter().find(|(k, _)| k == "max_24").unwrap().1;
|
||||
let warn24 = last.iter().find(|(k, _)| k == "warn_24").unwrap().1;
|
||||
println!(
|
||||
" serving: 336 h window -> features + {} heads for the last hour: {:?} per station; at {} level {:.2} m, predicted 24 h max {:.2} m, warn_24 head p={:.3}",
|
||||
bundle.heads.len(),
|
||||
per,
|
||||
window.times.last().unwrap(),
|
||||
level_now,
|
||||
(rise24 + level_now).max(level_now),
|
||||
warn24
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,162 +0,0 @@
|
||||
-- Northern Thailand Ping River Monitor - PostgreSQL Database Schema
|
||||
-- This script initializes the database tables for water monitoring data
|
||||
|
||||
-- Enable required extensions
|
||||
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
|
||||
|
||||
-- Create schema for better organization
|
||||
CREATE SCHEMA IF NOT EXISTS water_monitor;
|
||||
SET search_path TO water_monitor, public;
|
||||
|
||||
-- Stations table - stores monitoring station information
|
||||
CREATE TABLE IF NOT EXISTS stations (
|
||||
id SERIAL PRIMARY KEY,
|
||||
station_code VARCHAR(10) UNIQUE NOT NULL,
|
||||
thai_name VARCHAR(255) NOT NULL,
|
||||
english_name VARCHAR(255) NOT NULL,
|
||||
latitude DECIMAL(10,8),
|
||||
longitude DECIMAL(11,8),
|
||||
geohash VARCHAR(20),
|
||||
elevation DECIMAL(8,2), -- meters above sea level
|
||||
river_basin VARCHAR(100),
|
||||
province VARCHAR(100),
|
||||
district VARCHAR(100),
|
||||
is_active BOOLEAN DEFAULT true,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
-- Water measurements table - stores time series data
|
||||
CREATE TABLE IF NOT EXISTS water_measurements (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
timestamp TIMESTAMP NOT NULL,
|
||||
station_id INTEGER NOT NULL,
|
||||
water_level NUMERIC(10,3), -- meters
|
||||
discharge NUMERIC(10,2), -- cubic meters per second
|
||||
discharge_percent NUMERIC(5,2), -- percentage of normal discharge
|
||||
status VARCHAR(20) DEFAULT 'active',
|
||||
data_quality VARCHAR(20) DEFAULT 'good', -- good, fair, poor, missing
|
||||
remarks TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (station_id) REFERENCES stations(id) ON DELETE CASCADE,
|
||||
UNIQUE(timestamp, station_id)
|
||||
);
|
||||
|
||||
-- Alert thresholds table - stores warning/danger levels for each station
|
||||
CREATE TABLE IF NOT EXISTS alert_thresholds (
|
||||
id SERIAL PRIMARY KEY,
|
||||
station_id INTEGER NOT NULL,
|
||||
threshold_type VARCHAR(20) NOT NULL, -- 'warning', 'danger', 'critical'
|
||||
water_level_min NUMERIC(10,3),
|
||||
water_level_max NUMERIC(10,3),
|
||||
discharge_min NUMERIC(10,2),
|
||||
discharge_max NUMERIC(10,2),
|
||||
is_active BOOLEAN DEFAULT true,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (station_id) REFERENCES stations(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
-- Data quality log - tracks data collection issues
|
||||
CREATE TABLE IF NOT EXISTS data_quality_log (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
timestamp TIMESTAMP NOT NULL,
|
||||
station_id INTEGER,
|
||||
issue_type VARCHAR(50) NOT NULL, -- 'connection_failed', 'invalid_data', 'missing_data'
|
||||
description TEXT,
|
||||
severity VARCHAR(20) DEFAULT 'info', -- 'info', 'warning', 'error', 'critical'
|
||||
resolved_at TIMESTAMP,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (station_id) REFERENCES stations(id) ON DELETE SET NULL
|
||||
);
|
||||
|
||||
-- Create indexes for better query performance
|
||||
CREATE INDEX IF NOT EXISTS idx_water_measurements_timestamp ON water_measurements(timestamp DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_water_measurements_station_id ON water_measurements(station_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_water_measurements_station_timestamp ON water_measurements(station_id, timestamp DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_water_measurements_status ON water_measurements(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_stations_code ON stations(station_code);
|
||||
CREATE INDEX IF NOT EXISTS idx_stations_active ON stations(is_active);
|
||||
CREATE INDEX IF NOT EXISTS idx_data_quality_timestamp ON data_quality_log(timestamp DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_data_quality_station ON data_quality_log(station_id);
|
||||
|
||||
-- Create a view for latest measurements per station
|
||||
CREATE OR REPLACE VIEW latest_measurements AS
|
||||
SELECT
|
||||
s.id as station_id,
|
||||
s.station_code,
|
||||
s.english_name,
|
||||
s.thai_name,
|
||||
s.latitude,
|
||||
s.longitude,
|
||||
s.province,
|
||||
s.river_basin,
|
||||
m.timestamp,
|
||||
m.water_level,
|
||||
m.discharge,
|
||||
m.discharge_percent,
|
||||
m.status,
|
||||
m.data_quality,
|
||||
CASE
|
||||
WHEN m.timestamp > CURRENT_TIMESTAMP - INTERVAL '2 hours' THEN 'online'
|
||||
WHEN m.timestamp > CURRENT_TIMESTAMP - INTERVAL '24 hours' THEN 'delayed'
|
||||
ELSE 'offline'
|
||||
END as station_status
|
||||
FROM stations s
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT * FROM water_measurements
|
||||
WHERE station_id = s.id
|
||||
ORDER BY timestamp DESC
|
||||
LIMIT 1
|
||||
) m ON true
|
||||
WHERE s.is_active = true
|
||||
ORDER BY s.station_code;
|
||||
|
||||
-- Create a function to update the updated_at timestamp
|
||||
CREATE OR REPLACE FUNCTION update_modified_column()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
NEW.updated_at = CURRENT_TIMESTAMP;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ language 'plpgsql';
|
||||
|
||||
-- Create triggers to automatically update updated_at
|
||||
DROP TRIGGER IF EXISTS update_stations_modtime ON stations;
|
||||
CREATE TRIGGER update_stations_modtime
|
||||
BEFORE UPDATE ON stations
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_modified_column();
|
||||
|
||||
-- Insert sample stations (Northern Thailand Ping River stations)
|
||||
INSERT INTO stations (id, station_code, thai_name, english_name, latitude, longitude, province, river_basin) VALUES
|
||||
(1, 'P.1', 'เชียงใหม่', 'Chiang Mai', 18.7883, 98.9853, 'Chiang Mai', 'Ping River'),
|
||||
(2, 'P.4A', 'ท่าแพ', 'Tha Phae', 18.7875, 99.0045, 'Chiang Mai', 'Ping River'),
|
||||
(3, 'P.12', 'สันป่าตอง', 'San Pa Tong', 18.6167, 98.9500, 'Chiang Mai', 'Ping River'),
|
||||
(4, 'P.20', 'ลำพูน', 'Lamphun', 18.5737, 99.0081, 'Lamphun', 'Ping River'),
|
||||
(5, 'P.30', 'ลี้', 'Li', 17.4833, 99.3000, 'Lamphun', 'Ping River'),
|
||||
(6, 'P.35', 'ป่าซาง', 'Pa Sang', 18.5444, 98.9397, 'Lamphun', 'Ping River'),
|
||||
(7, 'P.67', 'ตาก', 'Tak', 16.8839, 99.1267, 'Tak', 'Ping River'),
|
||||
(8, 'P.75', 'สามเงา', 'Sam Ngao', 17.1019, 99.4644, 'Tak', 'Ping River')
|
||||
ON CONFLICT (id) DO NOTHING;
|
||||
|
||||
-- Insert sample alert thresholds
|
||||
INSERT INTO alert_thresholds (station_id, threshold_type, water_level_min, water_level_max) VALUES
|
||||
(1, 'warning', 4.5, NULL),
|
||||
(1, 'danger', 6.0, NULL),
|
||||
(1, 'critical', 7.5, NULL),
|
||||
(2, 'warning', 4.0, NULL),
|
||||
(2, 'danger', 5.5, NULL),
|
||||
(2, 'critical', 7.0, NULL)
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
-- Grant permissions (adjust as needed for your setup)
|
||||
GRANT USAGE ON SCHEMA water_monitor TO postgres;
|
||||
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA water_monitor TO postgres;
|
||||
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA water_monitor TO postgres;
|
||||
|
||||
-- Optional: Create a read-only user for reporting
|
||||
-- CREATE USER water_monitor_readonly WITH PASSWORD 'readonly_password';
|
||||
-- GRANT USAGE ON SCHEMA water_monitor TO water_monitor_readonly;
|
||||
-- GRANT SELECT ON ALL TABLES IN SCHEMA water_monitor TO water_monitor_readonly;
|
||||
|
||||
COMMIT;
|
||||
+3
-36
@@ -1,38 +1,5 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Northern Thailand Ping River Monitor Package
|
||||
|
||||
A comprehensive real-time water level monitoring system for the Ping River Basin
|
||||
in Northern Thailand, covering Royal Irrigation Department (RID) stations.
|
||||
"""
|
||||
"""Ping River Monitor: the Python side, i.e. the flood-model training and
|
||||
evaluation stack (src/ml) and the archive backfills. The service (collection,
|
||||
forecast, notifications, website/API) is ping-collector in rust/."""
|
||||
|
||||
__version__ = "3.1.3"
|
||||
__author__ = "Ping River Monitor Team"
|
||||
__description__ = "Northern Thailand Ping River Monitoring System"
|
||||
|
||||
from .config import Config
|
||||
from .database_adapters import DatabaseAdapter, create_database_adapter
|
||||
from .exceptions import (
|
||||
APIConnectionError,
|
||||
ConfigurationError,
|
||||
DatabaseConnectionError,
|
||||
DataValidationError,
|
||||
WaterMonitorException,
|
||||
)
|
||||
from .models import DatabaseConfig, StationInfo, WaterMeasurement
|
||||
from .water_scraper_v3 import EnhancedWaterMonitorScraper
|
||||
|
||||
__all__ = [
|
||||
"EnhancedWaterMonitorScraper",
|
||||
"create_database_adapter",
|
||||
"DatabaseAdapter",
|
||||
"Config",
|
||||
"WaterMeasurement",
|
||||
"StationInfo",
|
||||
"DatabaseConfig",
|
||||
"WaterMonitorException",
|
||||
"DatabaseConnectionError",
|
||||
"APIConnectionError",
|
||||
"DataValidationError",
|
||||
"ConfigurationError",
|
||||
]
|
||||
-602
@@ -1,602 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Water Level Alerting System with Matrix Integration
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import html
|
||||
import os
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
import requests
|
||||
|
||||
try:
|
||||
from .config import Config
|
||||
from .database_adapters import create_database_adapter
|
||||
from .logging_config import get_logger
|
||||
except ImportError:
|
||||
import logging
|
||||
|
||||
from config import Config
|
||||
from database_adapters import create_database_adapter
|
||||
|
||||
def get_logger(name):
|
||||
return logging.getLogger(name)
|
||||
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
_BOLD_RE = re.compile(r"\*\*(.+?)\*\*")
|
||||
_URL_RE = re.compile(r"(https?://[^\s<]+)")
|
||||
|
||||
|
||||
def markdown_to_matrix_html(text: str) -> str:
|
||||
"""Convert the small Markdown subset we emit into Matrix-compatible HTML.
|
||||
|
||||
Matrix clients do NOT render Markdown in the plain ``body`` field; formatting
|
||||
only shows when an HTML ``formatted_body`` is sent alongside it. We only use
|
||||
``**bold**``, bare URLs and newlines, so a minimal converter is sufficient and
|
||||
avoids adding a Markdown dependency.
|
||||
"""
|
||||
# Escape HTML special chars first so station/message data can't inject markup.
|
||||
result = html.escape(text, quote=False)
|
||||
result = _BOLD_RE.sub(r"<strong>\1</strong>", result)
|
||||
result = _URL_RE.sub(r'<a href="\1">\1</a>', result)
|
||||
result = result.replace("\n", "<br/>")
|
||||
return result
|
||||
|
||||
|
||||
def strip_markdown(text: str) -> str:
|
||||
"""Produce a clean plain-text fallback for the Matrix ``body`` field."""
|
||||
return _BOLD_RE.sub(r"\1", text)
|
||||
|
||||
|
||||
class AlertLevel(Enum):
|
||||
INFO = "info"
|
||||
WARNING = "warning"
|
||||
CRITICAL = "critical"
|
||||
EMERGENCY = "emergency"
|
||||
|
||||
|
||||
@dataclass
|
||||
class WaterAlert:
|
||||
station_code: str
|
||||
station_name: str
|
||||
alert_type: str
|
||||
level: AlertLevel
|
||||
water_level: float
|
||||
threshold: float
|
||||
discharge: Optional[float] = None
|
||||
timestamp: Optional[datetime.datetime] = None
|
||||
message: Optional[str] = None
|
||||
|
||||
|
||||
class MatrixNotifier:
|
||||
def __init__(self, homeserver: str, access_token: str, room_id: str):
|
||||
self.homeserver = homeserver.rstrip("/")
|
||||
self.access_token = access_token
|
||||
self.room_id = room_id
|
||||
self.session = requests.Session()
|
||||
|
||||
def send_message(
|
||||
self, message: str, msgtype: str = "m.text", markdown: bool = True
|
||||
) -> bool:
|
||||
"""Send a message to the Matrix room.
|
||||
|
||||
When ``markdown`` is True (default) the ``message`` is treated as Markdown:
|
||||
a rendered HTML ``formatted_body`` is sent so clients show real formatting,
|
||||
with a plain-text ``body`` fallback for clients that ignore HTML.
|
||||
"""
|
||||
try:
|
||||
# Add transaction ID to prevent duplicates
|
||||
txn_id = datetime.datetime.now().strftime("%Y%m%d_%H%M%S_%f")
|
||||
url = f"{self.homeserver}/_matrix/client/v3/rooms/{self.room_id}/send/m.room.message/{txn_id}"
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.access_token}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
if markdown:
|
||||
data = {
|
||||
"msgtype": msgtype,
|
||||
"body": strip_markdown(message),
|
||||
"format": "org.matrix.custom.html",
|
||||
"formatted_body": markdown_to_matrix_html(message),
|
||||
}
|
||||
else:
|
||||
data = {"msgtype": msgtype, "body": message}
|
||||
|
||||
# Matrix API requires PUT when transaction ID is in the URL path
|
||||
response = self.session.put(url, headers=headers, json=data, timeout=10)
|
||||
response.raise_for_status()
|
||||
|
||||
logger.info(
|
||||
f"Matrix message sent successfully: {response.json().get('event_id')}"
|
||||
)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to send Matrix message: {e}")
|
||||
return False
|
||||
|
||||
def send_alert(self, alert: WaterAlert) -> bool:
|
||||
"""Send formatted water alert to Matrix"""
|
||||
emoji_map = {
|
||||
AlertLevel.INFO: "ℹ️",
|
||||
AlertLevel.WARNING: "⚠️",
|
||||
AlertLevel.CRITICAL: "🚨",
|
||||
AlertLevel.EMERGENCY: "🆘",
|
||||
}
|
||||
|
||||
emoji = emoji_map.get(alert.level, "📊")
|
||||
|
||||
message = f"""{emoji} **WATER LEVEL ALERT**
|
||||
|
||||
**Station:** {alert.station_code} ({alert.station_name})
|
||||
**Alert Type:** {alert.alert_type}
|
||||
**Severity:** {alert.level.value.upper()}
|
||||
|
||||
**Current Level:** {alert.water_level:.2f}m
|
||||
**Threshold:** {alert.threshold:.2f}m
|
||||
**Difference:** {(alert.water_level - alert.threshold):+.2f}m
|
||||
"""
|
||||
|
||||
if alert.discharge:
|
||||
message += f"**Discharge:** {alert.discharge:.1f} cms\n"
|
||||
|
||||
if alert.timestamp:
|
||||
message += f"**Time:** {alert.timestamp.strftime('%Y-%m-%d %H:%M:%S')}\n"
|
||||
|
||||
if alert.message:
|
||||
message += f"\n**Details:** {alert.message}\n"
|
||||
|
||||
# Add live dashboard link
|
||||
dashboard_url = os.getenv("ALERT_DASHBOARD_URL", "https://water.buildfor.life/")
|
||||
message += f"\n📈 **View Dashboard:** {dashboard_url}"
|
||||
|
||||
return self.send_message(message)
|
||||
|
||||
|
||||
class WaterLevelAlertSystem:
|
||||
# Stations upstream of Chiang Mai (and CNX itself) to monitor
|
||||
UPSTREAM_STATIONS = {
|
||||
"P.20", # Ban Chiang Dao
|
||||
"P.75", # Ban Chai Lat
|
||||
"P.92", # Ban Muang Aut
|
||||
"P.4A", # Ban Mae Taeng
|
||||
"P.67", # Ban Tae
|
||||
"P.21", # Ban Rim Tai
|
||||
"P.103", # Ring Bridge 3
|
||||
"P.1", # Nawarat Bridge (Chiang Mai)
|
||||
}
|
||||
|
||||
def __init__(self):
|
||||
self.db_adapter = None
|
||||
self.matrix_notifier = None
|
||||
self.thresholds = self._load_thresholds()
|
||||
|
||||
# Matrix configuration from environment
|
||||
matrix_homeserver = os.getenv("MATRIX_HOMESERVER", "https://matrix.org")
|
||||
matrix_token = os.getenv("MATRIX_ACCESS_TOKEN")
|
||||
matrix_room = os.getenv("MATRIX_ROOM_ID")
|
||||
|
||||
if matrix_token and matrix_room:
|
||||
self.matrix_notifier = MatrixNotifier(
|
||||
matrix_homeserver, matrix_token, matrix_room
|
||||
)
|
||||
logger.info("Matrix notifications enabled")
|
||||
else:
|
||||
logger.warning("Matrix configuration missing - notifications disabled")
|
||||
|
||||
def _load_thresholds(self) -> Dict[str, Dict[str, float]]:
|
||||
"""Load alert thresholds from config or database"""
|
||||
# Default thresholds for Northern Thailand stations
|
||||
return {
|
||||
"P.1": {
|
||||
# Zone-based thresholds for Nawarat Bridge (P.1)
|
||||
"zone_1": 3.7,
|
||||
"zone_2": 3.9,
|
||||
"zone_3": 4.0,
|
||||
"zone_4": 4.1,
|
||||
"zone_5": 4.2,
|
||||
"zone_6": 4.3,
|
||||
"zone_7": 4.6,
|
||||
"zone_8": 4.8,
|
||||
"newedge": 4.8, # Same as zone 8 or adjust as needed
|
||||
# Keep legacy thresholds for compatibility
|
||||
"warning": 3.7,
|
||||
"critical": 4.3,
|
||||
"emergency": 4.8,
|
||||
},
|
||||
"P.4A": {"warning": 4.5, "critical": 6.0, "emergency": 7.5},
|
||||
"P.20": {"warning": 3.0, "critical": 4.5, "emergency": 6.0},
|
||||
"P.21": {"warning": 4.0, "critical": 5.5, "emergency": 7.0},
|
||||
"P.67": {"warning": 6.0, "critical": 8.0, "emergency": 10.0},
|
||||
"P.75": {"warning": 5.5, "critical": 7.5, "emergency": 9.5},
|
||||
"P.103": {"warning": 7.0, "critical": 9.0, "emergency": 11.0},
|
||||
# Default for unknown stations
|
||||
"default": {"warning": 4.0, "critical": 6.0, "emergency": 8.0},
|
||||
}
|
||||
|
||||
def connect_database(self):
|
||||
"""Initialize database connection"""
|
||||
try:
|
||||
db_config = Config.get_database_config()
|
||||
self.db_adapter = create_database_adapter(
|
||||
db_config["type"], connection_string=db_config["connection_string"]
|
||||
)
|
||||
|
||||
if self.db_adapter.connect():
|
||||
logger.info("Database connection established for alerting")
|
||||
return True
|
||||
else:
|
||||
logger.error("Failed to connect to database")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Database connection error: {e}")
|
||||
return False
|
||||
|
||||
def check_water_levels(self) -> List[WaterAlert]:
|
||||
"""Check current water levels against thresholds"""
|
||||
alerts = []
|
||||
|
||||
if not self.db_adapter:
|
||||
logger.error("Database not connected")
|
||||
return alerts
|
||||
|
||||
try:
|
||||
# Get latest measurements
|
||||
measurements = self.db_adapter.get_latest_measurements(limit=50)
|
||||
|
||||
for measurement in measurements:
|
||||
station_code = measurement.get("station_code", "UNKNOWN")
|
||||
water_level = measurement.get("water_level")
|
||||
|
||||
if not water_level:
|
||||
continue
|
||||
|
||||
# Only alert for upstream stations and Chiang Mai
|
||||
if station_code not in self.UPSTREAM_STATIONS:
|
||||
continue
|
||||
|
||||
# Get thresholds for this station
|
||||
station_thresholds = self.thresholds.get(
|
||||
station_code, self.thresholds["default"]
|
||||
)
|
||||
|
||||
# Check each threshold level
|
||||
alert_level = None
|
||||
threshold_value = None
|
||||
alert_type = None
|
||||
|
||||
# Special handling for P.1 with zone-based thresholds
|
||||
if station_code == "P.1" and "zone_1" in station_thresholds:
|
||||
# Check all zones in reverse order (highest to lowest)
|
||||
zones = [
|
||||
("zone_8", 4.8, AlertLevel.EMERGENCY, "Zone 8 - Emergency"),
|
||||
("newedge", 4.8, AlertLevel.EMERGENCY, "NewEdge Alert Level"),
|
||||
("zone_7", 4.6, AlertLevel.CRITICAL, "Zone 7 - Critical"),
|
||||
("zone_6", 4.3, AlertLevel.CRITICAL, "Zone 6 - Critical"),
|
||||
("zone_5", 4.2, AlertLevel.WARNING, "Zone 5 - Warning"),
|
||||
("zone_4", 4.1, AlertLevel.WARNING, "Zone 4 - Warning"),
|
||||
("zone_3", 4.0, AlertLevel.WARNING, "Zone 3 - Warning"),
|
||||
("zone_2", 3.9, AlertLevel.INFO, "Zone 2 - Info"),
|
||||
("zone_1", 3.7, AlertLevel.INFO, "Zone 1 - Info"),
|
||||
]
|
||||
|
||||
for (
|
||||
zone_name,
|
||||
zone_threshold,
|
||||
zone_alert_level,
|
||||
zone_description,
|
||||
) in zones:
|
||||
if water_level >= zone_threshold:
|
||||
alert_level = zone_alert_level
|
||||
threshold_value = zone_threshold
|
||||
alert_type = zone_description
|
||||
break
|
||||
|
||||
else:
|
||||
# Standard threshold checking for other stations
|
||||
if water_level >= station_thresholds.get("emergency", float("inf")):
|
||||
alert_level = AlertLevel.EMERGENCY
|
||||
threshold_value = station_thresholds["emergency"]
|
||||
alert_type = "Emergency Water Level"
|
||||
elif water_level >= station_thresholds.get(
|
||||
"critical", float("inf")
|
||||
):
|
||||
alert_level = AlertLevel.CRITICAL
|
||||
threshold_value = station_thresholds["critical"]
|
||||
alert_type = "Critical Water Level"
|
||||
elif water_level >= station_thresholds.get("warning", float("inf")):
|
||||
alert_level = AlertLevel.WARNING
|
||||
threshold_value = station_thresholds["warning"]
|
||||
alert_type = "High Water Level"
|
||||
|
||||
if alert_level:
|
||||
alert = WaterAlert(
|
||||
station_code=station_code,
|
||||
station_name=measurement.get(
|
||||
"station_name_th", f"Station {station_code}"
|
||||
),
|
||||
alert_type=alert_type,
|
||||
level=alert_level,
|
||||
water_level=water_level,
|
||||
threshold=threshold_value,
|
||||
discharge=measurement.get("discharge"),
|
||||
timestamp=measurement.get("timestamp"),
|
||||
)
|
||||
alerts.append(alert)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error checking water levels: {e}")
|
||||
|
||||
return alerts
|
||||
|
||||
def check_data_freshness(self, max_age_hours: int = 12) -> List[WaterAlert]:
|
||||
"""Check if data is fresh enough"""
|
||||
alerts = []
|
||||
|
||||
if not self.db_adapter:
|
||||
return alerts
|
||||
|
||||
try:
|
||||
measurements = self.db_adapter.get_latest_measurements(limit=20)
|
||||
cutoff_time = datetime.datetime.now() - datetime.timedelta(
|
||||
hours=max_age_hours
|
||||
)
|
||||
|
||||
for measurement in measurements:
|
||||
timestamp = measurement.get("timestamp")
|
||||
if timestamp and timestamp < cutoff_time:
|
||||
station_code = measurement.get("station_code", "UNKNOWN")
|
||||
|
||||
age_hours = (
|
||||
datetime.datetime.now() - timestamp
|
||||
).total_seconds() / 3600
|
||||
|
||||
alert = WaterAlert(
|
||||
station_code=station_code,
|
||||
station_name=measurement.get(
|
||||
"station_name_th", f"Station {station_code}"
|
||||
),
|
||||
alert_type="Stale Data",
|
||||
level=AlertLevel.WARNING,
|
||||
water_level=measurement.get("water_level", 0),
|
||||
threshold=max_age_hours,
|
||||
timestamp=timestamp,
|
||||
message=f"No fresh data for {age_hours:.1f} hours",
|
||||
)
|
||||
alerts.append(alert)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error checking data freshness: {e}")
|
||||
|
||||
return alerts
|
||||
|
||||
def check_rate_of_change(self, lookback_hours: int = 3) -> List[WaterAlert]:
|
||||
"""Check for rapid water level changes over recent hours"""
|
||||
alerts = []
|
||||
|
||||
if not self.db_adapter:
|
||||
return alerts
|
||||
|
||||
try:
|
||||
# Define rate-of-change thresholds (meters per hour)
|
||||
rate_thresholds = {
|
||||
"P.1": {
|
||||
"warning": 0.15, # 15cm/hour - moderate rise
|
||||
"critical": 0.25, # 25cm/hour - rapid rise
|
||||
"emergency": 0.40, # 40cm/hour - very rapid rise
|
||||
},
|
||||
"default": {"warning": 0.20, "critical": 0.35, "emergency": 0.50},
|
||||
}
|
||||
|
||||
# Get recent measurements for each station
|
||||
cutoff_time = datetime.datetime.now() - datetime.timedelta(
|
||||
hours=lookback_hours
|
||||
)
|
||||
|
||||
# Get unique stations from latest data
|
||||
latest = self.db_adapter.get_latest_measurements(limit=20)
|
||||
station_codes = set(
|
||||
m.get("station_code") for m in latest if m.get("station_code")
|
||||
)
|
||||
|
||||
for station_code in station_codes:
|
||||
try:
|
||||
# Only alert for upstream stations and Chiang Mai
|
||||
if station_code not in self.UPSTREAM_STATIONS:
|
||||
continue
|
||||
|
||||
# Get measurements for this station in the time window
|
||||
current_time = datetime.datetime.now()
|
||||
measurements = self.db_adapter.get_measurements_by_timerange(
|
||||
start_time=cutoff_time,
|
||||
end_time=current_time,
|
||||
station_codes=[station_code],
|
||||
)
|
||||
|
||||
if len(measurements) < 2:
|
||||
continue # Need at least 2 points to calculate rate
|
||||
|
||||
# Sort by timestamp
|
||||
measurements = sorted(
|
||||
measurements, key=lambda m: m.get("timestamp")
|
||||
)
|
||||
|
||||
# Get oldest and newest measurements
|
||||
oldest = measurements[0]
|
||||
newest = measurements[-1]
|
||||
|
||||
oldest_time = oldest.get("timestamp")
|
||||
oldest_level = oldest.get("water_level")
|
||||
newest_time = newest.get("timestamp")
|
||||
newest_level = newest.get("water_level")
|
||||
|
||||
# Convert timestamp strings to datetime if needed
|
||||
if isinstance(oldest_time, str):
|
||||
oldest_time = datetime.datetime.fromisoformat(oldest_time)
|
||||
if isinstance(newest_time, str):
|
||||
newest_time = datetime.datetime.fromisoformat(newest_time)
|
||||
|
||||
# Calculate rate of change
|
||||
time_diff_hours = (newest_time - oldest_time).total_seconds() / 3600
|
||||
if time_diff_hours == 0:
|
||||
continue
|
||||
|
||||
level_change = newest_level - oldest_level
|
||||
rate_per_hour = level_change / time_diff_hours
|
||||
|
||||
# Only alert on rising water (positive rate)
|
||||
if rate_per_hour <= 0:
|
||||
continue
|
||||
|
||||
# Get station info from latest data
|
||||
station_info = next(
|
||||
(m for m in latest if m.get("station_code") == station_code), {}
|
||||
)
|
||||
station_name = station_info.get("station_name_th", station_code)
|
||||
|
||||
# Get thresholds for this station
|
||||
station_rate_threshold = rate_thresholds.get(
|
||||
station_code, rate_thresholds["default"]
|
||||
)
|
||||
|
||||
alert_level = None
|
||||
threshold_value = None
|
||||
alert_type = None
|
||||
|
||||
if rate_per_hour >= station_rate_threshold["emergency"]:
|
||||
alert_level = AlertLevel.EMERGENCY
|
||||
threshold_value = station_rate_threshold["emergency"]
|
||||
alert_type = "Very Rapid Water Level Rise"
|
||||
elif rate_per_hour >= station_rate_threshold["critical"]:
|
||||
alert_level = AlertLevel.CRITICAL
|
||||
threshold_value = station_rate_threshold["critical"]
|
||||
alert_type = "Rapid Water Level Rise"
|
||||
elif rate_per_hour >= station_rate_threshold["warning"]:
|
||||
alert_level = AlertLevel.WARNING
|
||||
threshold_value = station_rate_threshold["warning"]
|
||||
alert_type = "Moderate Water Level Rise"
|
||||
|
||||
if alert_level:
|
||||
message = (
|
||||
f"Rising at {rate_per_hour:.2f}m/h over last {time_diff_hours:.1f}h "
|
||||
f"(change: {level_change:+.2f}m)"
|
||||
)
|
||||
|
||||
alert = WaterAlert(
|
||||
station_code=station_code,
|
||||
station_name=station_name or f"Station {station_code}",
|
||||
alert_type=alert_type,
|
||||
level=alert_level,
|
||||
water_level=newest_level,
|
||||
threshold=threshold_value,
|
||||
timestamp=newest_time,
|
||||
message=message,
|
||||
)
|
||||
alerts.append(alert)
|
||||
|
||||
except Exception as station_error:
|
||||
logger.debug(
|
||||
f"Error checking rate of change for station {station_code}: {station_error}"
|
||||
)
|
||||
continue
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error checking rate of change: {e}")
|
||||
|
||||
return alerts
|
||||
|
||||
def send_alerts(self, alerts: List[WaterAlert]) -> int:
|
||||
"""Send alerts via configured channels"""
|
||||
sent_count = 0
|
||||
|
||||
if not alerts:
|
||||
return sent_count
|
||||
|
||||
if self.matrix_notifier:
|
||||
for alert in alerts:
|
||||
if self.matrix_notifier.send_alert(alert):
|
||||
sent_count += 1
|
||||
|
||||
# Could add other notification channels here:
|
||||
# - Email
|
||||
# - Discord
|
||||
# - Telegram
|
||||
# - SMS
|
||||
|
||||
return sent_count
|
||||
|
||||
def run_alert_check(self) -> Dict[str, int]:
|
||||
"""Run complete alert check cycle"""
|
||||
if not self.connect_database():
|
||||
return {"error": 1}
|
||||
|
||||
# Check water levels
|
||||
water_alerts = self.check_water_levels()
|
||||
|
||||
# Check data freshness
|
||||
data_alerts = self.check_data_freshness()
|
||||
|
||||
# Check rate of change (rapid rises)
|
||||
rate_alerts = self.check_rate_of_change()
|
||||
|
||||
# Combine alerts
|
||||
all_alerts = water_alerts + rate_alerts
|
||||
|
||||
# Send alerts
|
||||
sent_count = self.send_alerts(all_alerts)
|
||||
|
||||
logger.info(
|
||||
f"Alert check complete: {len(all_alerts)} alerts, {sent_count} sent"
|
||||
)
|
||||
|
||||
return {
|
||||
"water_alerts": len(water_alerts),
|
||||
"data_alerts": len(data_alerts),
|
||||
"rate_alerts": len(rate_alerts),
|
||||
"total_alerts": len(all_alerts),
|
||||
"sent": sent_count,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
"""Standalone alerting check"""
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description="Water Level Alert System")
|
||||
parser.add_argument("--check", action="store_true", help="Run alert check")
|
||||
parser.add_argument("--test", action="store_true", help="Send test message")
|
||||
args = parser.parse_args()
|
||||
|
||||
alerting = WaterLevelAlertSystem()
|
||||
|
||||
if args.test:
|
||||
if alerting.matrix_notifier:
|
||||
test_message = (
|
||||
"🧪 **Test Alert**\n\nThis is a test message from the Water Level Alert System.\n\n"
|
||||
"If you received this, Matrix notifications are working correctly!"
|
||||
)
|
||||
success = alerting.matrix_notifier.send_message(test_message)
|
||||
print(f"Test message sent: {success}")
|
||||
else:
|
||||
print("Matrix notifier not configured")
|
||||
|
||||
elif args.check:
|
||||
results = alerting.run_alert_check()
|
||||
print(f"Alert check results: {results}")
|
||||
|
||||
else:
|
||||
print("Use --check or --test")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+31
-342
@@ -1,354 +1,43 @@
|
||||
import os
|
||||
from typing import Any, Dict
|
||||
"""Database settings for the Python tools (training, evaluation, backfills).
|
||||
|
||||
The service itself is ping-collector (rust/collector); it reads the same
|
||||
environment / `.env` file (rust/collector/src/config.rs). PostgreSQL only:
|
||||
POSTGRES_CONNECTION_STRING, or POSTGRES_HOST / POSTGRES_PORT / POSTGRES_DB /
|
||||
POSTGRES_USER / POSTGRES_PASSWORD with the same defaults as ping-collector.
|
||||
"""
|
||||
|
||||
import os
|
||||
import urllib.parse
|
||||
from typing import Optional
|
||||
|
||||
# Load environment variables from .env file
|
||||
try:
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
except ImportError:
|
||||
# python-dotenv not installed, continue without it
|
||||
except ImportError: # the systemd units pass the variables in directly
|
||||
pass
|
||||
|
||||
try:
|
||||
from .exceptions import ConfigurationError
|
||||
from .models import DatabaseType
|
||||
except ImportError:
|
||||
# Handle case when running as standalone script
|
||||
class ConfigurationError(Exception):
|
||||
pass
|
||||
|
||||
from enum import Enum
|
||||
|
||||
class DatabaseType(Enum):
|
||||
SQLITE = "sqlite"
|
||||
MYSQL = "mysql"
|
||||
POSTGRESQL = "postgresql"
|
||||
INFLUXDB = "influxdb"
|
||||
VICTORIAMETRICS = "victoriametrics"
|
||||
|
||||
|
||||
class Config:
|
||||
"""Configuration class for the Water Level Monitor"""
|
||||
|
||||
# Database settings
|
||||
DATABASE_PATH = os.getenv("WATER_DB_PATH", "water_levels.db")
|
||||
|
||||
# Website settings
|
||||
TARGET_URL = "https://hyd-app-db.rid.go.th/hydro1h.html"
|
||||
API_URL = "https://hyd-app-db.rid.go.th/webservice/getGroupHourlyWaterLevelReportAllHL.ashx"
|
||||
THAIWATER_API_KEY = os.getenv("THAIWATER_API_KEY")
|
||||
|
||||
# Public flood notifications (ntfy). Off unless NTFY_SERVER is set.
|
||||
# NTFY_SERVER is what subscribers use (public https URL, shown on the
|
||||
# dashboard). NTFY_PUBLISH_URL is where the monitor POSTs; defaults to
|
||||
# NTFY_SERVER, set it to http://127.0.0.1:2586 when ntfy runs on the same
|
||||
# host so publishing never depends on DNS/proxy/tunnel being up.
|
||||
NTFY_SERVER = os.getenv("NTFY_SERVER", "").strip()
|
||||
NTFY_PUBLISH_URL = os.getenv("NTFY_PUBLISH_URL", "").strip() or NTFY_SERVER
|
||||
NTFY_TOPIC_PREFIX = os.getenv("NTFY_TOPIC_PREFIX", "ping").strip()
|
||||
NTFY_TOKEN = os.getenv("NTFY_TOKEN", "").strip() # publish token if ACL enabled
|
||||
PUBLIC_URL = os.getenv("PUBLIC_URL", "https://water.buildfor.life/").strip()
|
||||
REQUEST_TIMEOUT = int(os.getenv("REQUEST_TIMEOUT", "30"))
|
||||
USER_AGENT = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
|
||||
def database_url() -> Optional[str]:
|
||||
"""The PostgreSQL connection string, or None when none is configured."""
|
||||
url = os.getenv("POSTGRES_CONNECTION_STRING", "").strip()
|
||||
if url:
|
||||
return url
|
||||
password = os.getenv("POSTGRES_PASSWORD")
|
||||
if not password:
|
||||
return None
|
||||
user = urllib.parse.quote(os.getenv("POSTGRES_USER", "postgres"), safe="")
|
||||
host = os.getenv("POSTGRES_HOST", "localhost")
|
||||
port = os.getenv("POSTGRES_PORT", "5432")
|
||||
name = os.getenv("POSTGRES_DB", "water_monitoring")
|
||||
return (
|
||||
f"postgresql://{user}:{urllib.parse.quote(password, safe='')}"
|
||||
f"@{host}:{port}/{name}"
|
||||
)
|
||||
|
||||
# Database configuration
|
||||
# When DB_TYPE is not set explicitly, a configured Postgres connection wins over the sqlite default
|
||||
DB_TYPE = os.getenv(
|
||||
"DB_TYPE",
|
||||
"postgresql" if os.getenv("POSTGRES_CONNECTION_STRING") else "sqlite",
|
||||
).lower()
|
||||
|
||||
# VictoriaMetrics settings
|
||||
# Default to localhost; set VM_HOST in the environment for real deployments
|
||||
# (avoids committing infrastructure hostnames to the repo).
|
||||
VM_HOST = os.getenv("VM_HOST", "localhost")
|
||||
VM_PORT = int(os.getenv("VM_PORT", "443"))
|
||||
|
||||
# Support for HTTPS URLs (e.g., behind reverse proxy)
|
||||
VM_URL = os.getenv("VM_URL") # Full URL override (e.g., https://vm.example.com)
|
||||
|
||||
# InfluxDB settings
|
||||
INFLUX_HOST = os.getenv("INFLUX_HOST", "localhost")
|
||||
INFLUX_PORT = int(os.getenv("INFLUX_PORT", "8086"))
|
||||
INFLUX_DATABASE = os.getenv("INFLUX_DATABASE", "water_monitoring")
|
||||
INFLUX_USERNAME = os.getenv("INFLUX_USERNAME")
|
||||
INFLUX_PASSWORD = os.getenv("INFLUX_PASSWORD")
|
||||
|
||||
# PostgreSQL settings
|
||||
POSTGRES_CONNECTION_STRING = os.getenv("POSTGRES_CONNECTION_STRING")
|
||||
POSTGRES_HOST = os.getenv("POSTGRES_HOST", "localhost")
|
||||
POSTGRES_PORT = int(os.getenv("POSTGRES_PORT", "5432"))
|
||||
POSTGRES_DB = os.getenv("POSTGRES_DB", "water_monitoring")
|
||||
POSTGRES_USER = os.getenv("POSTGRES_USER", "postgres")
|
||||
POSTGRES_PASSWORD = os.getenv("POSTGRES_PASSWORD")
|
||||
|
||||
# MySQL settings
|
||||
MYSQL_CONNECTION_STRING = os.getenv("MYSQL_CONNECTION_STRING")
|
||||
|
||||
# HII/ThaiWater open api-v3 collection (rainfall + backup water level)
|
||||
# See docs/DATA_SOURCES.md. Requires a SQL DB_TYPE (sqlite/postgresql/mysql).
|
||||
ENABLE_HII_COLLECTION = os.getenv("ENABLE_HII_COLLECTION", "true").lower() in (
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
)
|
||||
HII_BASIN_CODE = int(os.getenv("HII_BASIN_CODE", "6")) # 6 = Ping Basin
|
||||
|
||||
# RID large-dam daily status (app.rid.go.th/reservoir) — Mae Ngat et al.
|
||||
ENABLE_RESERVOIR_COLLECTION = os.getenv(
|
||||
"ENABLE_RESERVOIR_COLLECTION", "true"
|
||||
).lower() in ("1", "true", "yes")
|
||||
# TTL for the /api/hii/*/latest response cache; source data changes hourly
|
||||
HII_CACHE_TTL_SECONDS = int(os.getenv("HII_CACHE_TTL_SECONDS", "120"))
|
||||
# TTL for the /measurements/latest response cache (hottest endpoint)
|
||||
LATEST_CACHE_TTL_SECONDS = int(os.getenv("LATEST_CACHE_TTL_SECONDS", "45"))
|
||||
|
||||
# TTL for /health check results (includes an external RID-API probe)
|
||||
HEALTH_CACHE_TTL_SECONDS = int(os.getenv("HEALTH_CACHE_TTL_SECONDS", "30"))
|
||||
|
||||
# Thread-pool size for blocking work in the web process (DB queries,
|
||||
# inference, health probes). Waiting threads are cheap; starving the pool
|
||||
# stalls every endpoint that needs a thread.
|
||||
EXECUTOR_THREADS = int(os.getenv("EXECUTOR_THREADS", "48"))
|
||||
|
||||
# Web server worker processes. Above 1, uvicorn forks workers and a
|
||||
# localhost lock port elects a single background-collection leader.
|
||||
WEB_WORKERS = int(os.getenv("WEB_WORKERS", "2"))
|
||||
COLLECTION_LEADER_PORT = int(os.getenv("COLLECTION_LEADER_PORT", "8901"))
|
||||
|
||||
# Umami analytics (self-hosted). The website id is public (it ships in the
|
||||
# dashboard <script> tag); server-side API tracking posts to /api/send.
|
||||
UMAMI_API_URL = os.getenv("UMAMI_API_URL", "https://stats.buildfor.life/api/send")
|
||||
UMAMI_WEBSITE_ID = os.getenv(
|
||||
"UMAMI_WEBSITE_ID", "00b2be73-8f5f-4400-9029-3be852eb08f7"
|
||||
)
|
||||
UMAMI_TRACK_API = os.getenv("UMAMI_TRACK_API", "true").lower() in (
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
)
|
||||
|
||||
# Scheduler settings
|
||||
SCRAPING_INTERVAL_HOURS = int(os.getenv("SCRAPING_INTERVAL_HOURS", "1"))
|
||||
# RID is polled this many minutes past each hour; while the current hour is
|
||||
# not published yet (the evening report often lags 1-2 h) it is retried
|
||||
# every RID_RETRY_MINUTES. Late readings are stored either way.
|
||||
RID_POLL_OFFSET_MINUTES = int(os.getenv("RID_POLL_OFFSET_MINUTES", "5"))
|
||||
RID_RETRY_MINUTES = int(os.getenv("RID_RETRY_MINUTES", "10"))
|
||||
# On startup the collection leader re-fetches RID reports with holes from
|
||||
# the last N days (0 = off). Older history: `run.py --fill-gaps all`.
|
||||
GAP_FILL_DAYS_ON_START = int(os.getenv("GAP_FILL_DAYS_ON_START", "14"))
|
||||
# Leader -> other workers hand-off for /scraping/status (default: temp dir).
|
||||
# With ping-collector, point both services at the same file inside the app
|
||||
# dir (the systemd units use PrivateTmp).
|
||||
SCRAPING_STATUS_FILE = os.getenv("SCRAPING_STATUS_FILE", "")
|
||||
# false: data collection runs in the Rust ping-collector service
|
||||
# (rust/collector); this app then only serves, forecasts and notifies,
|
||||
# re-forecasting when the newest RID reading advances (polled every
|
||||
# FORECAST_POLL_SECONDS) and at least once per clock hour.
|
||||
COLLECTION_ENABLED = os.getenv("COLLECTION_ENABLED", "true").lower() in (
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
)
|
||||
FORECAST_POLL_SECONDS = int(os.getenv("FORECAST_POLL_SECONDS", "60"))
|
||||
# true: ping-collector also forecasts (rust/forecast) and sends the ntfy
|
||||
# notifications; this app serves its stored forecast (forecast_current)
|
||||
# and does neither. Read by both services from the shared .env.
|
||||
FORECAST_IN_COLLECTOR = os.getenv("FORECAST_IN_COLLECTOR", "false").lower() in (
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
)
|
||||
# A stored forecast older than this is not served; /forecast then computes
|
||||
# in-process as before (the collector is down or has no models).
|
||||
FORECAST_MAX_AGE_SECONDS = int(os.getenv("FORECAST_MAX_AGE_SECONDS", "7200"))
|
||||
|
||||
# Logging settings
|
||||
LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO")
|
||||
LOG_FILE = os.getenv("LOG_FILE", "water_monitor.log")
|
||||
LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s"
|
||||
|
||||
# Data retention
|
||||
DATA_RETENTION_DAYS = int(os.getenv("DATA_RETENTION_DAYS", "365"))
|
||||
|
||||
# Retry settings
|
||||
MAX_RETRIES = int(os.getenv("MAX_RETRIES", "3"))
|
||||
RETRY_DELAY_SECONDS = int(os.getenv("RETRY_DELAY_SECONDS", "60"))
|
||||
|
||||
# Station configuration
|
||||
# Runtime-writable JSON file that persists the station mapping across restarts
|
||||
# (station CRUD via the API writes here). If it does not exist, the bundled
|
||||
# defaults in src/data/stations.json are used to seed it.
|
||||
STATION_CONFIG_PATH = os.getenv("STATION_CONFIG_PATH", "stations.json")
|
||||
|
||||
# Web API / CORS settings
|
||||
# Comma-separated list of allowed origins. Defaults to none (same-origin only);
|
||||
# set CORS_ALLOW_ORIGINS to a specific list of front-end origins in production.
|
||||
# Credentials are only enabled when explicit (non-wildcard) origins are set,
|
||||
# because "*" + credentials is rejected by browsers and unsafe.
|
||||
CORS_ALLOW_ORIGINS = [
|
||||
origin.strip()
|
||||
for origin in os.getenv("CORS_ALLOW_ORIGINS", "").split(",")
|
||||
if origin.strip()
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def validate_config(cls) -> bool:
|
||||
"""Validate configuration settings"""
|
||||
errors = []
|
||||
|
||||
# Validate database type
|
||||
try:
|
||||
DatabaseType(cls.DB_TYPE)
|
||||
except ValueError:
|
||||
errors.append(f"Invalid DB_TYPE: {cls.DB_TYPE}")
|
||||
|
||||
# Validate database-specific settings
|
||||
if cls.DB_TYPE == "victoriametrics":
|
||||
if not cls.VM_HOST:
|
||||
errors.append("VM_HOST is required for VictoriaMetrics")
|
||||
if not isinstance(cls.VM_PORT, int) or cls.VM_PORT <= 0:
|
||||
errors.append("VM_PORT must be a positive integer")
|
||||
|
||||
elif cls.DB_TYPE == "influxdb":
|
||||
if not cls.INFLUX_HOST:
|
||||
errors.append("INFLUX_HOST is required for InfluxDB")
|
||||
if not cls.INFLUX_DATABASE:
|
||||
errors.append("INFLUX_DATABASE is required for InfluxDB")
|
||||
|
||||
elif cls.DB_TYPE in ["postgresql", "mysql"]:
|
||||
if cls.DB_TYPE == "postgresql":
|
||||
# Check if either connection string or individual components are provided
|
||||
if not cls.POSTGRES_CONNECTION_STRING:
|
||||
# If no connection string, check individual components
|
||||
if not cls.POSTGRES_HOST:
|
||||
errors.append("POSTGRES_HOST is required for PostgreSQL")
|
||||
if not cls.POSTGRES_USER:
|
||||
errors.append("POSTGRES_USER is required for PostgreSQL")
|
||||
if not cls.POSTGRES_PASSWORD:
|
||||
errors.append("POSTGRES_PASSWORD is required for PostgreSQL")
|
||||
if not cls.POSTGRES_DB:
|
||||
errors.append("POSTGRES_DB is required for PostgreSQL")
|
||||
else: # mysql
|
||||
if not cls.MYSQL_CONNECTION_STRING:
|
||||
errors.append("MYSQL_CONNECTION_STRING is required for MySQL")
|
||||
|
||||
# Validate numeric settings
|
||||
if cls.SCRAPING_INTERVAL_HOURS <= 0:
|
||||
errors.append("SCRAPING_INTERVAL_HOURS must be positive")
|
||||
|
||||
if not 0 <= cls.RID_POLL_OFFSET_MINUTES < 60:
|
||||
errors.append("RID_POLL_OFFSET_MINUTES must be between 0 and 59")
|
||||
|
||||
if cls.RID_RETRY_MINUTES <= 0:
|
||||
errors.append("RID_RETRY_MINUTES must be positive")
|
||||
|
||||
if cls.DATA_RETENTION_DAYS <= 0:
|
||||
errors.append("DATA_RETENTION_DAYS must be positive")
|
||||
|
||||
if errors:
|
||||
raise ConfigurationError(f"Configuration errors: {'; '.join(errors)}")
|
||||
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def get_database_config(cls) -> Dict[str, Any]:
|
||||
"""Returns database configuration based on DB_TYPE"""
|
||||
if cls.DB_TYPE == "victoriametrics":
|
||||
return {"type": "victoriametrics", "host": cls.VM_HOST, "port": cls.VM_PORT}
|
||||
elif cls.DB_TYPE == "influxdb":
|
||||
return {
|
||||
"type": "influxdb",
|
||||
"host": cls.INFLUX_HOST,
|
||||
"port": cls.INFLUX_PORT,
|
||||
"database": cls.INFLUX_DATABASE,
|
||||
"username": cls.INFLUX_USERNAME,
|
||||
"password": cls.INFLUX_PASSWORD,
|
||||
}
|
||||
elif cls.DB_TYPE == "postgresql":
|
||||
# Use individual components if POSTGRES_CONNECTION_STRING is not provided
|
||||
if cls.POSTGRES_CONNECTION_STRING:
|
||||
return {
|
||||
"type": "postgresql",
|
||||
"connection_string": cls.POSTGRES_CONNECTION_STRING,
|
||||
}
|
||||
else:
|
||||
# Build connection string from components (automatically URL-encodes password)
|
||||
import urllib.parse
|
||||
|
||||
if not cls.POSTGRES_PASSWORD:
|
||||
raise ConfigurationError(
|
||||
"POSTGRES_PASSWORD is required for PostgreSQL (no default is provided)"
|
||||
)
|
||||
password = urllib.parse.quote(cls.POSTGRES_PASSWORD, safe="")
|
||||
connection_string = (
|
||||
f"postgresql://{cls.POSTGRES_USER}:{password}"
|
||||
f"@{cls.POSTGRES_HOST}:{cls.POSTGRES_PORT}/{cls.POSTGRES_DB}"
|
||||
)
|
||||
return {"type": "postgresql", "connection_string": connection_string}
|
||||
elif cls.DB_TYPE == "mysql":
|
||||
if not cls.MYSQL_CONNECTION_STRING:
|
||||
raise ConfigurationError(
|
||||
"MYSQL_CONNECTION_STRING is required for MySQL (no default is provided)"
|
||||
)
|
||||
return {"type": "mysql", "connection_string": cls.MYSQL_CONNECTION_STRING}
|
||||
else: # sqlite
|
||||
return {
|
||||
"type": "sqlite",
|
||||
"connection_string": f"sqlite:///{cls.DATABASE_PATH}",
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def get_all_settings(cls) -> Dict[str, Any]:
|
||||
"""Returns all configuration settings"""
|
||||
return {
|
||||
"DB_TYPE": cls.DB_TYPE,
|
||||
"DATABASE_PATH": cls.DATABASE_PATH,
|
||||
"TARGET_URL": cls.TARGET_URL,
|
||||
"API_URL": cls.API_URL,
|
||||
"REQUEST_TIMEOUT": cls.REQUEST_TIMEOUT,
|
||||
"SCRAPING_INTERVAL_HOURS": cls.SCRAPING_INTERVAL_HOURS,
|
||||
"RID_POLL_OFFSET_MINUTES": cls.RID_POLL_OFFSET_MINUTES,
|
||||
"RID_RETRY_MINUTES": cls.RID_RETRY_MINUTES,
|
||||
"GAP_FILL_DAYS_ON_START": cls.GAP_FILL_DAYS_ON_START,
|
||||
"COLLECTION_ENABLED": cls.COLLECTION_ENABLED,
|
||||
"FORECAST_IN_COLLECTOR": cls.FORECAST_IN_COLLECTOR,
|
||||
"LOG_LEVEL": cls.LOG_LEVEL,
|
||||
"LOG_FILE": cls.LOG_FILE,
|
||||
"DATA_RETENTION_DAYS": cls.DATA_RETENTION_DAYS,
|
||||
"MAX_RETRIES": cls.MAX_RETRIES,
|
||||
"RETRY_DELAY_SECONDS": cls.RETRY_DELAY_SECONDS,
|
||||
"VM_HOST": cls.VM_HOST,
|
||||
"VM_PORT": cls.VM_PORT,
|
||||
"INFLUX_HOST": cls.INFLUX_HOST,
|
||||
"INFLUX_PORT": cls.INFLUX_PORT,
|
||||
"INFLUX_DATABASE": cls.INFLUX_DATABASE,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def print_settings(cls):
|
||||
"""Prints all current settings"""
|
||||
print("=== Water Level Monitor Configuration ===")
|
||||
for key, value in cls.get_all_settings().items():
|
||||
# Hide sensitive information
|
||||
if "PASSWORD" in key and value:
|
||||
value = "*" * len(str(value))
|
||||
print(f"{key}: {value}")
|
||||
print("=" * 45)
|
||||
|
||||
print("\nDatabase Configuration:")
|
||||
db_config = cls.get_database_config()
|
||||
for key, value in db_config.items():
|
||||
if "password" in key and value:
|
||||
value = "*" * len(str(value))
|
||||
print(f" {key}: {value}")
|
||||
print("=" * 45)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
Config.print_settings()
|
||||
url = database_url()
|
||||
if url and "@" in url:
|
||||
url = url.split("://", 1)[0] + "://***@" + url.rsplit("@", 1)[1]
|
||||
print(url or "no database configured")
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,352 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Demo script showing different database backend options for water monitoring
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import os
|
||||
import sys
|
||||
|
||||
from water_scraper_v3 import EnhancedWaterMonitorScraper
|
||||
|
||||
|
||||
def demo_sqlite():
|
||||
"""Demo with SQLite (local development)"""
|
||||
print("=" * 60)
|
||||
print("🗄️ SQLite Demo (Local Development)")
|
||||
print("=" * 60)
|
||||
|
||||
config = {"type": "sqlite", "connection_string": "sqlite:///demo_water_sqlite.db"}
|
||||
|
||||
try:
|
||||
scraper = EnhancedWaterMonitorScraper(config)
|
||||
|
||||
# Fetch and save data
|
||||
print("Fetching data from API...")
|
||||
data = scraper.fetch_water_data()
|
||||
|
||||
if data:
|
||||
print(f"✓ Fetched {len(data)} data points")
|
||||
success = scraper.save_to_database(data)
|
||||
|
||||
if success:
|
||||
print("✓ Data saved to SQLite database")
|
||||
|
||||
# Show latest data
|
||||
latest = scraper.get_latest_data(5)
|
||||
print(f"\nLatest 5 measurements:")
|
||||
for measurement in latest:
|
||||
print(
|
||||
f" • {measurement['station_code']} ({measurement['station_name_en']}): "
|
||||
f"{measurement['water_level']:.2f}m, {measurement['discharge']:.1f} cms"
|
||||
)
|
||||
else:
|
||||
print("✗ Failed to save data")
|
||||
else:
|
||||
print("✗ No data fetched")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
|
||||
|
||||
def demo_influxdb():
|
||||
"""Demo with InfluxDB (requires InfluxDB running)"""
|
||||
print("\n" + "=" * 60)
|
||||
print("📊 InfluxDB Demo (Time-Series Database)")
|
||||
print("=" * 60)
|
||||
|
||||
config = {
|
||||
"type": "influxdb",
|
||||
"host": "localhost",
|
||||
"port": 8086,
|
||||
"database": "water_monitoring_demo",
|
||||
"username": None, # Set if authentication is enabled
|
||||
"password": None,
|
||||
}
|
||||
|
||||
try:
|
||||
scraper = EnhancedWaterMonitorScraper(config)
|
||||
|
||||
if scraper.db_adapter and scraper.db_adapter.client:
|
||||
print("✓ Connected to InfluxDB")
|
||||
|
||||
# Fetch and save data
|
||||
print("Fetching data from API...")
|
||||
data = scraper.fetch_water_data()
|
||||
|
||||
if data:
|
||||
print(f"✓ Fetched {len(data)} data points")
|
||||
success = scraper.save_to_database(data)
|
||||
|
||||
if success:
|
||||
print("✓ Data saved to InfluxDB")
|
||||
print("💡 You can now query this data in Grafana or InfluxDB CLI")
|
||||
print(
|
||||
" Example query: SELECT * FROM water_data ORDER BY time DESC LIMIT 10"
|
||||
)
|
||||
else:
|
||||
print("✗ Failed to save data")
|
||||
else:
|
||||
print("✗ No data fetched")
|
||||
else:
|
||||
print("✗ Could not connect to InfluxDB")
|
||||
print(
|
||||
"💡 Make sure InfluxDB is running: docker run -p 8086:8086 influxdb:1.8"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
print("💡 InfluxDB might not be running or accessible")
|
||||
|
||||
|
||||
def demo_postgresql():
|
||||
"""Demo with PostgreSQL (requires PostgreSQL running)"""
|
||||
print("\n" + "=" * 60)
|
||||
print("🐘 PostgreSQL Demo (Relational Database)")
|
||||
print("=" * 60)
|
||||
|
||||
config = {
|
||||
"type": "postgresql",
|
||||
"connection_string": "postgresql://postgres:password@localhost:5432/water_monitoring",
|
||||
}
|
||||
|
||||
try:
|
||||
scraper = EnhancedWaterMonitorScraper(config)
|
||||
|
||||
if scraper.db_adapter and scraper.db_adapter.engine:
|
||||
print("✓ Connected to PostgreSQL")
|
||||
|
||||
# Fetch and save data
|
||||
print("Fetching data from API...")
|
||||
data = scraper.fetch_water_data()
|
||||
|
||||
if data:
|
||||
print(f"✓ Fetched {len(data)} data points")
|
||||
success = scraper.save_to_database(data)
|
||||
|
||||
if success:
|
||||
print("✓ Data saved to PostgreSQL")
|
||||
print("💡 You can now query this data with SQL")
|
||||
print(
|
||||
" Example: SELECT * FROM water_measurements ORDER BY timestamp DESC LIMIT 10;"
|
||||
)
|
||||
else:
|
||||
print("✗ Failed to save data")
|
||||
else:
|
||||
print("✗ No data fetched")
|
||||
else:
|
||||
print("✗ Could not connect to PostgreSQL")
|
||||
print("💡 Make sure PostgreSQL is running with correct credentials")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
print("💡 PostgreSQL might not be running or credentials might be wrong")
|
||||
|
||||
|
||||
def demo_mysql():
|
||||
"""Demo with MySQL (requires MySQL running)"""
|
||||
print("\n" + "=" * 60)
|
||||
print("🐬 MySQL Demo (Relational Database)")
|
||||
print("=" * 60)
|
||||
|
||||
config = {
|
||||
"type": "mysql",
|
||||
"connection_string": "mysql://root:password@localhost:3306/water_monitoring",
|
||||
}
|
||||
|
||||
try:
|
||||
scraper = EnhancedWaterMonitorScraper(config)
|
||||
|
||||
if scraper.db_adapter and scraper.db_adapter.engine:
|
||||
print("✓ Connected to MySQL")
|
||||
|
||||
# Fetch and save data
|
||||
print("Fetching data from API...")
|
||||
data = scraper.fetch_water_data()
|
||||
|
||||
if data:
|
||||
print(f"✓ Fetched {len(data)} data points")
|
||||
success = scraper.save_to_database(data)
|
||||
|
||||
if success:
|
||||
print("✓ Data saved to MySQL")
|
||||
print("💡 You can now query this data with SQL")
|
||||
print(
|
||||
" Example: SELECT * FROM water_measurements ORDER BY timestamp DESC LIMIT 10;"
|
||||
)
|
||||
else:
|
||||
print("✗ Failed to save data")
|
||||
else:
|
||||
print("✗ No data fetched")
|
||||
else:
|
||||
print("✗ Could not connect to MySQL")
|
||||
print("💡 Make sure MySQL is running with correct credentials")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
print("💡 MySQL might not be running or credentials might be wrong")
|
||||
|
||||
|
||||
def demo_victoriametrics():
|
||||
"""Demo with VictoriaMetrics (supports both local and HTTPS configurations)"""
|
||||
print("\n" + "=" * 60)
|
||||
print("⚡ VictoriaMetrics Demo (High-Performance Metrics)")
|
||||
print("=" * 60)
|
||||
|
||||
# Use configuration from environment or config.py
|
||||
from config import Config
|
||||
|
||||
db_config = Config.get_database_config()
|
||||
|
||||
if db_config["type"] != "victoriametrics":
|
||||
# Fallback to default local configuration
|
||||
config = {"type": "victoriametrics", "host": "vm.newedge.house", "port": 443}
|
||||
else:
|
||||
config = db_config
|
||||
|
||||
print(f"Connecting to: {config['host']}:{config['port']}")
|
||||
|
||||
try:
|
||||
scraper = EnhancedWaterMonitorScraper(config)
|
||||
|
||||
if scraper.db_adapter:
|
||||
# Test connection using the adapter's connect method
|
||||
if scraper.db_adapter.connect():
|
||||
print("✓ Connected to VictoriaMetrics")
|
||||
|
||||
# Fetch and save data
|
||||
print("Fetching data from API...")
|
||||
data = scraper.fetch_water_data()
|
||||
|
||||
if data:
|
||||
print(f"✓ Fetched {len(data)} data points")
|
||||
success = scraper.save_to_database(data)
|
||||
|
||||
if success:
|
||||
print("✓ Data saved to VictoriaMetrics")
|
||||
print("💡 You can now query this data via Prometheus API")
|
||||
|
||||
# Show appropriate query URL based on configuration
|
||||
base_url = scraper.db_adapter.base_url
|
||||
print(f" Example: {base_url}/api/v1/query?query=water_level")
|
||||
print(f" Health check: {base_url}/health")
|
||||
else:
|
||||
print("✗ Failed to save data")
|
||||
else:
|
||||
print("✗ No data fetched")
|
||||
else:
|
||||
print("✗ Could not connect to VictoriaMetrics")
|
||||
if config["host"] == "localhost":
|
||||
print("💡 Make sure VictoriaMetrics is running locally:")
|
||||
print(" docker run -p 8428:8428 victoriametrics/victoria-metrics")
|
||||
else:
|
||||
print(
|
||||
f"💡 Check if VictoriaMetrics is accessible at {config['host']}:{config['port']}"
|
||||
)
|
||||
print("💡 Verify HTTPS configuration and network connectivity")
|
||||
else:
|
||||
print("✗ Failed to initialize VictoriaMetrics adapter")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
print("💡 Check your VictoriaMetrics configuration and network connectivity")
|
||||
|
||||
|
||||
def show_recommendations():
|
||||
"""Show database recommendations"""
|
||||
print("\n" + "=" * 60)
|
||||
print("🏆 Database Recommendations")
|
||||
print("=" * 60)
|
||||
|
||||
recommendations = [
|
||||
{
|
||||
"name": "InfluxDB",
|
||||
"best_for": "Time-series data, Grafana dashboards",
|
||||
"pros": [
|
||||
"Purpose-built for time-series",
|
||||
"Great compression",
|
||||
"Built-in retention",
|
||||
],
|
||||
"cons": ["Learning curve", "Less flexible for complex queries"],
|
||||
"use_case": "Recommended for most water monitoring deployments",
|
||||
},
|
||||
{
|
||||
"name": "PostgreSQL + TimescaleDB",
|
||||
"best_for": "Complex queries, existing PostgreSQL infrastructure",
|
||||
"pros": ["Mature ecosystem", "SQL compatibility", "ACID compliance"],
|
||||
"cons": ["More complex setup", "Higher resource usage"],
|
||||
"use_case": "Best for organizations already using PostgreSQL",
|
||||
},
|
||||
{
|
||||
"name": "VictoriaMetrics",
|
||||
"best_for": "High-performance metrics, Prometheus compatibility",
|
||||
"pros": ["Extremely fast", "Low resource usage", "Better compression"],
|
||||
"cons": ["Newer ecosystem", "Less tooling"],
|
||||
"use_case": "Best for high-volume, performance-critical deployments",
|
||||
},
|
||||
{
|
||||
"name": "MySQL",
|
||||
"best_for": "Existing MySQL infrastructure, familiar SQL",
|
||||
"pros": ["Familiar", "Mature", "Wide support"],
|
||||
"cons": ["Not optimized for time-series", "Manual optimization needed"],
|
||||
"use_case": "Good for organizations with existing MySQL expertise",
|
||||
},
|
||||
]
|
||||
|
||||
for rec in recommendations:
|
||||
print(f"\n📊 {rec['name']}")
|
||||
print(f" Best for: {rec['best_for']}")
|
||||
print(f" Pros: {', '.join(rec['pros'])}")
|
||||
print(f" Cons: {', '.join(rec['cons'])}")
|
||||
print(f" 💡 {rec['use_case']}")
|
||||
|
||||
|
||||
def main():
|
||||
"""Main demo function"""
|
||||
print("🌊 Thailand Water Monitor - Database Backend Demo")
|
||||
print("This demo shows how to use different database backends")
|
||||
|
||||
# Always run SQLite demo (no external dependencies)
|
||||
demo_sqlite()
|
||||
|
||||
# Check for command line arguments to run specific demos
|
||||
if len(sys.argv) > 1:
|
||||
db_type = sys.argv[1].lower()
|
||||
|
||||
if db_type == "influxdb":
|
||||
demo_influxdb()
|
||||
elif db_type == "postgresql":
|
||||
demo_postgresql()
|
||||
elif db_type == "mysql":
|
||||
demo_mysql()
|
||||
elif db_type == "victoriametrics":
|
||||
demo_victoriametrics()
|
||||
elif db_type == "all":
|
||||
demo_influxdb()
|
||||
demo_postgresql()
|
||||
demo_mysql()
|
||||
demo_victoriametrics()
|
||||
else:
|
||||
print(f"\nUnknown database type: {db_type}")
|
||||
print(
|
||||
"Available options: influxdb, postgresql, mysql, victoriametrics, all"
|
||||
)
|
||||
else:
|
||||
print("\n💡 To test other databases, run:")
|
||||
print(" python demo_databases.py influxdb")
|
||||
print(" python demo_databases.py postgresql")
|
||||
print(" python demo_databases.py mysql")
|
||||
print(" python demo_databases.py victoriametrics")
|
||||
print(" python demo_databases.py all")
|
||||
|
||||
# Show recommendations
|
||||
show_recommendations()
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("✅ Demo completed!")
|
||||
print("📖 See DATABASE_DEPLOYMENT_GUIDE.md for production setup instructions")
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,46 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Custom exceptions for water monitoring system
|
||||
"""
|
||||
|
||||
|
||||
class WaterMonitorException(Exception):
|
||||
"""Base exception for water monitoring system"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class DatabaseConnectionError(WaterMonitorException):
|
||||
"""Raised when database connection fails"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class APIConnectionError(WaterMonitorException):
|
||||
"""Raised when API connection fails"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class DataValidationError(WaterMonitorException):
|
||||
"""Raised when data validation fails"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class ConfigurationError(WaterMonitorException):
|
||||
"""Raised when configuration is invalid"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class DataParsingError(WaterMonitorException):
|
||||
"""Raised when data parsing fails"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class RetryExhaustedError(WaterMonitorException):
|
||||
"""Raised when all retry attempts are exhausted"""
|
||||
|
||||
pass
|
||||
@@ -1,172 +0,0 @@
|
||||
"""Persistence for issued flood forecasts.
|
||||
|
||||
Every background precompute stores what the deployed model predicted at that
|
||||
moment — predicted 24/12/6 h peak, warning/danger probabilities, model
|
||||
version. Keyed by (as_of, station, horizon), so hourly data yields one row
|
||||
per station-horizon per hour regardless of how often the precompute runs.
|
||||
This is the operational record that lets "predicted vs actual" be graphed
|
||||
later without retraining historical models.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import logging
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ForecastHistoryStore:
|
||||
"""SQL store (sqlite / postgresql / mysql), same pattern as HiiStore."""
|
||||
|
||||
def __init__(self, connection_string: str, db_type: str):
|
||||
self.db_type = db_type.lower()
|
||||
if self.db_type not in ("sqlite", "postgresql", "mysql"):
|
||||
raise ValueError(
|
||||
f"Forecast history requires a SQL database, got '{db_type}'"
|
||||
)
|
||||
self.connection_string = connection_string
|
||||
self.engine = None
|
||||
|
||||
def connect(self) -> bool:
|
||||
try:
|
||||
from sqlalchemy import create_engine, text
|
||||
|
||||
self.engine = create_engine(self.connection_string, pool_pre_ping=True)
|
||||
ddl = """
|
||||
CREATE TABLE IF NOT EXISTS forecast_history (
|
||||
as_of TIMESTAMP NOT NULL,
|
||||
station_code VARCHAR(10) NOT NULL,
|
||||
horizon_hours INTEGER NOT NULL,
|
||||
predicted_max_level NUMERIC(8,3),
|
||||
p_warning NUMERIC(7,5),
|
||||
p_danger NUMERIC(7,5),
|
||||
current_level NUMERIC(8,3),
|
||||
model_version VARCHAR(64),
|
||||
source VARCHAR(16),
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (as_of, station_code, horizon_hours)
|
||||
)
|
||||
"""
|
||||
index = (
|
||||
"CREATE INDEX IF NOT EXISTS idx_forecast_history_station "
|
||||
"ON forecast_history(station_code, as_of)"
|
||||
)
|
||||
with self.engine.begin() as conn:
|
||||
conn.execute(text(ddl))
|
||||
if self.db_type != "mysql": # MySQL lacks IF NOT EXISTS for indexes
|
||||
conn.execute(text(index))
|
||||
return True
|
||||
except Exception as error:
|
||||
logger.error(f"ForecastHistoryStore failed to connect: {error}")
|
||||
self.engine = None
|
||||
return False
|
||||
|
||||
def save_rows(self, rows: List[Dict]) -> int:
|
||||
"""Upsert forecast rows as returned by ml.predict (idempotent)."""
|
||||
if not rows:
|
||||
return 0
|
||||
if not self.engine and not self.connect():
|
||||
return 0
|
||||
from sqlalchemy import text
|
||||
|
||||
cols = (
|
||||
"(as_of, station_code, horizon_hours, predicted_max_level, "
|
||||
"p_warning, p_danger, current_level, model_version, source)"
|
||||
)
|
||||
values = (
|
||||
"(:as_of, :station_code, :horizon_hours, :predicted_max_level, "
|
||||
":p_warning, :p_danger, :current_level, :model_version, :source)"
|
||||
)
|
||||
update_cols = (
|
||||
"predicted_max_level",
|
||||
"p_warning",
|
||||
"p_danger",
|
||||
"current_level",
|
||||
"model_version",
|
||||
"source",
|
||||
)
|
||||
if self.db_type == "mysql":
|
||||
updates = ", ".join(f"{c} = VALUES({c})" for c in update_cols)
|
||||
sql = (
|
||||
f"INSERT INTO forecast_history {cols} VALUES {values} "
|
||||
f"ON DUPLICATE KEY UPDATE {updates}"
|
||||
)
|
||||
else:
|
||||
updates = ", ".join(f"{c} = EXCLUDED.{c}" for c in update_cols)
|
||||
sql = (
|
||||
f"INSERT INTO forecast_history {cols} VALUES {values} "
|
||||
f"ON CONFLICT (as_of, station_code, horizon_hours) "
|
||||
f"DO UPDATE SET {updates}"
|
||||
)
|
||||
|
||||
params = []
|
||||
for row in rows:
|
||||
as_of = row.get("as_of")
|
||||
if isinstance(as_of, str):
|
||||
as_of = datetime.datetime.fromisoformat(as_of)
|
||||
if as_of is None or row.get("station_code") is None:
|
||||
continue
|
||||
params.append(
|
||||
{
|
||||
"as_of": as_of,
|
||||
"station_code": row["station_code"],
|
||||
"horizon_hours": row.get("horizon_hours"),
|
||||
"predicted_max_level": row.get("predicted_max_level"),
|
||||
"p_warning": row.get("p_warning"),
|
||||
"p_danger": row.get("p_danger"),
|
||||
"current_level": row.get("current_level"),
|
||||
"model_version": row.get("model_version"),
|
||||
"source": row.get("source"),
|
||||
}
|
||||
)
|
||||
if not params:
|
||||
return 0
|
||||
try:
|
||||
with self.engine.begin() as conn:
|
||||
conn.execute(text(sql), params)
|
||||
return len(params)
|
||||
except Exception as error:
|
||||
logger.error(f"ForecastHistoryStore save failed: {error}")
|
||||
return 0
|
||||
|
||||
def fetch(
|
||||
self,
|
||||
station_code: str,
|
||||
start: Optional[datetime.datetime] = None,
|
||||
end: Optional[datetime.datetime] = None,
|
||||
horizon_hours: Optional[int] = None,
|
||||
limit: int = 5000,
|
||||
) -> List[Dict]:
|
||||
"""Issued forecasts for one station, ascending by as_of."""
|
||||
if not self.engine and not self.connect():
|
||||
return []
|
||||
from sqlalchemy import text
|
||||
|
||||
clauses = ["station_code = :code"]
|
||||
params: Dict = {"code": station_code, "limit": limit}
|
||||
if start is not None:
|
||||
clauses.append("as_of >= :start")
|
||||
params["start"] = start
|
||||
if end is not None:
|
||||
clauses.append("as_of <= :end")
|
||||
params["end"] = end
|
||||
if horizon_hours is not None:
|
||||
clauses.append("horizon_hours = :horizon")
|
||||
params["horizon"] = horizon_hours
|
||||
sql = (
|
||||
"SELECT as_of, station_code, horizon_hours, predicted_max_level, "
|
||||
"p_warning, p_danger, current_level, model_version, source "
|
||||
f"FROM forecast_history WHERE {' AND '.join(clauses)} "
|
||||
"ORDER BY as_of ASC, horizon_hours ASC LIMIT :limit"
|
||||
)
|
||||
try:
|
||||
with self.engine.connect() as conn:
|
||||
rows = [dict(r._mapping) for r in conn.execute(text(sql), params)]
|
||||
for row in rows:
|
||||
for key, value in row.items():
|
||||
if hasattr(value, "is_finite"): # Decimal -> float
|
||||
row[key] = float(value)
|
||||
return rows
|
||||
except Exception as error:
|
||||
logger.error(f"ForecastHistoryStore fetch failed: {error}")
|
||||
return []
|
||||
@@ -1,290 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Health check system for water monitoring application
|
||||
"""
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta
|
||||
from enum import Enum
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HealthStatus(Enum):
|
||||
HEALTHY = "healthy"
|
||||
DEGRADED = "degraded"
|
||||
UNHEALTHY = "unhealthy"
|
||||
|
||||
|
||||
@dataclass
|
||||
class HealthCheckResult:
|
||||
"""Result of a health check"""
|
||||
|
||||
name: str
|
||||
status: HealthStatus
|
||||
message: str
|
||||
timestamp: datetime
|
||||
response_time_ms: Optional[float] = None
|
||||
details: Optional[Dict[str, Any]] = None
|
||||
|
||||
|
||||
class HealthCheck:
|
||||
"""Base health check class"""
|
||||
|
||||
def __init__(self, name: str, timeout_seconds: int = 30):
|
||||
self.name = name
|
||||
self.timeout_seconds = timeout_seconds
|
||||
|
||||
def check(self) -> HealthCheckResult:
|
||||
"""Perform the health check"""
|
||||
start_time = time.time()
|
||||
|
||||
try:
|
||||
result = self._perform_check()
|
||||
response_time = (time.time() - start_time) * 1000
|
||||
|
||||
return HealthCheckResult(
|
||||
name=self.name,
|
||||
status=result.get("status", HealthStatus.HEALTHY),
|
||||
message=result.get("message", "OK"),
|
||||
timestamp=datetime.now(),
|
||||
response_time_ms=response_time,
|
||||
details=result.get("details"),
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
response_time = (time.time() - start_time) * 1000
|
||||
logger.error(f"Health check {self.name} failed: {e}")
|
||||
|
||||
return HealthCheckResult(
|
||||
name=self.name,
|
||||
status=HealthStatus.UNHEALTHY,
|
||||
message=f"Check failed: {str(e)}",
|
||||
timestamp=datetime.now(),
|
||||
response_time_ms=response_time,
|
||||
)
|
||||
|
||||
def _perform_check(self) -> Dict[str, Any]:
|
||||
"""Override this method to implement the actual check"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class DatabaseHealthCheck(HealthCheck):
|
||||
"""Health check for database connectivity"""
|
||||
|
||||
def __init__(self, db_adapter, name: str = "database"):
|
||||
super().__init__(name)
|
||||
self.db_adapter = db_adapter
|
||||
|
||||
def _perform_check(self) -> Dict[str, Any]:
|
||||
if not self.db_adapter:
|
||||
return {
|
||||
"status": HealthStatus.UNHEALTHY,
|
||||
"message": "Database adapter not initialized",
|
||||
}
|
||||
|
||||
try:
|
||||
# Connect only when there is no live engine yet: connect() re-runs
|
||||
# the CREATE TABLE DDL suite, which is far too heavy per probe.
|
||||
if getattr(self.db_adapter, "engine", None) is None and hasattr(
|
||||
self.db_adapter, "connect"
|
||||
):
|
||||
connected = self.db_adapter.connect()
|
||||
if not connected:
|
||||
return {
|
||||
"status": HealthStatus.UNHEALTHY,
|
||||
"message": "Database connection failed",
|
||||
}
|
||||
|
||||
# Try to get latest data
|
||||
latest_data = self.db_adapter.get_latest_measurements(limit=1)
|
||||
|
||||
if latest_data:
|
||||
latest_timestamp = latest_data[0].get("timestamp")
|
||||
if isinstance(latest_timestamp, str):
|
||||
latest_timestamp = datetime.fromisoformat(
|
||||
latest_timestamp.replace("Z", "+00:00")
|
||||
)
|
||||
|
||||
# Check if data is recent (within last 2 hours)
|
||||
if datetime.now() - latest_timestamp.replace(tzinfo=None) > timedelta(
|
||||
hours=2
|
||||
):
|
||||
return {
|
||||
"status": HealthStatus.DEGRADED,
|
||||
"message": f"Latest data is old: {latest_timestamp}",
|
||||
"details": {"latest_data_timestamp": str(latest_timestamp)},
|
||||
}
|
||||
|
||||
return {
|
||||
"status": HealthStatus.HEALTHY,
|
||||
"message": "Database connection OK",
|
||||
"details": {
|
||||
"latest_data_count": len(latest_data),
|
||||
"latest_timestamp": (
|
||||
str(latest_data[0].get("timestamp")) if latest_data else None
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
return {
|
||||
"status": HealthStatus.UNHEALTHY,
|
||||
"message": f"Database check failed: {str(e)}",
|
||||
}
|
||||
|
||||
|
||||
class APIHealthCheck(HealthCheck):
|
||||
"""Health check for external API connectivity"""
|
||||
|
||||
def __init__(self, api_url: str, session, name: str = "api"):
|
||||
super().__init__(name)
|
||||
# A liveness probe should fail fast: the default 30s timeout meant a
|
||||
# slow upstream pinned executor threads for longer than the /health
|
||||
# cache TTL, so the pool never drained under load.
|
||||
self.timeout_seconds = 5
|
||||
self.api_url = api_url
|
||||
self.session = session
|
||||
|
||||
def _perform_check(self) -> Dict[str, Any]:
|
||||
try:
|
||||
# Simple GET request to check API availability
|
||||
response = self.session.get(self.api_url, timeout=self.timeout_seconds)
|
||||
|
||||
if response.status_code == 200:
|
||||
return {
|
||||
"status": HealthStatus.HEALTHY,
|
||||
"message": "API connection OK",
|
||||
"details": {
|
||||
"status_code": response.status_code,
|
||||
"response_size": len(response.content),
|
||||
},
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"status": HealthStatus.DEGRADED,
|
||||
"message": f"API returned status {response.status_code}",
|
||||
"details": {"status_code": response.status_code},
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
return {
|
||||
"status": HealthStatus.UNHEALTHY,
|
||||
"message": f"API check failed: {str(e)}",
|
||||
}
|
||||
|
||||
|
||||
class MemoryHealthCheck(HealthCheck):
|
||||
"""Health check for memory usage"""
|
||||
|
||||
def __init__(self, max_memory_mb: int = 1000, name: str = "memory"):
|
||||
super().__init__(name)
|
||||
self.max_memory_mb = max_memory_mb
|
||||
|
||||
def _perform_check(self) -> Dict[str, Any]:
|
||||
try:
|
||||
import psutil
|
||||
|
||||
process = psutil.Process()
|
||||
memory_info = process.memory_info()
|
||||
memory_mb = memory_info.rss / 1024 / 1024
|
||||
|
||||
if memory_mb > self.max_memory_mb:
|
||||
return {
|
||||
"status": HealthStatus.DEGRADED,
|
||||
"message": f"High memory usage: {memory_mb:.1f}MB",
|
||||
"details": {
|
||||
"memory_mb": memory_mb,
|
||||
"max_memory_mb": self.max_memory_mb,
|
||||
},
|
||||
}
|
||||
|
||||
return {
|
||||
"status": HealthStatus.HEALTHY,
|
||||
"message": f"Memory usage OK: {memory_mb:.1f}MB",
|
||||
"details": {"memory_mb": memory_mb},
|
||||
}
|
||||
|
||||
except ImportError:
|
||||
return {
|
||||
"status": HealthStatus.HEALTHY,
|
||||
"message": "Memory check skipped (psutil not available)",
|
||||
}
|
||||
except Exception as e:
|
||||
return {
|
||||
"status": HealthStatus.UNHEALTHY,
|
||||
"message": f"Memory check failed: {str(e)}",
|
||||
}
|
||||
|
||||
|
||||
class HealthCheckManager:
|
||||
"""Manages multiple health checks"""
|
||||
|
||||
def __init__(self):
|
||||
self.checks: List[HealthCheck] = []
|
||||
self.last_results: Dict[str, HealthCheckResult] = {}
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def add_check(self, health_check: HealthCheck):
|
||||
"""Add a health check"""
|
||||
with self._lock:
|
||||
self.checks.append(health_check)
|
||||
|
||||
def run_all_checks(self) -> Dict[str, HealthCheckResult]:
|
||||
"""Run all health checks"""
|
||||
results = {}
|
||||
|
||||
for check in self.checks:
|
||||
try:
|
||||
result = check.check()
|
||||
results[check.name] = result
|
||||
|
||||
with self._lock:
|
||||
self.last_results[check.name] = result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error running health check {check.name}: {e}")
|
||||
results[check.name] = HealthCheckResult(
|
||||
name=check.name,
|
||||
status=HealthStatus.UNHEALTHY,
|
||||
message=f"Check execution failed: {str(e)}",
|
||||
timestamp=datetime.now(),
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
def get_overall_status(self) -> HealthStatus:
|
||||
"""Get overall system health status"""
|
||||
if not self.last_results:
|
||||
return HealthStatus.UNHEALTHY
|
||||
|
||||
statuses = [result.status for result in self.last_results.values()]
|
||||
|
||||
if any(status == HealthStatus.UNHEALTHY for status in statuses):
|
||||
return HealthStatus.UNHEALTHY
|
||||
elif any(status == HealthStatus.DEGRADED for status in statuses):
|
||||
return HealthStatus.DEGRADED
|
||||
else:
|
||||
return HealthStatus.HEALTHY
|
||||
|
||||
def get_health_summary(self) -> Dict[str, Any]:
|
||||
"""Get a summary of system health"""
|
||||
overall_status = self.get_overall_status()
|
||||
|
||||
return {
|
||||
"overall_status": overall_status.value,
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"checks": {
|
||||
name: {
|
||||
"status": result.status.value,
|
||||
"message": result.message,
|
||||
"response_time_ms": result.response_time_ms,
|
||||
"timestamp": result.timestamp.isoformat(),
|
||||
}
|
||||
for name, result in self.last_results.items()
|
||||
},
|
||||
}
|
||||
+5
-5
@@ -198,13 +198,13 @@ def main(argv: Optional[List[str]] = None) -> bool:
|
||||
level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
|
||||
)
|
||||
|
||||
from .config import Config
|
||||
from .config import database_url
|
||||
|
||||
db_config = Config.get_database_config()
|
||||
if db_config["type"] not in ("sqlite", "postgresql", "mysql"):
|
||||
logger.error(f"Backfill requires a SQL DB_TYPE, got '{db_config['type']}'")
|
||||
url = database_url()
|
||||
if not url:
|
||||
logger.error("no database: set POSTGRES_CONNECTION_STRING or POSTGRES_*")
|
||||
return False
|
||||
store = HiiStore(db_config["connection_string"], db_config["type"])
|
||||
store = HiiStore(url)
|
||||
if not store.connect():
|
||||
return False
|
||||
|
||||
|
||||
+19
-95
@@ -1,7 +1,10 @@
|
||||
"""Collector for HII/ThaiWater open api-v3 feeds (rainfall + water level).
|
||||
"""HII/ThaiWater open api-v3 client and PostgreSQL store (rainfall + water level).
|
||||
|
||||
Polls the unauthenticated api-v3.thaiwater.net public endpoints, filters to the
|
||||
Ping basin, and persists to dedicated tables alongside the RID data:
|
||||
The hourly collection runs in ping-collector (rust/collector/src/hii.rs, same
|
||||
parsing and tables); this module is used by the history backfill
|
||||
(src/hii_backfill.py, scripts/backfill_hii_waterlevel.py) and the tests.
|
||||
Endpoints: the unauthenticated api-v3.thaiwater.net public API, filtered to the
|
||||
Ping basin. Tables:
|
||||
|
||||
- hii_rain_stations / hii_rainfall (rain_1h / rain_24h gauge telemetry)
|
||||
- hii_wl_stations / hii_waterlevel (independent water-level source, m MSL)
|
||||
@@ -175,16 +178,10 @@ class HiiClient:
|
||||
|
||||
|
||||
class HiiStore:
|
||||
"""SQL persistence for HII feeds (sqlite / postgresql / mysql).
|
||||
"""PostgreSQL persistence for the HII feeds: the hii_* tables next to the
|
||||
RID data in the same database (the DDL matches rust/collector/src/db.rs)."""
|
||||
|
||||
Reuses the app's main relational database (same connection string as
|
||||
the RID tables) but writes to its own hii_* tables.
|
||||
"""
|
||||
|
||||
def __init__(self, connection_string: str, db_type: str):
|
||||
self.db_type = db_type.lower()
|
||||
if self.db_type not in ("sqlite", "postgresql", "mysql"):
|
||||
raise ValueError(f"HII collection requires a SQL database, got '{db_type}'")
|
||||
def __init__(self, connection_string: str):
|
||||
self.connection_string = connection_string
|
||||
self.engine = None
|
||||
|
||||
@@ -203,7 +200,6 @@ class HiiStore:
|
||||
def _create_tables(self):
|
||||
from sqlalchemy import text
|
||||
|
||||
bool_type = "BOOLEAN" if self.db_type != "mysql" else "TINYINT(1)"
|
||||
ddl = [
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS hii_rain_stations (
|
||||
@@ -230,7 +226,7 @@ class HiiStore:
|
||||
PRIMARY KEY (station_id, timestamp)
|
||||
)
|
||||
""",
|
||||
f"""
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS hii_wl_stations (
|
||||
id INTEGER PRIMARY KEY,
|
||||
oldcode VARCHAR(60),
|
||||
@@ -247,7 +243,7 @@ class HiiStore:
|
||||
critical_level_msl NUMERIC(8,3),
|
||||
critical_level_m NUMERIC(8,3),
|
||||
qmax NUMERIC(10,2),
|
||||
is_key_station {bool_type},
|
||||
is_key_station BOOLEAN,
|
||||
updated_at TIMESTAMP
|
||||
)
|
||||
""",
|
||||
@@ -269,10 +265,6 @@ class HiiStore:
|
||||
"CREATE INDEX IF NOT EXISTS idx_hii_rainfall_ts ON hii_rainfall(timestamp)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_hii_waterlevel_ts ON hii_waterlevel(timestamp)",
|
||||
]
|
||||
# MySQL (<8.0.13 semantics) lacks CREATE INDEX IF NOT EXISTS; the unique
|
||||
# constraints already cover the hot (station_id, timestamp) lookups there.
|
||||
if self.db_type == "mysql":
|
||||
ddl = ddl[:4]
|
||||
with self.engine.begin() as conn:
|
||||
for statement in ddl:
|
||||
conn.execute(text(statement))
|
||||
@@ -281,19 +273,11 @@ class HiiStore:
|
||||
cols = key_cols + value_cols
|
||||
col_list = ", ".join(cols)
|
||||
params = ", ".join(f":{c}" for c in cols)
|
||||
if self.db_type == "sqlite":
|
||||
return f"INSERT OR REPLACE INTO {table} ({col_list}) VALUES ({params})"
|
||||
if self.db_type == "postgresql":
|
||||
updates = ", ".join(f"{c} = EXCLUDED.{c}" for c in value_cols)
|
||||
conflict = ", ".join(key_cols)
|
||||
return (
|
||||
f"INSERT INTO {table} ({col_list}) VALUES ({params}) "
|
||||
f"ON CONFLICT ({conflict}) DO UPDATE SET {updates}"
|
||||
)
|
||||
updates = ", ".join(f"{c} = VALUES({c})" for c in value_cols)
|
||||
updates = ", ".join(f"{c} = EXCLUDED.{c}" for c in value_cols)
|
||||
conflict = ", ".join(key_cols)
|
||||
return (
|
||||
f"INSERT INTO {table} ({col_list}) VALUES ({params}) "
|
||||
f"ON DUPLICATE KEY UPDATE {updates}"
|
||||
f"ON CONFLICT ({conflict}) DO UPDATE SET {updates}"
|
||||
)
|
||||
|
||||
def save_rain(self, records: List[Dict]) -> int:
|
||||
@@ -360,18 +344,11 @@ class HiiStore:
|
||||
|
||||
cols = "(station_id, timestamp, wl_msl, discharge)"
|
||||
values = "(:station_id, :timestamp, :wl_msl, :discharge)"
|
||||
if self.db_type == "mysql":
|
||||
sql = (
|
||||
f"INSERT INTO hii_waterlevel {cols} VALUES {values} "
|
||||
"ON DUPLICATE KEY UPDATE wl_msl = VALUES(wl_msl), "
|
||||
"discharge = VALUES(discharge)"
|
||||
)
|
||||
else: # sqlite (>=3.24) and postgresql share upsert syntax
|
||||
sql = (
|
||||
f"INSERT INTO hii_waterlevel {cols} VALUES {values} "
|
||||
"ON CONFLICT (station_id, timestamp) DO UPDATE SET "
|
||||
"wl_msl = EXCLUDED.wl_msl, discharge = EXCLUDED.discharge"
|
||||
)
|
||||
sql = (
|
||||
f"INSERT INTO hii_waterlevel {cols} VALUES {values} "
|
||||
"ON CONFLICT (station_id, timestamp) DO UPDATE SET "
|
||||
"wl_msl = EXCLUDED.wl_msl, discharge = EXCLUDED.discharge"
|
||||
)
|
||||
params = [{**row, "station_id": station_id} for row in rows]
|
||||
try:
|
||||
with self.engine.begin() as conn:
|
||||
@@ -423,56 +400,3 @@ class HiiStore:
|
||||
except Exception as e:
|
||||
logger.error(f"HiiStore save to {measurement_table} failed: {e}")
|
||||
return 0
|
||||
|
||||
|
||||
class HiiCollector:
|
||||
"""Fetch + persist one snapshot of both HII feeds."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
db_config: Dict,
|
||||
basin_code: int = PING_BASIN_CODE,
|
||||
client: Optional[HiiClient] = None,
|
||||
):
|
||||
self.client = client or HiiClient()
|
||||
self.basin_code = basin_code
|
||||
self.store = HiiStore(
|
||||
connection_string=db_config["connection_string"],
|
||||
db_type=db_config["type"],
|
||||
)
|
||||
|
||||
def run_cycle(self) -> Dict[str, int]:
|
||||
"""Collect both feeds; each is independent and failure-isolated."""
|
||||
counts = {"rainfall": 0, "waterlevel": 0}
|
||||
try:
|
||||
counts["rainfall"] = self.store.save_rain(
|
||||
self.client.fetch_rain(self.basin_code)
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"HII rainfall collection failed: {e}")
|
||||
try:
|
||||
counts["waterlevel"] = self.store.save_waterlevel(
|
||||
self.client.fetch_waterlevel(self.basin_code)
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"HII waterlevel collection failed: {e}")
|
||||
logger.info(
|
||||
f"HII collection: {counts['rainfall']} rainfall, "
|
||||
f"{counts['waterlevel']} waterlevel rows saved"
|
||||
)
|
||||
return counts
|
||||
|
||||
|
||||
def create_collector_from_config() -> Optional[HiiCollector]:
|
||||
"""Build a collector from app Config; None when disabled or non-SQL DB."""
|
||||
from .config import Config
|
||||
|
||||
if not Config.ENABLE_HII_COLLECTION:
|
||||
return None
|
||||
db_config = Config.get_database_config()
|
||||
if db_config["type"] not in ("sqlite", "postgresql", "mysql"):
|
||||
logger.warning(
|
||||
f"HII collection skipped: DB_TYPE '{db_config['type']}' is not SQL"
|
||||
)
|
||||
return None
|
||||
return HiiCollector(db_config, basin_code=Config.HII_BASIN_CODE)
|
||||
@@ -1,135 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Centralized logging configuration for water monitoring system
|
||||
"""
|
||||
|
||||
import logging
|
||||
import logging.handlers
|
||||
import os
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class ColoredFormatter(logging.Formatter):
|
||||
"""Colored console formatter"""
|
||||
|
||||
COLORS = {
|
||||
"DEBUG": "\033[36m", # Cyan
|
||||
"INFO": "\033[32m", # Green
|
||||
"WARNING": "\033[33m", # Yellow
|
||||
"ERROR": "\033[31m", # Red
|
||||
"CRITICAL": "\033[35m", # Magenta
|
||||
"RESET": "\033[0m", # Reset
|
||||
}
|
||||
|
||||
def format(self, record):
|
||||
if hasattr(record, "levelname"):
|
||||
color = self.COLORS.get(record.levelname, self.COLORS["RESET"])
|
||||
record.levelname = f"{color}{record.levelname}{self.COLORS['RESET']}"
|
||||
return super().format(record)
|
||||
|
||||
|
||||
def setup_logging(
|
||||
log_level: str = "INFO",
|
||||
log_file: Optional[str] = None,
|
||||
max_file_size: int = 10 * 1024 * 1024, # 10MB
|
||||
backup_count: int = 5,
|
||||
enable_console: bool = True,
|
||||
enable_colors: bool = True,
|
||||
) -> logging.Logger:
|
||||
"""
|
||||
Setup comprehensive logging configuration
|
||||
|
||||
Args:
|
||||
log_level: Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
|
||||
log_file: Path to log file (optional)
|
||||
max_file_size: Maximum size of log file before rotation
|
||||
backup_count: Number of backup files to keep
|
||||
enable_console: Whether to enable console logging
|
||||
enable_colors: Whether to enable colored console output
|
||||
|
||||
Returns:
|
||||
Configured logger instance
|
||||
"""
|
||||
|
||||
# Create logs directory if it doesn't exist
|
||||
if log_file:
|
||||
log_dir = os.path.dirname(log_file)
|
||||
if log_dir and not os.path.exists(log_dir):
|
||||
os.makedirs(log_dir)
|
||||
|
||||
# Configure root logger
|
||||
logger = logging.getLogger()
|
||||
logger.setLevel(getattr(logging, log_level.upper()))
|
||||
|
||||
# Clear existing handlers
|
||||
logger.handlers.clear()
|
||||
|
||||
# Create formatters
|
||||
detailed_formatter = logging.Formatter(
|
||||
"%(asctime)s - %(name)s - %(levelname)s - %(filename)s:%(lineno)d - %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
|
||||
simple_formatter = logging.Formatter(
|
||||
"%(asctime)s - %(levelname)s - %(message)s", datefmt="%H:%M:%S"
|
||||
)
|
||||
|
||||
# Console handler
|
||||
if enable_console:
|
||||
console_handler = logging.StreamHandler()
|
||||
if enable_colors and os.name != "nt": # Don't use colors on Windows
|
||||
console_formatter = ColoredFormatter(
|
||||
"%(asctime)s - %(levelname)s - %(message)s", datefmt="%H:%M:%S"
|
||||
)
|
||||
else:
|
||||
console_formatter = simple_formatter
|
||||
|
||||
console_handler.setFormatter(console_formatter)
|
||||
console_handler.setLevel(getattr(logging, log_level.upper()))
|
||||
logger.addHandler(console_handler)
|
||||
|
||||
# File handler with rotation
|
||||
if log_file:
|
||||
file_handler = logging.handlers.RotatingFileHandler(
|
||||
log_file, maxBytes=max_file_size, backupCount=backup_count, encoding="utf-8"
|
||||
)
|
||||
file_handler.setFormatter(detailed_formatter)
|
||||
file_handler.setLevel(logging.DEBUG) # Always log everything to file
|
||||
logger.addHandler(file_handler)
|
||||
|
||||
# Add performance logger for metrics
|
||||
perf_logger = logging.getLogger("performance")
|
||||
if log_file:
|
||||
perf_file = log_file.replace(".log", "_performance.log")
|
||||
perf_handler = logging.handlers.RotatingFileHandler(
|
||||
perf_file,
|
||||
maxBytes=max_file_size,
|
||||
backupCount=backup_count,
|
||||
encoding="utf-8",
|
||||
)
|
||||
perf_formatter = logging.Formatter(
|
||||
"%(asctime)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S"
|
||||
)
|
||||
perf_handler.setFormatter(perf_formatter)
|
||||
perf_logger.addHandler(perf_handler)
|
||||
perf_logger.setLevel(logging.INFO)
|
||||
perf_logger.propagate = False
|
||||
|
||||
return logger
|
||||
|
||||
|
||||
def log_performance_metric(
|
||||
operation: str, duration: float, additional_info: Optional[str] = None
|
||||
):
|
||||
"""Log performance metrics"""
|
||||
perf_logger = logging.getLogger("performance")
|
||||
message = f"PERF: {operation} took {duration:.3f}s"
|
||||
if additional_info:
|
||||
message += f" - {additional_info}"
|
||||
perf_logger.info(message)
|
||||
|
||||
|
||||
def get_logger(name: str) -> logging.Logger:
|
||||
"""Get a logger with the specified name"""
|
||||
return logging.getLogger(name)
|
||||
-654
@@ -1,654 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Main entry point for the Thailand Water Monitor system
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import signal
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
from .config import Config
|
||||
from .exceptions import ConfigurationError, DatabaseConnectionError
|
||||
from .logging_config import get_logger, setup_logging
|
||||
from .metrics import get_metrics_collector
|
||||
from .water_scraper_v3 import EnhancedWaterMonitorScraper
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
def setup_signal_handlers(scraper: Optional[EnhancedWaterMonitorScraper] = None):
|
||||
"""Setup signal handlers for graceful shutdown"""
|
||||
|
||||
def signal_handler(signum, frame):
|
||||
logger.info(f"Received signal {signum}, shutting down gracefully...")
|
||||
if scraper:
|
||||
logger.info("Stopping scraper...")
|
||||
sys.exit(0)
|
||||
|
||||
signal.signal(signal.SIGINT, signal_handler)
|
||||
signal.signal(signal.SIGTERM, signal_handler)
|
||||
|
||||
|
||||
def run_test_cycle():
|
||||
"""Run a single test cycle"""
|
||||
logger.info("Running test cycle...")
|
||||
|
||||
try:
|
||||
# Validate configuration
|
||||
Config.validate_config()
|
||||
|
||||
# Initialize scraper
|
||||
db_config = Config.get_database_config()
|
||||
scraper = EnhancedWaterMonitorScraper(db_config)
|
||||
|
||||
# Run single scraping cycle
|
||||
result = scraper.run_scraping_cycle()
|
||||
|
||||
if result:
|
||||
logger.info("✅ Test cycle completed successfully")
|
||||
|
||||
# Show some statistics
|
||||
latest_data = scraper.get_latest_data(5)
|
||||
if latest_data:
|
||||
logger.info(f"Latest data points: {len(latest_data)}")
|
||||
for data in latest_data[:3]: # Show first 3
|
||||
logger.info(
|
||||
f" • {data['station_code']}: {data['water_level']:.2f}m, {data['discharge']:.1f} cms"
|
||||
)
|
||||
else:
|
||||
logger.warning("⚠️ Test cycle completed but no new data was found")
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Test cycle failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def run_continuous_monitoring():
|
||||
"""Run continuous monitoring with adaptive scheduling and alerting"""
|
||||
logger.info("Starting continuous monitoring...")
|
||||
|
||||
try:
|
||||
# Validate configuration
|
||||
Config.validate_config()
|
||||
|
||||
# Initialize scraper
|
||||
db_config = Config.get_database_config()
|
||||
scraper = EnhancedWaterMonitorScraper(db_config)
|
||||
|
||||
# Initialize alerting system
|
||||
from .alerting import WaterLevelAlertSystem
|
||||
|
||||
alerting = WaterLevelAlertSystem()
|
||||
|
||||
# Initialize HII/ThaiWater collector (rainfall + backup water level)
|
||||
hii_collector = None
|
||||
try:
|
||||
from .hii_collector import create_collector_from_config
|
||||
|
||||
hii_collector = create_collector_from_config()
|
||||
if hii_collector:
|
||||
logger.info(
|
||||
"HII collection enabled (Ping-basin rainfall + water level)"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"HII collector initialization failed: {e}")
|
||||
|
||||
# Setup signal handlers
|
||||
setup_signal_handlers(scraper)
|
||||
|
||||
logger.info(
|
||||
f"Monitoring started with {Config.SCRAPING_INTERVAL_HOURS}h interval"
|
||||
)
|
||||
logger.info(
|
||||
"Adaptive retry: switches to 1-minute intervals when no data available"
|
||||
)
|
||||
logger.info("Alerts: automatic check after each successful data fetch")
|
||||
logger.info("Press Ctrl+C to stop")
|
||||
|
||||
# Run initial cycle
|
||||
logger.info("Running initial data collection...")
|
||||
initial_success = scraper.run_scraping_cycle()
|
||||
|
||||
# Adaptive scheduling state
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
retry_mode = not initial_success
|
||||
last_successful_fetch = None if not initial_success else datetime.now()
|
||||
|
||||
last_hii_run = None
|
||||
if hii_collector:
|
||||
try:
|
||||
hii_collector.run_cycle()
|
||||
last_hii_run = datetime.now()
|
||||
except Exception as e:
|
||||
logger.error(f"HII collection failed: {e}")
|
||||
|
||||
if retry_mode:
|
||||
logger.warning("No data fetched in initial run - entering retry mode")
|
||||
next_run = datetime.now() + timedelta(minutes=1)
|
||||
else:
|
||||
logger.info("Initial data fetch successful - using hourly schedule")
|
||||
next_run = (datetime.now() + timedelta(hours=1)).replace(
|
||||
minute=0, second=0, microsecond=0
|
||||
)
|
||||
|
||||
logger.info(f"Next run at {next_run.strftime('%H:%M')}")
|
||||
|
||||
while True:
|
||||
current_time = datetime.now()
|
||||
|
||||
if current_time >= next_run:
|
||||
logger.info("Running scheduled data collection...")
|
||||
success = scraper.run_scraping_cycle()
|
||||
|
||||
# HII feeds update hourly; keep collecting on that cadence even
|
||||
# when the RID scraper is in 1-minute retry mode.
|
||||
if hii_collector and (
|
||||
last_hii_run is None
|
||||
or current_time - last_hii_run >= timedelta(minutes=55)
|
||||
):
|
||||
try:
|
||||
hii_collector.run_cycle()
|
||||
last_hii_run = current_time
|
||||
except Exception as e:
|
||||
logger.error(f"HII collection failed: {e}")
|
||||
|
||||
if success:
|
||||
last_successful_fetch = current_time
|
||||
|
||||
# Run alert check after every successful new data fetch
|
||||
logger.info("Running alert check...")
|
||||
try:
|
||||
alert_results = alerting.run_alert_check()
|
||||
if alert_results.get("total_alerts", 0) > 0:
|
||||
logger.info(
|
||||
f"Alerts: {alert_results['total_alerts']} generated, {alert_results['sent']} sent"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Alert check failed: {e}")
|
||||
|
||||
if retry_mode:
|
||||
logger.info(
|
||||
"✅ Data fetch successful - switching back to hourly schedule"
|
||||
)
|
||||
retry_mode = False
|
||||
# Schedule next run at the next full hour
|
||||
next_run = (current_time + timedelta(hours=1)).replace(
|
||||
minute=0, second=0, microsecond=0
|
||||
)
|
||||
else:
|
||||
# Continue hourly schedule
|
||||
next_run = (
|
||||
current_time
|
||||
+ timedelta(hours=Config.SCRAPING_INTERVAL_HOURS)
|
||||
).replace(minute=0, second=0, microsecond=0)
|
||||
|
||||
logger.info(f"Next scheduled run at {next_run.strftime('%H:%M')}")
|
||||
else:
|
||||
if not retry_mode:
|
||||
logger.warning(
|
||||
"⚠️ No data fetched - switching to retry mode (1-minute intervals)"
|
||||
)
|
||||
retry_mode = True
|
||||
|
||||
# Schedule retry in 1 minute
|
||||
next_run = current_time + timedelta(minutes=1)
|
||||
logger.info(f"Retrying in 1 minute at {next_run.strftime('%H:%M')}")
|
||||
|
||||
# Sleep for 10 seconds and check again
|
||||
time.sleep(10)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
logger.info("Monitoring stopped by user")
|
||||
except Exception as e:
|
||||
logger.error(f"Monitoring failed: {e}")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def run_hii_collection():
|
||||
"""Run a single HII/ThaiWater collection cycle (rainfall + water level)"""
|
||||
try:
|
||||
Config.validate_config()
|
||||
from .hii_collector import create_collector_from_config
|
||||
|
||||
collector = create_collector_from_config()
|
||||
if not collector:
|
||||
logger.error(
|
||||
"HII collection unavailable (disabled via ENABLE_HII_COLLECTION "
|
||||
"or DB_TYPE is not a SQL database)"
|
||||
)
|
||||
return False
|
||||
|
||||
counts = collector.run_cycle()
|
||||
return counts["rainfall"] > 0 or counts["waterlevel"] > 0
|
||||
except Exception as e:
|
||||
logger.error(f"HII collection failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def run_gap_filling(days_back: Optional[int]):
|
||||
"""Run gap filling for missing data (days_back=None scans the whole range)"""
|
||||
if days_back is None:
|
||||
logger.info("Checking for data gaps across the whole data range...")
|
||||
else:
|
||||
logger.info(f"Checking for data gaps in the last {days_back} days...")
|
||||
|
||||
try:
|
||||
# Validate configuration
|
||||
Config.validate_config()
|
||||
|
||||
# Initialize scraper
|
||||
db_config = Config.get_database_config()
|
||||
scraper = EnhancedWaterMonitorScraper(db_config)
|
||||
|
||||
# Fill gaps
|
||||
filled_count = scraper.fill_data_gaps(days_back)
|
||||
|
||||
if filled_count > 0:
|
||||
logger.info(f"✅ Filled {filled_count} missing data points")
|
||||
else:
|
||||
logger.info("✅ No data gaps found")
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Gap filling failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def run_data_update(days_back: int):
|
||||
"""Update existing data with latest values"""
|
||||
logger.info(f"Updating existing data for the last {days_back} days...")
|
||||
|
||||
try:
|
||||
# Validate configuration
|
||||
Config.validate_config()
|
||||
|
||||
# Initialize scraper
|
||||
db_config = Config.get_database_config()
|
||||
scraper = EnhancedWaterMonitorScraper(db_config)
|
||||
|
||||
# Update data
|
||||
updated_count = scraper.update_existing_data(days_back)
|
||||
|
||||
if updated_count > 0:
|
||||
logger.info(f"✅ Updated {updated_count} data points")
|
||||
else:
|
||||
logger.info("✅ No data updates needed")
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Data update failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def run_historical_import(
|
||||
start_date_str: str, end_date_str: str, skip_existing: bool = True
|
||||
):
|
||||
"""Import historical data for a date range"""
|
||||
try:
|
||||
# Parse dates
|
||||
start_date = datetime.strptime(start_date_str, "%Y-%m-%d")
|
||||
end_date = datetime.strptime(end_date_str, "%Y-%m-%d")
|
||||
|
||||
if start_date > end_date:
|
||||
logger.error("Start date must be before or equal to end date")
|
||||
return False
|
||||
|
||||
logger.info(
|
||||
f"Importing historical data from {start_date.date()} to {end_date.date()}"
|
||||
)
|
||||
if skip_existing:
|
||||
logger.info("Skipping dates that already have data")
|
||||
|
||||
# Validate configuration
|
||||
Config.validate_config()
|
||||
|
||||
# Initialize scraper
|
||||
db_config = Config.get_database_config()
|
||||
scraper = EnhancedWaterMonitorScraper(db_config)
|
||||
|
||||
# Import historical data
|
||||
imported_count = scraper.import_historical_data(
|
||||
start_date, end_date, skip_existing
|
||||
)
|
||||
|
||||
if imported_count > 0:
|
||||
logger.info(f"✅ Imported {imported_count} historical data points")
|
||||
else:
|
||||
logger.info("✅ No new data imported")
|
||||
|
||||
return True
|
||||
|
||||
except ValueError as e:
|
||||
logger.error(f"❌ Invalid date format. Use YYYY-MM-DD: {e}")
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Historical import failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def run_web_api():
|
||||
"""Run the FastAPI web interface"""
|
||||
logger.info("Starting web API server...")
|
||||
|
||||
try:
|
||||
import uvicorn
|
||||
|
||||
# Validate configuration
|
||||
Config.validate_config()
|
||||
|
||||
workers = max(1, Config.WEB_WORKERS)
|
||||
if workers > 1:
|
||||
# Multi-worker needs the app as an import string; a localhost lock
|
||||
# port keeps background collection in exactly one worker.
|
||||
uvicorn.run(
|
||||
"src.web_api:app",
|
||||
host="0.0.0.0",
|
||||
port=8000,
|
||||
workers=workers,
|
||||
log_config=None,
|
||||
)
|
||||
else:
|
||||
from .web_api import app
|
||||
|
||||
uvicorn.run(app, host="0.0.0.0", port=8000, log_config=None)
|
||||
|
||||
except ImportError:
|
||||
logger.error("FastAPI not installed. Run: pip install fastapi uvicorn")
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"Web API failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def run_alert_check():
|
||||
"""Run water level alert check"""
|
||||
logger.info("Running water level alert check...")
|
||||
|
||||
try:
|
||||
from .alerting import WaterLevelAlertSystem
|
||||
|
||||
# Initialize alerting system
|
||||
alerting = WaterLevelAlertSystem()
|
||||
|
||||
# Run alert check
|
||||
results = alerting.run_alert_check()
|
||||
|
||||
if "error" in results:
|
||||
logger.error("❌ Alert check failed due to database connection")
|
||||
return False
|
||||
|
||||
logger.info(f"✅ Alert check completed:")
|
||||
logger.info(f" • Water level alerts: {results['water_alerts']}")
|
||||
logger.info(f" • Data freshness alerts: {results['data_alerts']}")
|
||||
logger.info(f" • Total alerts generated: {results['total_alerts']}")
|
||||
logger.info(f" • Alerts sent: {results['sent']}")
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Alert check failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def run_alert_test():
|
||||
"""Send test alert message"""
|
||||
logger.info("Sending test alert message...")
|
||||
|
||||
try:
|
||||
from .alerting import WaterLevelAlertSystem
|
||||
|
||||
# Initialize alerting system
|
||||
alerting = WaterLevelAlertSystem()
|
||||
|
||||
if not alerting.matrix_notifier:
|
||||
logger.error("❌ Matrix notifier not configured")
|
||||
logger.info(
|
||||
"Please set MATRIX_ACCESS_TOKEN and MATRIX_ROOM_ID in your .env file"
|
||||
)
|
||||
return False
|
||||
|
||||
# Send test message
|
||||
test_message = "🧪 **Test Alert**\n\nThis is a test message from the Northern Thailand Ping River Monitor.\n\nIf you received this, Matrix notifications are working correctly!"
|
||||
success = alerting.matrix_notifier.send_message(test_message)
|
||||
|
||||
if success:
|
||||
logger.info("✅ Test alert message sent successfully")
|
||||
else:
|
||||
logger.error("❌ Test alert message failed to send")
|
||||
|
||||
return success
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Test alert failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def show_status():
|
||||
"""Show current system status"""
|
||||
logger.info("=== Northern Thailand Ping River Monitor Status ===")
|
||||
|
||||
try:
|
||||
# Show configuration
|
||||
Config.print_settings()
|
||||
|
||||
# Test database connection
|
||||
logger.info("\n=== Database Connection Test ===")
|
||||
db_config = Config.get_database_config()
|
||||
scraper = EnhancedWaterMonitorScraper(db_config)
|
||||
|
||||
if scraper.db_adapter:
|
||||
logger.info("✅ Database connection successful")
|
||||
|
||||
# Show latest data
|
||||
latest_data = scraper.get_latest_data(3)
|
||||
if latest_data:
|
||||
logger.info(f"\n=== Latest Data ({len(latest_data)} points) ===")
|
||||
for data in latest_data:
|
||||
timestamp = data["timestamp"]
|
||||
if isinstance(timestamp, str):
|
||||
timestamp = datetime.fromisoformat(
|
||||
timestamp.replace("Z", "+00:00")
|
||||
)
|
||||
logger.info(
|
||||
f" • {data['station_code']} ({timestamp}): {data['water_level']:.2f}m"
|
||||
)
|
||||
else:
|
||||
logger.info("No data found in database")
|
||||
else:
|
||||
logger.error("❌ Database connection failed")
|
||||
|
||||
# Test alerting system
|
||||
logger.info("\n=== Alerting System Status ===")
|
||||
try:
|
||||
from .alerting import WaterLevelAlertSystem
|
||||
|
||||
alerting = WaterLevelAlertSystem()
|
||||
|
||||
if alerting.matrix_notifier:
|
||||
logger.info("✅ Matrix notifications configured")
|
||||
else:
|
||||
logger.warning("⚠️ Matrix notifications not configured")
|
||||
logger.info("Set MATRIX_ACCESS_TOKEN and MATRIX_ROOM_ID in .env file")
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Alerting system error: {e}")
|
||||
|
||||
# Show metrics if available
|
||||
metrics_collector = get_metrics_collector()
|
||||
metrics = metrics_collector.get_all_metrics()
|
||||
|
||||
if any(metrics.values()):
|
||||
logger.info("\n=== Metrics Summary ===")
|
||||
for metric_type, values in metrics.items():
|
||||
if values:
|
||||
logger.info(f"{metric_type.title()}: {len(values)} metrics")
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Status check failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Northern Thailand Ping River Monitor",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
%(prog)s --test # Run single test cycle
|
||||
%(prog)s # Run continuous monitoring
|
||||
%(prog)s --web-api # Start web API server
|
||||
%(prog)s --fill-gaps 7 # Fill missing data for last 7 days
|
||||
%(prog)s --fill-gaps all # Fill missing data across the whole data range
|
||||
%(prog)s --update-data 2 # Update existing data for last 2 days
|
||||
%(prog)s --import-historical 2024-01-01 2024-01-31 # Import historical data
|
||||
%(prog)s --status # Show system status
|
||||
%(prog)s --alert-check # Check water levels and send alerts
|
||||
%(prog)s --alert-test # Send test Matrix message
|
||||
""",
|
||||
)
|
||||
|
||||
parser.add_argument("--test", action="store_true", help="Run a single test cycle")
|
||||
|
||||
parser.add_argument(
|
||||
"--web-api", action="store_true", help="Start the web API server"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--fill-gaps",
|
||||
metavar="DAYS|all",
|
||||
help=(
|
||||
"Fill missing data gaps for the specified number of days back, "
|
||||
"or 'all' to scan the entire data range in the database"
|
||||
),
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--update-data",
|
||||
type=int,
|
||||
metavar="DAYS",
|
||||
help="Update existing data for the specified number of days back",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--import-historical",
|
||||
nargs=2,
|
||||
metavar=("START_DATE", "END_DATE"),
|
||||
help="Import historical data for date range (YYYY-MM-DD format)",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--force-overwrite",
|
||||
action="store_true",
|
||||
help="Overwrite existing data when importing historical data",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--status", action="store_true", help="Show current system status"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--alert-check", action="store_true", help="Run water level alert check"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--alert-test", action="store_true", help="Send test alert message to Matrix"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--collect-hii",
|
||||
action="store_true",
|
||||
help="Run one HII/ThaiWater collection cycle (rainfall + water level)",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--log-level",
|
||||
choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
|
||||
default=Config.LOG_LEVEL,
|
||||
help="Set logging level",
|
||||
)
|
||||
|
||||
parser.add_argument("--log-file", default=Config.LOG_FILE, help="Log file path")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Setup logging
|
||||
setup_logging(
|
||||
log_level=args.log_level,
|
||||
log_file=args.log_file,
|
||||
enable_console=True,
|
||||
enable_colors=True,
|
||||
)
|
||||
|
||||
logger.info("🏔️ Northern Thailand Ping River Monitor starting...")
|
||||
logger.info(f"Version: 3.1.3")
|
||||
logger.info(f"Log level: {args.log_level}")
|
||||
|
||||
try:
|
||||
success = False
|
||||
|
||||
if args.test:
|
||||
success = run_test_cycle()
|
||||
elif args.web_api:
|
||||
success = run_web_api()
|
||||
elif args.fill_gaps is not None:
|
||||
if args.fill_gaps.lower() == "all":
|
||||
success = run_gap_filling(None)
|
||||
else:
|
||||
try:
|
||||
success = run_gap_filling(int(args.fill_gaps))
|
||||
except ValueError:
|
||||
logger.error(
|
||||
f"Invalid --fill-gaps value '{args.fill_gaps}': "
|
||||
"expected a number of days or 'all'"
|
||||
)
|
||||
sys.exit(1)
|
||||
elif args.update_data is not None:
|
||||
success = run_data_update(args.update_data)
|
||||
elif args.import_historical is not None:
|
||||
start_date, end_date = args.import_historical
|
||||
skip_existing = not args.force_overwrite
|
||||
success = run_historical_import(start_date, end_date, skip_existing)
|
||||
elif args.status:
|
||||
success = show_status()
|
||||
elif args.alert_check:
|
||||
success = run_alert_check()
|
||||
elif args.alert_test:
|
||||
success = run_alert_test()
|
||||
elif args.collect_hii:
|
||||
success = run_hii_collection()
|
||||
else:
|
||||
success = run_continuous_monitoring()
|
||||
|
||||
if success:
|
||||
logger.info("✅ Operation completed successfully")
|
||||
sys.exit(0)
|
||||
else:
|
||||
logger.error("❌ Operation failed")
|
||||
sys.exit(1)
|
||||
|
||||
except ConfigurationError as e:
|
||||
logger.error(f"Configuration error: {e}")
|
||||
sys.exit(1)
|
||||
except KeyboardInterrupt:
|
||||
logger.info("Operation cancelled by user")
|
||||
sys.exit(0)
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
-198
@@ -1,198 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Metrics collection and monitoring for water monitoring system
|
||||
"""
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from collections import defaultdict, deque
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MetricPoint:
|
||||
"""Single metric data point"""
|
||||
|
||||
timestamp: datetime
|
||||
value: float
|
||||
labels: Dict[str, str] = field(default_factory=dict)
|
||||
|
||||
|
||||
class MetricsCollector:
|
||||
"""Collects and manages application metrics"""
|
||||
|
||||
def __init__(self, retention_hours: int = 24):
|
||||
self.retention_hours = retention_hours
|
||||
self.metrics: Dict[str, deque] = defaultdict(lambda: deque(maxlen=1000))
|
||||
self.counters: Dict[str, float] = defaultdict(float)
|
||||
self.gauges: Dict[str, float] = defaultdict(float)
|
||||
self.histograms: Dict[str, List[float]] = defaultdict(list)
|
||||
self._lock = threading.Lock()
|
||||
|
||||
# Start cleanup thread
|
||||
self._cleanup_thread = threading.Thread(
|
||||
target=self._cleanup_old_metrics, daemon=True
|
||||
)
|
||||
self._cleanup_thread.start()
|
||||
|
||||
def increment_counter(
|
||||
self, name: str, value: float = 1.0, labels: Optional[Dict[str, str]] = None
|
||||
):
|
||||
"""Increment a counter metric"""
|
||||
with self._lock:
|
||||
key = self._make_key(name, labels)
|
||||
self.counters[key] += value
|
||||
self.metrics[key].append(
|
||||
MetricPoint(datetime.now(), self.counters[key], labels or {})
|
||||
)
|
||||
|
||||
def set_gauge(
|
||||
self, name: str, value: float, labels: Optional[Dict[str, str]] = None
|
||||
):
|
||||
"""Set a gauge metric"""
|
||||
with self._lock:
|
||||
key = self._make_key(name, labels)
|
||||
self.gauges[key] = value
|
||||
self.metrics[key].append(MetricPoint(datetime.now(), value, labels or {}))
|
||||
|
||||
def record_histogram(
|
||||
self, name: str, value: float, labels: Optional[Dict[str, str]] = None
|
||||
):
|
||||
"""Record a histogram value"""
|
||||
with self._lock:
|
||||
key = self._make_key(name, labels)
|
||||
self.histograms[key].append(value)
|
||||
# Keep only recent values
|
||||
if len(self.histograms[key]) > 1000:
|
||||
self.histograms[key] = self.histograms[key][-1000:]
|
||||
|
||||
self.metrics[key].append(MetricPoint(datetime.now(), value, labels or {}))
|
||||
|
||||
def get_counter(self, name: str, labels: Optional[Dict[str, str]] = None) -> float:
|
||||
"""Get current counter value"""
|
||||
key = self._make_key(name, labels)
|
||||
return self.counters.get(key, 0.0)
|
||||
|
||||
def get_gauge(self, name: str, labels: Optional[Dict[str, str]] = None) -> float:
|
||||
"""Get current gauge value"""
|
||||
key = self._make_key(name, labels)
|
||||
return self.gauges.get(key, 0.0)
|
||||
|
||||
def get_histogram_stats(
|
||||
self, name: str, labels: Optional[Dict[str, str]] = None
|
||||
) -> Dict[str, float]:
|
||||
"""Get histogram statistics"""
|
||||
key = self._make_key(name, labels)
|
||||
values = self.histograms.get(key, [])
|
||||
|
||||
if not values:
|
||||
return {"count": 0, "sum": 0, "avg": 0, "min": 0, "max": 0}
|
||||
|
||||
return {
|
||||
"count": len(values),
|
||||
"sum": sum(values),
|
||||
"avg": sum(values) / len(values),
|
||||
"min": min(values),
|
||||
"max": max(values),
|
||||
}
|
||||
|
||||
def get_all_metrics(self) -> Dict[str, Any]:
|
||||
"""Get all current metrics"""
|
||||
with self._lock:
|
||||
return {
|
||||
"counters": dict(self.counters),
|
||||
"gauges": dict(self.gauges),
|
||||
"histograms": {k: self.get_histogram_stats(k) for k in self.histograms},
|
||||
}
|
||||
|
||||
def _make_key(self, name: str, labels: Optional[Dict[str, str]]) -> str:
|
||||
"""Create a unique key for metric with labels"""
|
||||
if not labels:
|
||||
return name
|
||||
|
||||
label_str = ",".join(f"{k}={v}" for k, v in sorted(labels.items()))
|
||||
return f"{name}{{{label_str}}}"
|
||||
|
||||
def _cleanup_old_metrics(self):
|
||||
"""Clean up old metric data points"""
|
||||
while True:
|
||||
try:
|
||||
cutoff_time = datetime.now() - timedelta(hours=self.retention_hours)
|
||||
|
||||
with self._lock:
|
||||
for metric_name, points in self.metrics.items():
|
||||
# Remove old points
|
||||
while points and points[0].timestamp < cutoff_time:
|
||||
points.popleft()
|
||||
|
||||
time.sleep(3600) # Run cleanup every hour
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in metrics cleanup: {e}")
|
||||
time.sleep(60) # Wait a minute before retrying
|
||||
|
||||
|
||||
# Global metrics collector instance
|
||||
_metrics_collector = None
|
||||
|
||||
|
||||
def get_metrics_collector() -> MetricsCollector:
|
||||
"""Get the global metrics collector instance"""
|
||||
global _metrics_collector
|
||||
if _metrics_collector is None:
|
||||
_metrics_collector = MetricsCollector()
|
||||
return _metrics_collector
|
||||
|
||||
|
||||
# Convenience functions
|
||||
def increment_counter(
|
||||
name: str, value: float = 1.0, labels: Optional[Dict[str, str]] = None
|
||||
):
|
||||
"""Increment a counter metric"""
|
||||
get_metrics_collector().increment_counter(name, value, labels)
|
||||
|
||||
|
||||
def set_gauge(name: str, value: float, labels: Optional[Dict[str, str]] = None):
|
||||
"""Set a gauge metric"""
|
||||
get_metrics_collector().set_gauge(name, value, labels)
|
||||
|
||||
|
||||
def record_histogram(name: str, value: float, labels: Optional[Dict[str, str]] = None):
|
||||
"""Record a histogram value"""
|
||||
get_metrics_collector().record_histogram(name, value, labels)
|
||||
|
||||
|
||||
class Timer:
|
||||
"""Context manager for timing operations"""
|
||||
|
||||
def __init__(self, metric_name: str, labels: Optional[Dict[str, str]] = None):
|
||||
self.metric_name = metric_name
|
||||
self.labels = labels
|
||||
self.start_time = None
|
||||
|
||||
def __enter__(self):
|
||||
self.start_time = time.time()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
if self.start_time:
|
||||
duration = time.time() - self.start_time
|
||||
record_histogram(self.metric_name, duration, self.labels)
|
||||
|
||||
|
||||
def timer(metric_name: str, labels: Optional[Dict[str, str]] = None):
|
||||
"""Decorator for timing function execution"""
|
||||
|
||||
def decorator(func):
|
||||
def wrapper(*args, **kwargs):
|
||||
with Timer(metric_name, labels):
|
||||
return func(*args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
return decorator
|
||||
+3
-11
@@ -18,7 +18,7 @@ from typing import Dict, List, Optional
|
||||
import pandas as pd
|
||||
from sqlalchemy import create_engine, text
|
||||
|
||||
from ..config import Config
|
||||
from ..config import database_url
|
||||
from .features import UPSTREAM_LEADS
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -34,7 +34,7 @@ _MEASUREMENT_COLUMNS = ["timestamp", "station_code", "water_level", "discharge"]
|
||||
|
||||
def resolve_db_url(db_url: Optional[str] = None) -> Optional[str]:
|
||||
"""Resolve a Postgres connection string: explicit param > FLOOD_ML_DB_URL env >
|
||||
Config's postgresql connection string > None (caller should fall back to HTTP)."""
|
||||
POSTGRES_* settings (src/config.py) > None (caller should fall back to HTTP)."""
|
||||
if db_url:
|
||||
return db_url
|
||||
|
||||
@@ -42,15 +42,7 @@ def resolve_db_url(db_url: Optional[str] = None) -> Optional[str]:
|
||||
if env_url:
|
||||
return env_url
|
||||
|
||||
try:
|
||||
db_config = Config.get_database_config()
|
||||
except Exception as error:
|
||||
logger.warning(f"Could not resolve database config: {error}")
|
||||
return None
|
||||
|
||||
if db_config.get("type") == "postgresql":
|
||||
return db_config.get("connection_string")
|
||||
return None
|
||||
return database_url()
|
||||
|
||||
|
||||
def _default_stations() -> List[str]:
|
||||
|
||||
+54
-12
@@ -8,8 +8,8 @@ joblib/pickle. It reads:
|
||||
table). ``train_all()`` writes one next to each ``.joblib``;
|
||||
``python -m src.ml.export`` (re)exports the bundles already in ``models/``.
|
||||
* ``rust/forecast/flood_config.json`` -- thresholds, flood stages, upstream
|
||||
leads and the other constants features.py / predict.py / notify.py use,
|
||||
compiled into the Rust binary. ``python -m src.ml.export --config``
|
||||
leads and the other constants features.py / predict.py use, plus the
|
||||
notifier settings below, compiled into the Rust binary. ``python -m src.ml.export --config``
|
||||
regenerates it; tests/test_model_export.py fails while it is out of date,
|
||||
so a threshold change cannot reach Python without reaching Rust.
|
||||
|
||||
@@ -31,6 +31,49 @@ import numpy as np
|
||||
from . import data as data_mod
|
||||
from . import features, predict
|
||||
|
||||
# --- Notifier settings (rust/collector/src/notify.rs; docs/NOTIFICATIONS.md).
|
||||
# Only the Rust notifier uses these; they live here so the thresholds, stages
|
||||
# and these values reach it through one generated file.
|
||||
|
||||
# Hysteresis: an all-clear needs the level this far BELOW the threshold, so a
|
||||
# river bobbing around 3.70 m does not toggle warning/clear every hour.
|
||||
CLEAR_MARGIN_M = 0.10
|
||||
# Capacity guard. The level thresholds in features.THRESHOLDS were calibrated
|
||||
# from RID's discharge_percent (% of channel capacity); if RID re-rates a
|
||||
# gauge or moves its datum, the level crosses while capacity says the channel
|
||||
# is nearly empty (P.77, 2026-09: 3.0 m "warning" at 22 %). A crossing is
|
||||
# only announced when the reported capacity agrees that the river is high.
|
||||
# P.1 is exempt: its stages come from the municipal inundation map, not from
|
||||
# capacity. Readings without a capacity figure fall back to level only.
|
||||
CAPACITY_GUARD_MIN_PCT = 60.0
|
||||
CAPACITY_GUARD_EXEMPT = {"P.1"}
|
||||
# The P.1 outlook fires when p_warning at OUTLOOK_HORIZON hours rises through
|
||||
# OUTLOOK_ON and clears below OUTLOOK_OFF.
|
||||
OUTLOOK_ON = 0.50
|
||||
OUTLOOK_OFF = 0.25
|
||||
OUTLOOK_HORIZON = 24
|
||||
# No reading this many hours old or newer: the feed is stale.
|
||||
FEED_STALE_AFTER_H = 3.0
|
||||
|
||||
STATION_NAMES = {
|
||||
"P.1": "Nawarat Bridge, Chiang Mai city",
|
||||
"P.103": "Ring Road Bridge 3, Chiang Mai",
|
||||
"P.67": "Ban Tae (Mae Taeng)",
|
||||
"P.21": "Ban Rim Tai (Mae Rim)",
|
||||
"P.75": "Ban Chai Lat",
|
||||
"P.92": "Ban Muang Aut",
|
||||
"P.20": "Ban Chiang Dao",
|
||||
"P.4A": "Ban Mae Taeng",
|
||||
"P.5": "Tha Nang Bridge (downstream)",
|
||||
"P.81": "Ban Pong (downstream)",
|
||||
"P.82": "Ban Sob Win",
|
||||
"P.84": "Ban Panton",
|
||||
"P.87": "Ban Pa Sang",
|
||||
"P.77": "Ban Sop Mae Sapuat",
|
||||
"P.85": "Ban Lai Kaew",
|
||||
"P.76": "Ban Mae I Hai",
|
||||
}
|
||||
|
||||
FORMAT_VERSION = 1
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
CONFIG_PATH = REPO_ROOT / "rust" / "forecast" / "flood_config.json"
|
||||
@@ -123,8 +166,7 @@ def export_models_dir(models_dir: Path, force: bool = False) -> int:
|
||||
|
||||
|
||||
def flood_config() -> dict:
|
||||
"""Every constant the Rust forecaster and notifier share with Python."""
|
||||
from .. import notify
|
||||
"""Every constant the Rust forecaster and notifier take from Python."""
|
||||
from . import dam as dam_mod
|
||||
|
||||
return {
|
||||
@@ -166,14 +208,14 @@ def flood_config() -> dict:
|
||||
"serving_days": dam_mod.SERVING_DAYS,
|
||||
},
|
||||
"notify": {
|
||||
"clear_margin_m": float(notify.CLEAR_MARGIN_M),
|
||||
"capacity_guard_min_pct": float(notify.CAPACITY_GUARD_MIN_PCT),
|
||||
"capacity_guard_exempt": sorted(notify.CAPACITY_GUARD_EXEMPT),
|
||||
"outlook_on": float(notify.OUTLOOK_ON),
|
||||
"outlook_off": float(notify.OUTLOOK_OFF),
|
||||
"outlook_horizon": int(notify.OUTLOOK_HORIZON),
|
||||
"feed_stale_after_h": float(notify.FEED_STALE_AFTER_H),
|
||||
"station_names": dict(notify.STATION_NAMES),
|
||||
"clear_margin_m": float(CLEAR_MARGIN_M),
|
||||
"capacity_guard_min_pct": float(CAPACITY_GUARD_MIN_PCT),
|
||||
"capacity_guard_exempt": sorted(CAPACITY_GUARD_EXEMPT),
|
||||
"outlook_on": float(OUTLOOK_ON),
|
||||
"outlook_off": float(OUTLOOK_OFF),
|
||||
"outlook_horizon": int(OUTLOOK_HORIZON),
|
||||
"feed_stale_after_h": float(FEED_STALE_AFTER_H),
|
||||
"station_names": dict(STATION_NAMES),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
+4
-3
@@ -385,10 +385,11 @@ def get_latest_forecasts(
|
||||
now: Optional[datetime.datetime] = None,
|
||||
rain: Optional[pd.Series] = None,
|
||||
) -> List[dict]:
|
||||
"""Convenience wrapper for web_api: load the latest window from the DB/API and forecast.
|
||||
"""Load the latest window from the DB/API and forecast: the Python reference
|
||||
for the Rust forecaster (scripts/forecast_parity.py,
|
||||
rust/forecast/testdata/gen_forecast_fixtures.py).
|
||||
|
||||
Raises FileNotFoundError when no trained model bundle exists at all, so the
|
||||
API can 503 instead of serving purely heuristic output as if it were a forecast.
|
||||
Raises FileNotFoundError when no trained model bundle exists at all.
|
||||
``now`` and ``rain`` are for reproducible comparisons
|
||||
(scripts/forecast_parity.py); by default the clock and a live fetch.
|
||||
"""
|
||||
|
||||
+12
-21
@@ -158,7 +158,7 @@ def serving_series() -> Optional[pd.Series]:
|
||||
return None
|
||||
|
||||
|
||||
def backfill_db(engine, db_type: str, chunk_rows: int = 5000) -> int:
|
||||
def backfill_db(engine, chunk_rows: int = 5000) -> int:
|
||||
"""Push the full Open-Meteo history (2021+) into openmeteo_rain.
|
||||
|
||||
Loads (or fetches) the archive cache and upserts in chunks; idempotent,
|
||||
@@ -171,19 +171,19 @@ def backfill_db(engine, db_type: str, chunk_rows: int = 5000) -> int:
|
||||
total = 0
|
||||
for start in range(0, len(history), chunk_rows):
|
||||
part = history.iloc[start : start + chunk_rows]
|
||||
total += save_to_db(part, engine, db_type)
|
||||
total += save_to_db(part, engine)
|
||||
logger.info(f"openmeteo_rain backfill: {total}/{len(history)} rows")
|
||||
return total
|
||||
|
||||
|
||||
def save_to_db(df: pd.DataFrame, engine, db_type: str) -> int:
|
||||
def save_to_db(df: pd.DataFrame, engine) -> int:
|
||||
"""Upsert per-point + catchment-mean hourly rain into openmeteo_rain.
|
||||
|
||||
Called by the leader worker's hourly precompute with the live forecast
|
||||
frame, so the DB accumulates both what fell (past rows are the model
|
||||
analysis) and what was forecast (future rows, overwritten as they become
|
||||
past). The ML training path reads Open-Meteo's own archive, not this
|
||||
table — this is for dashboards, SQL analysis, and source independence.
|
||||
The live writer is ping-collector (rust/collector/src/rain.rs, hourly,
|
||||
same table and values): past rows are the model analysis, future rows
|
||||
the forecast, overwritten as they become past. This Python path is the
|
||||
archive backfill (scripts/backfill_rain_db.py). The ML training path
|
||||
reads Open-Meteo's own archive, not this table.
|
||||
"""
|
||||
if df is None or df.empty:
|
||||
return 0
|
||||
@@ -199,20 +199,11 @@ def save_to_db(df: pd.DataFrame, engine, db_type: str) -> int:
|
||||
)
|
||||
cols = ["timestamp"] + point_cols + ["catchment_mean"]
|
||||
placeholders = ", ".join(f":{c}" for c in cols)
|
||||
updates = ", ".join(
|
||||
f"{c} = " + (f"VALUES({c})" if db_type == "mysql" else f"EXCLUDED.{c}")
|
||||
for c in cols[1:]
|
||||
updates = ", ".join(f"{c} = EXCLUDED.{c}" for c in cols[1:])
|
||||
sql = (
|
||||
f"INSERT INTO openmeteo_rain ({', '.join(cols)}) VALUES ({placeholders}) "
|
||||
f"ON CONFLICT (timestamp) DO UPDATE SET {updates}"
|
||||
)
|
||||
if db_type == "mysql":
|
||||
sql = (
|
||||
f"INSERT INTO openmeteo_rain ({', '.join(cols)}) VALUES ({placeholders}) "
|
||||
f"ON DUPLICATE KEY UPDATE {updates}"
|
||||
)
|
||||
else:
|
||||
sql = (
|
||||
f"INSERT INTO openmeteo_rain ({', '.join(cols)}) VALUES ({placeholders}) "
|
||||
f"ON CONFLICT (timestamp) DO UPDATE SET {updates}"
|
||||
)
|
||||
mean = df.mean(axis=1)
|
||||
params = [
|
||||
{
|
||||
|
||||
-170
@@ -1,170 +0,0 @@
|
||||
"""Live forecast skill: what the deployed model said versus what the river did.
|
||||
|
||||
Every hour the precompute stores the issued 24 h forecast (forecast_history);
|
||||
water_measurements holds what actually happened. Joining the two gives a
|
||||
verification that needs no retraining and answers the question the dashboard
|
||||
is asked most: "is the model getting better?" — per model version, on the
|
||||
hours that version was actually serving.
|
||||
|
||||
Metrics per version and horizon:
|
||||
n verified forecasts (issued, and the horizon has since elapsed)
|
||||
mae |predicted_max - observed_max| over the horizon window, metres
|
||||
bias mean(predicted - observed): >0 over-predicts the peak
|
||||
persistence MAE of the trivial "peak = current level" forecast on the
|
||||
same rows; a model is only useful if it beats this
|
||||
skill 1 - mae/persistence (0 = no better than persistence, 1 = perfect)
|
||||
above_2m same MAE restricted to rows where the observed peak >= 2 m,
|
||||
i.e. the flood-relevant regime
|
||||
|
||||
Only the P.1 gauge is verified by default: it is the one the city threshold
|
||||
is keyed to, and one station keeps the query cheap enough to run on request.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import logging
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DEFAULT_STATION = "P.1"
|
||||
DEFAULT_HORIZON = 24
|
||||
MIN_VERIFIED = 24 # fewer than a day of verified hours is not a number
|
||||
|
||||
|
||||
def _sql_for(db_type: str) -> str:
|
||||
"""Join each issued forecast to the observed max over (as_of, as_of + h]."""
|
||||
if db_type == "postgresql":
|
||||
window_end = "f.as_of + (f.horizon_hours || ' hours')::interval"
|
||||
elif db_type == "mysql":
|
||||
window_end = "DATE_ADD(f.as_of, INTERVAL f.horizon_hours HOUR)"
|
||||
else: # sqlite
|
||||
window_end = "datetime(f.as_of, '+' || f.horizon_hours || ' hours')"
|
||||
return f"""
|
||||
SELECT f.as_of, f.model_version, f.predicted_max_level, f.current_level,
|
||||
(SELECT MAX(m.water_level) FROM water_measurements m
|
||||
JOIN stations s ON s.id = m.station_id
|
||||
WHERE s.station_code = f.station_code
|
||||
AND m.timestamp > f.as_of AND m.timestamp <= {window_end}) AS observed_max,
|
||||
(SELECT COUNT(m.water_level) FROM water_measurements m
|
||||
JOIN stations s ON s.id = m.station_id
|
||||
WHERE s.station_code = f.station_code
|
||||
AND m.timestamp > f.as_of AND m.timestamp <= {window_end}) AS observed_n
|
||||
FROM forecast_history f
|
||||
WHERE f.station_code = :code AND f.horizon_hours = :horizon
|
||||
AND f.source = 'model' AND f.predicted_max_level IS NOT NULL
|
||||
AND f.as_of <= :verifiable_before
|
||||
ORDER BY f.as_of
|
||||
"""
|
||||
|
||||
|
||||
def compute_skill(
|
||||
engine,
|
||||
db_type: str,
|
||||
station_code: str = DEFAULT_STATION,
|
||||
horizon_hours: int = DEFAULT_HORIZON,
|
||||
now: Optional[datetime.datetime] = None,
|
||||
) -> Dict:
|
||||
"""Per-model-version verification of issued forecasts against observations."""
|
||||
from sqlalchemy import text
|
||||
|
||||
now = now or datetime.datetime.now()
|
||||
verifiable_before = now - datetime.timedelta(hours=horizon_hours)
|
||||
with engine.connect() as conn:
|
||||
rows = [
|
||||
dict(r._mapping)
|
||||
for r in conn.execute(
|
||||
text(_sql_for(db_type)),
|
||||
{
|
||||
"code": station_code,
|
||||
"horizon": horizon_hours,
|
||||
"verifiable_before": verifiable_before,
|
||||
},
|
||||
)
|
||||
]
|
||||
|
||||
def _ts(value):
|
||||
# sqlite hands back strings; postgres/mysql give datetimes
|
||||
if isinstance(value, datetime.datetime):
|
||||
return value
|
||||
return datetime.datetime.fromisoformat(str(value).replace(" ", "T"))
|
||||
|
||||
by_version: Dict[str, List[dict]] = {}
|
||||
for r in rows:
|
||||
r["as_of"] = _ts(r["as_of"])
|
||||
# need most of the window observed, or the "max" is not the peak
|
||||
if r["observed_max"] is None or (r["observed_n"] or 0) < horizon_hours * 0.75:
|
||||
continue
|
||||
by_version.setdefault(r["model_version"] or "unknown", []).append(r)
|
||||
|
||||
versions = []
|
||||
for version, vrows in by_version.items():
|
||||
pred = [float(r["predicted_max_level"]) for r in vrows]
|
||||
obs = [float(r["observed_max"]) for r in vrows]
|
||||
cur = [
|
||||
float(r["current_level"]) if r["current_level"] is not None else None
|
||||
for r in vrows
|
||||
]
|
||||
err = [p - o for p, o in zip(pred, obs)]
|
||||
mae = sum(abs(e) for e in err) / len(err)
|
||||
bias = sum(err) / len(err)
|
||||
pers_rows = [(c, o) for c, o in zip(cur, obs) if c is not None]
|
||||
persistence = (
|
||||
sum(abs(c - o) for c, o in pers_rows) / len(pers_rows)
|
||||
if pers_rows
|
||||
else None
|
||||
)
|
||||
high = [(p, o) for p, o in zip(pred, obs) if o >= 2.0]
|
||||
versions.append(
|
||||
{
|
||||
"model_version": version,
|
||||
"first_issued": min(r["as_of"] for r in vrows).isoformat(),
|
||||
"last_issued": max(r["as_of"] for r in vrows).isoformat(),
|
||||
"n": len(vrows),
|
||||
"mae_m": round(mae, 3),
|
||||
"bias_m": round(bias, 3),
|
||||
"persistence_mae_m": (
|
||||
None if persistence is None else round(persistence, 3)
|
||||
),
|
||||
"skill": (
|
||||
None if not persistence else round(1.0 - mae / persistence, 3)
|
||||
),
|
||||
"above_2m_n": len(high),
|
||||
"above_2m_mae_m": (
|
||||
round(sum(abs(p - o) for p, o in high) / len(high), 3)
|
||||
if high
|
||||
else None
|
||||
),
|
||||
"enough_data": len(vrows) >= MIN_VERIFIED,
|
||||
}
|
||||
)
|
||||
versions.sort(key=lambda v: v["first_issued"])
|
||||
|
||||
# Headline: current version vs the previous one that had enough data
|
||||
current = versions[-1] if versions else None
|
||||
previous = (
|
||||
next((v for v in reversed(versions[:-1]) if v["enough_data"]), None)
|
||||
if versions
|
||||
else None
|
||||
)
|
||||
trend = None
|
||||
if current and previous and current["enough_data"]:
|
||||
trend = {
|
||||
"previous_version": previous["model_version"],
|
||||
"mae_delta_m": round(current["mae_m"] - previous["mae_m"], 3),
|
||||
"skill_delta": (
|
||||
None
|
||||
if current["skill"] is None or previous["skill"] is None
|
||||
else round(current["skill"] - previous["skill"], 3)
|
||||
),
|
||||
"better": current["mae_m"] < previous["mae_m"],
|
||||
}
|
||||
|
||||
return {
|
||||
"station_code": station_code,
|
||||
"horizon_hours": horizon_hours,
|
||||
"verified_until": verifiable_before.isoformat(),
|
||||
"min_verified": MIN_VERIFIED,
|
||||
"versions": versions,
|
||||
"current": current,
|
||||
"trend": trend,
|
||||
}
|
||||
-121
@@ -1,121 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Data models for water monitoring system
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
|
||||
class DatabaseType(Enum):
|
||||
SQLITE = "sqlite"
|
||||
MYSQL = "mysql"
|
||||
POSTGRESQL = "postgresql"
|
||||
INFLUXDB = "influxdb"
|
||||
VICTORIAMETRICS = "victoriametrics"
|
||||
|
||||
|
||||
class StationStatus(Enum):
|
||||
ACTIVE = "active"
|
||||
INACTIVE = "inactive"
|
||||
MAINTENANCE = "maintenance"
|
||||
ERROR = "error"
|
||||
|
||||
|
||||
@dataclass
|
||||
class StationInfo:
|
||||
"""Station information model"""
|
||||
|
||||
station_id: int
|
||||
station_code: str
|
||||
thai_name: str
|
||||
english_name: str
|
||||
latitude: Optional[float] = None
|
||||
longitude: Optional[float] = None
|
||||
geohash: Optional[str] = None
|
||||
status: StationStatus = StationStatus.ACTIVE
|
||||
|
||||
|
||||
@dataclass
|
||||
class WaterMeasurement:
|
||||
"""Water measurement data model"""
|
||||
|
||||
timestamp: datetime
|
||||
station_info: StationInfo
|
||||
water_level: float
|
||||
discharge: float
|
||||
water_level_unit: str = "m"
|
||||
discharge_unit: str = "cms"
|
||||
discharge_percent: Optional[float] = None
|
||||
status: StationStatus = StationStatus.ACTIVE
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert to dictionary for database storage"""
|
||||
return {
|
||||
"timestamp": self.timestamp,
|
||||
"station_id": self.station_info.station_id,
|
||||
"station_code": self.station_info.station_code,
|
||||
"station_name_en": self.station_info.english_name,
|
||||
"station_name_th": self.station_info.thai_name,
|
||||
"latitude": self.station_info.latitude,
|
||||
"longitude": self.station_info.longitude,
|
||||
"geohash": self.station_info.geohash,
|
||||
"water_level": self.water_level,
|
||||
"water_level_unit": self.water_level_unit,
|
||||
"discharge": self.discharge,
|
||||
"discharge_unit": self.discharge_unit,
|
||||
"discharge_percent": self.discharge_percent,
|
||||
"status": self.status.value,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class DatabaseConfig:
|
||||
"""Database configuration model"""
|
||||
|
||||
db_type: DatabaseType
|
||||
connection_string: Optional[str] = None
|
||||
host: Optional[str] = None
|
||||
port: Optional[int] = None
|
||||
database: Optional[str] = None
|
||||
username: Optional[str] = None
|
||||
password: Optional[str] = None
|
||||
additional_params: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScrapingResult:
|
||||
"""Result of a scraping operation"""
|
||||
|
||||
success: bool
|
||||
measurements_count: int
|
||||
error_message: Optional[str] = None
|
||||
timestamp: datetime = field(default_factory=datetime.now)
|
||||
processing_time_seconds: Optional[float] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class StationCreateRequest:
|
||||
"""Request model for creating a new station"""
|
||||
|
||||
station_code: str
|
||||
thai_name: str
|
||||
english_name: str
|
||||
latitude: Optional[float] = None
|
||||
longitude: Optional[float] = None
|
||||
geohash: Optional[str] = None
|
||||
status: StationStatus = StationStatus.ACTIVE
|
||||
|
||||
|
||||
@dataclass
|
||||
class StationUpdateRequest:
|
||||
"""Request model for updating an existing station"""
|
||||
|
||||
thai_name: Optional[str] = None
|
||||
english_name: Optional[str] = None
|
||||
latitude: Optional[float] = None
|
||||
longitude: Optional[float] = None
|
||||
geohash: Optional[str] = None
|
||||
status: Optional[StationStatus] = None
|
||||
-456
@@ -1,456 +0,0 @@
|
||||
"""Public flood notifications over ntfy.
|
||||
|
||||
Runs once per collection cycle inside the API process (leader only), right
|
||||
after the forecast precompute, so it sees the same readings and forecasts the
|
||||
dashboard shows. Publishes to a self-hosted ntfy server; anyone subscribes to
|
||||
a topic from the free app or a browser, no account needed.
|
||||
|
||||
Topics (all under one configurable prefix, default "ping"):
|
||||
|
||||
{prefix}-{station}-warning observed level crossed the station's warning threshold
|
||||
{prefix}-{station}-danger observed level crossed the danger threshold
|
||||
{prefix}-warning any station crossed warning (basin-wide digest)
|
||||
{prefix}-danger any station crossed danger
|
||||
{prefix}-p1-outlook model early warning for Chiang Mai city: P.1's 24 h
|
||||
warning probability crossed the alert level (opt-in;
|
||||
the forecast is experimental and says so)
|
||||
{prefix}-status feed/monitor health: data stale, recovered
|
||||
|
||||
Each notification is a TRANSITION, not a state: crossing UP into a level sends
|
||||
one message; dropping back below (with hysteresis) sends an all-clear. While
|
||||
the river sits above a threshold nothing is repeated, so a subscriber in a
|
||||
flood gets a handful of messages, not one an hour. The per-topic state is
|
||||
persisted (notification_state table) so a restart never re-sends.
|
||||
|
||||
Everything is fail-safe: ntfy unreachable, table missing, malformed
|
||||
reading -> a logged warning, never an exception into the collection loop.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from typing import Dict, Iterable, List, Optional
|
||||
|
||||
import requests
|
||||
|
||||
from .ml import features
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Hysteresis: an all-clear needs the level this far BELOW the threshold, so a
|
||||
# river bobbing around 3.70 m does not toggle warning/clear every hour.
|
||||
CLEAR_MARGIN_M = 0.10
|
||||
# Capacity guard. The level thresholds in features.THRESHOLDS were calibrated
|
||||
# from RID's discharge_percent (% of channel capacity); if RID re-rates a
|
||||
# gauge or moves its datum, the level crosses while capacity says the channel
|
||||
# is nearly empty (P.77, 2026-09: 3.0 m "warning" at 22 %). A crossing is
|
||||
# only announced when the reported capacity agrees that the river is high.
|
||||
# P.1 is exempt: its stages come from the municipal inundation map, not from
|
||||
# capacity. Readings without a capacity figure fall back to level only.
|
||||
CAPACITY_GUARD_MIN_PCT = 60.0
|
||||
CAPACITY_GUARD_EXEMPT = {"P.1"}
|
||||
# Outlook alert fires when p_warning(24h) rises through ON, clears below OFF.
|
||||
OUTLOOK_ON = 0.50
|
||||
OUTLOOK_OFF = 0.25
|
||||
# Below this the outlook is not announced at all (avoid "5 % chance" noise).
|
||||
OUTLOOK_HORIZON = 24
|
||||
# No reading this many hours old or newer: the feed is stale.
|
||||
FEED_STALE_AFTER_H = 3.0
|
||||
|
||||
STATION_NAMES: Dict[str, str] = {
|
||||
"P.1": "Nawarat Bridge, Chiang Mai city",
|
||||
"P.103": "Ring Road Bridge 3, Chiang Mai",
|
||||
"P.67": "Ban Tae (Mae Taeng)",
|
||||
"P.21": "Ban Rim Tai (Mae Rim)",
|
||||
"P.75": "Ban Chai Lat",
|
||||
"P.92": "Ban Muang Aut",
|
||||
"P.20": "Ban Chiang Dao",
|
||||
"P.4A": "Ban Mae Taeng",
|
||||
"P.5": "Tha Nang Bridge (downstream)",
|
||||
"P.81": "Ban Pong (downstream)",
|
||||
"P.82": "Ban Sob Win",
|
||||
"P.84": "Ban Panton",
|
||||
"P.87": "Ban Pa Sang",
|
||||
"P.77": "Ban Sop Mae Sapuat",
|
||||
"P.85": "Ban Lai Kaew",
|
||||
"P.76": "Ban Mae I Hai",
|
||||
}
|
||||
|
||||
|
||||
def _slug(code: str) -> str:
|
||||
return code.lower().replace(".", "")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Notification:
|
||||
topic: str
|
||||
title: str
|
||||
message: str
|
||||
priority: int = 3 # ntfy: 1 min .. 5 max
|
||||
tags: Optional[List[str]] = None
|
||||
click: Optional[str] = None
|
||||
|
||||
|
||||
class NtfyPublisher:
|
||||
def __init__(
|
||||
self,
|
||||
server: str,
|
||||
prefix: str = "ping",
|
||||
token: Optional[str] = None,
|
||||
dashboard_url: str = "https://water.buildfor.life/",
|
||||
timeout: int = 10,
|
||||
):
|
||||
self.server = server.rstrip("/")
|
||||
self.prefix = prefix
|
||||
self.token = token
|
||||
self.dashboard_url = dashboard_url
|
||||
self.timeout = timeout
|
||||
|
||||
def topic(self, *parts: str) -> str:
|
||||
return "-".join([self.prefix, *parts])
|
||||
|
||||
def publish(self, n: Notification) -> bool:
|
||||
headers = {
|
||||
"Title": n.title,
|
||||
"Priority": str(n.priority),
|
||||
"Click": n.click or self.dashboard_url,
|
||||
"Actions": f"view, Open dashboard, {n.click or self.dashboard_url}",
|
||||
}
|
||||
if n.tags:
|
||||
headers["Tags"] = ",".join(n.tags)
|
||||
if self.token:
|
||||
headers["Authorization"] = f"Bearer {self.token}"
|
||||
try:
|
||||
r = requests.post(
|
||||
f"{self.server}/{n.topic}",
|
||||
data=n.message.encode("utf-8"),
|
||||
headers=headers,
|
||||
timeout=self.timeout,
|
||||
)
|
||||
if r.status_code >= 300:
|
||||
logger.warning(f"ntfy {n.topic}: HTTP {r.status_code} {r.text[:120]}")
|
||||
return False
|
||||
return True
|
||||
except Exception as error:
|
||||
logger.warning(f"ntfy {n.topic}: {error}")
|
||||
return False
|
||||
|
||||
|
||||
class NotificationState:
|
||||
"""Per-key last-sent state, in the monitor's own SQL database."""
|
||||
|
||||
def __init__(self, engine, db_type: str):
|
||||
self.engine = engine
|
||||
self.db_type = db_type
|
||||
self._ensure()
|
||||
|
||||
def _ensure(self) -> None:
|
||||
from sqlalchemy import text
|
||||
|
||||
ddl = (
|
||||
"CREATE TABLE IF NOT EXISTS notification_state ("
|
||||
"key VARCHAR(64) PRIMARY KEY, state VARCHAR(16) NOT NULL, "
|
||||
"value NUMERIC(8,3), updated_at TIMESTAMP NOT NULL)"
|
||||
)
|
||||
try:
|
||||
with self.engine.begin() as conn:
|
||||
conn.execute(text(ddl))
|
||||
except Exception as error:
|
||||
# Postgres: two sessions racing CREATE TABLE IF NOT EXISTS can
|
||||
# both pass the existence check; the loser fails with a unique
|
||||
# violation on pg_type. The table exists either way; verify.
|
||||
with self.engine.connect() as conn:
|
||||
conn.execute(text("SELECT 1 FROM notification_state WHERE 1=0"))
|
||||
logger.debug(f"notification_state DDL raced, table present: {error}")
|
||||
|
||||
def get(self, key: str) -> Optional[str]:
|
||||
from sqlalchemy import text
|
||||
|
||||
with self.engine.connect() as conn:
|
||||
row = conn.execute(
|
||||
text("SELECT state FROM notification_state WHERE key = :k"), {"k": key}
|
||||
).fetchone()
|
||||
return row[0] if row else None
|
||||
|
||||
def set(self, key: str, state: str, value: Optional[float] = None) -> None:
|
||||
from sqlalchemy import text
|
||||
|
||||
now = datetime.datetime.now()
|
||||
with self.engine.begin() as conn:
|
||||
if self.db_type == "mysql":
|
||||
sql = (
|
||||
"INSERT INTO notification_state (key, state, value, updated_at) "
|
||||
"VALUES (:k, :s, :v, :t) ON DUPLICATE KEY UPDATE "
|
||||
"state = VALUES(state), value = VALUES(value), updated_at = VALUES(updated_at)"
|
||||
)
|
||||
else:
|
||||
sql = (
|
||||
"INSERT INTO notification_state (key, state, value, updated_at) "
|
||||
"VALUES (:k, :s, :v, :t) ON CONFLICT (key) DO UPDATE SET "
|
||||
"state = EXCLUDED.state, value = EXCLUDED.value, updated_at = EXCLUDED.updated_at"
|
||||
)
|
||||
conn.execute(text(sql), {"k": key, "s": state, "v": value, "t": now})
|
||||
|
||||
|
||||
class InMemoryState(NotificationState):
|
||||
"""For tests and when no SQL engine is available (loses state on restart)."""
|
||||
|
||||
def __init__(self): # noqa: D107 - intentionally skips the SQL parent
|
||||
self._d: Dict[str, str] = {}
|
||||
|
||||
def get(self, key: str) -> Optional[str]:
|
||||
return self._d.get(key)
|
||||
|
||||
def set(self, key: str, state: str, value: Optional[float] = None) -> None:
|
||||
self._d[key] = state
|
||||
|
||||
|
||||
def _level_state(level: float, warn: float, danger: float, prev: Optional[str]) -> str:
|
||||
"""'clear' | 'warning' | 'danger', with hysteresis on the way down."""
|
||||
if level >= danger:
|
||||
return "danger"
|
||||
if level >= warn:
|
||||
# from danger: stay 'danger' until below danger - margin
|
||||
if prev == "danger" and level >= danger - CLEAR_MARGIN_M:
|
||||
return "danger"
|
||||
return "warning"
|
||||
if prev in ("warning", "danger") and level >= warn - CLEAR_MARGIN_M:
|
||||
return "warning"
|
||||
return "clear"
|
||||
|
||||
|
||||
def evaluate(
|
||||
readings: Iterable[dict],
|
||||
forecasts: Iterable[dict],
|
||||
state: NotificationState,
|
||||
publisher: NtfyPublisher,
|
||||
stale_after_h: float = FEED_STALE_AFTER_H,
|
||||
now: Optional[datetime.datetime] = None,
|
||||
) -> List[Notification]:
|
||||
"""Compare current readings/forecasts with last-sent state; publish transitions.
|
||||
|
||||
readings: rows with station_code, water_level, timestamp (latest per station)
|
||||
forecasts: /forecast rows (station_code, horizon_hours, p_warning, predicted_max_level)
|
||||
Returns the notifications that were published (for logs/tests).
|
||||
"""
|
||||
now = now or datetime.datetime.now()
|
||||
sent: List[Notification] = []
|
||||
|
||||
def emit(n: Notification) -> bool:
|
||||
ok = publisher.publish(n)
|
||||
if ok:
|
||||
sent.append(n)
|
||||
return ok
|
||||
|
||||
# ---- observed levels, per station, plus basin-wide fan-out
|
||||
basin_changes: Dict[str, List[str]] = {"warning": [], "danger": [], "clear": []}
|
||||
latest_ts: Optional[datetime.datetime] = None
|
||||
for r in readings:
|
||||
code = r.get("station_code")
|
||||
level = r.get("water_level")
|
||||
if not code or level is None:
|
||||
continue
|
||||
try:
|
||||
level = float(level)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
ts = r.get("timestamp")
|
||||
if isinstance(ts, str):
|
||||
try:
|
||||
ts = datetime.datetime.fromisoformat(ts)
|
||||
except ValueError:
|
||||
ts = None
|
||||
if isinstance(ts, datetime.datetime) and (latest_ts is None or ts > latest_ts):
|
||||
latest_ts = ts
|
||||
warn, danger = features.get_thresholds(code)
|
||||
key = f"level:{code}"
|
||||
prev = state.get(key) or "clear"
|
||||
cur = _level_state(level, warn, danger, prev)
|
||||
pct = r.get("discharge_percent")
|
||||
if (
|
||||
cur != "clear"
|
||||
and prev == "clear"
|
||||
and code not in CAPACITY_GUARD_EXEMPT
|
||||
and pct is not None
|
||||
):
|
||||
try:
|
||||
if float(pct) < CAPACITY_GUARD_MIN_PCT:
|
||||
logger.info(
|
||||
f"{code}: level {level:.2f} m >= {warn:.2f} but only "
|
||||
f"{float(pct):.0f}% capacity; threshold looks stale, not alerting"
|
||||
)
|
||||
continue
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
if cur == prev:
|
||||
continue
|
||||
name = STATION_NAMES.get(code, code)
|
||||
slug = _slug(code)
|
||||
when = (
|
||||
ts.strftime("%d %b %H:%M") if isinstance(ts, datetime.datetime) else "now"
|
||||
)
|
||||
if cur == "danger":
|
||||
ok = emit(
|
||||
Notification(
|
||||
publisher.topic(slug, "danger"),
|
||||
f"DANGER level at {code}",
|
||||
f"{name}: {level:.2f} m at {when}, above the danger level of {danger:.2f} m.",
|
||||
priority=5,
|
||||
tags=["rotating_light", code],
|
||||
)
|
||||
)
|
||||
basin_changes["danger"].append(f"{code} {level:.2f} m")
|
||||
elif cur == "warning":
|
||||
if prev == "danger":
|
||||
ok = emit(
|
||||
Notification(
|
||||
publisher.topic(slug, "danger"),
|
||||
f"{code} back below danger level",
|
||||
f"{name}: {level:.2f} m at {when}; still above the warning level of {warn:.2f} m.",
|
||||
priority=3,
|
||||
tags=["arrow_down", code],
|
||||
)
|
||||
)
|
||||
basin_changes["clear"].append(f"{code} below danger ({level:.2f} m)")
|
||||
else:
|
||||
ok = emit(
|
||||
Notification(
|
||||
publisher.topic(slug, "warning"),
|
||||
f"Warning level at {code}",
|
||||
f"{name}: {level:.2f} m at {when}, above the warning level of {warn:.2f} m.",
|
||||
priority=4,
|
||||
tags=["warning", code],
|
||||
)
|
||||
)
|
||||
basin_changes["warning"].append(f"{code} {level:.2f} m")
|
||||
else: # clear
|
||||
ok = emit(
|
||||
Notification(
|
||||
publisher.topic(slug, "warning"),
|
||||
f"{code} back to normal",
|
||||
f"{name}: {level:.2f} m at {when}, below the warning level of {warn:.2f} m.",
|
||||
priority=2,
|
||||
tags=["white_check_mark", code],
|
||||
)
|
||||
)
|
||||
basin_changes["clear"].append(f"{code} normal ({level:.2f} m)")
|
||||
# Only remember the transition once it was actually delivered: if ntfy
|
||||
# was down, the next cycle retries instead of silently swallowing a
|
||||
# flood crossing.
|
||||
if ok:
|
||||
state.set(key, cur, level)
|
||||
|
||||
if basin_changes["danger"]:
|
||||
emit(
|
||||
Notification(
|
||||
publisher.topic("danger"),
|
||||
"Ping River: danger level reached",
|
||||
"; ".join(basin_changes["danger"]),
|
||||
priority=5,
|
||||
tags=["rotating_light"],
|
||||
)
|
||||
)
|
||||
if basin_changes["warning"]:
|
||||
emit(
|
||||
Notification(
|
||||
publisher.topic("warning"),
|
||||
"Ping River: warning level reached",
|
||||
"; ".join(basin_changes["warning"]),
|
||||
priority=4,
|
||||
tags=["warning"],
|
||||
)
|
||||
)
|
||||
if basin_changes["clear"]:
|
||||
emit(
|
||||
Notification(
|
||||
publisher.topic("warning"),
|
||||
"Ping River: levels falling",
|
||||
"; ".join(basin_changes["clear"]),
|
||||
priority=2,
|
||||
tags=["white_check_mark"],
|
||||
)
|
||||
)
|
||||
|
||||
# ---- model outlook for the city gauge (opt-in topic, experimental)
|
||||
p1 = next(
|
||||
(
|
||||
f
|
||||
for f in forecasts
|
||||
if f.get("station_code") == "P.1"
|
||||
and f.get("horizon_hours") == OUTLOOK_HORIZON
|
||||
and f.get("source") == "model"
|
||||
),
|
||||
None,
|
||||
)
|
||||
if p1 and p1.get("p_warning") is not None:
|
||||
p = float(p1["p_warning"])
|
||||
key = "outlook:P.1"
|
||||
prev = state.get(key) or "off"
|
||||
cur = (
|
||||
"on" if (p >= OUTLOOK_ON or (prev == "on" and p >= OUTLOOK_OFF)) else "off"
|
||||
)
|
||||
if cur != prev:
|
||||
peak = p1.get("predicted_max_level")
|
||||
warn, _ = features.get_thresholds("P.1")
|
||||
if cur == "on":
|
||||
ok = emit(
|
||||
Notification(
|
||||
publisher.topic("p1-outlook"),
|
||||
"Early warning: Chiang Mai flood risk rising",
|
||||
f"The forecast model gives a {p * 100:.0f}% chance that Nawarat Bridge (P.1) "
|
||||
f"reaches {warn:.2f} m within 24 h"
|
||||
+ (
|
||||
f" (expected peak {float(peak):.2f} m)"
|
||||
if peak is not None
|
||||
else ""
|
||||
)
|
||||
+ ". Experimental model output, not an official warning; "
|
||||
"follow ThaiWater/TMD for official alerts.",
|
||||
priority=4,
|
||||
tags=["crystal_ball"],
|
||||
)
|
||||
)
|
||||
else:
|
||||
ok = emit(
|
||||
Notification(
|
||||
publisher.topic("p1-outlook"),
|
||||
"Chiang Mai flood risk easing",
|
||||
f"The model's 24 h probability of reaching {warn:.2f} m at P.1 has dropped to {p * 100:.0f}%.",
|
||||
priority=2,
|
||||
tags=["crystal_ball"],
|
||||
)
|
||||
)
|
||||
if ok:
|
||||
state.set(key, cur, p)
|
||||
|
||||
# ---- feed health
|
||||
if latest_ts is not None:
|
||||
age_h = (now - latest_ts).total_seconds() / 3600.0
|
||||
key = "feed"
|
||||
prev = state.get(key) or "ok"
|
||||
cur = "stale" if age_h >= stale_after_h else "ok"
|
||||
if cur != prev:
|
||||
if cur == "stale":
|
||||
ok = emit(
|
||||
Notification(
|
||||
publisher.topic("status"),
|
||||
"Ping River monitor: gauge feed stale",
|
||||
f"No new readings for {age_h:.0f} h (last {latest_ts:%d %b %H:%M}). "
|
||||
"Levels and forecasts on the dashboard are not current.",
|
||||
priority=3,
|
||||
tags=["hourglass"],
|
||||
)
|
||||
)
|
||||
else:
|
||||
ok = emit(
|
||||
Notification(
|
||||
publisher.topic("status"),
|
||||
"Ping River monitor: feed recovered",
|
||||
f"Readings are current again (latest {latest_ts:%d %b %H:%M}).",
|
||||
priority=2,
|
||||
tags=["white_check_mark"],
|
||||
)
|
||||
)
|
||||
if ok:
|
||||
state.set(key, cur, age_h)
|
||||
return sent
|
||||
@@ -1,97 +0,0 @@
|
||||
"""Read historical station measurements from PostgreSQL."""
|
||||
|
||||
import datetime
|
||||
import os
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
from sqlalchemy import create_engine, text
|
||||
|
||||
# Stage-discharge rating curves: Q = a * (H - b)^c
|
||||
# Key: station_code, Value: (a, b, c)
|
||||
# Use linear fallback Q = slope * H if a curve is not defined.
|
||||
_RATING_CURVES: Dict[str, Tuple[float, float, float]] = {}
|
||||
_DEFAULT_LINEAR_SLOPE = 20.0 # m^3/s per meter
|
||||
|
||||
|
||||
def _calculate_discharge(
|
||||
water_level: Optional[float], station_code: str = None
|
||||
) -> Optional[float]:
|
||||
"""Estimate discharge from water level using a rating curve or linear fallback."""
|
||||
if water_level is None:
|
||||
return None
|
||||
|
||||
curve = _RATING_CURVES.get(station_code)
|
||||
if curve:
|
||||
a, b, c = curve
|
||||
h_excess = water_level - b
|
||||
if h_excess <= 0:
|
||||
return 0.0
|
||||
return round(a * (h_excess**c), 2)
|
||||
|
||||
# Linear fallback: Q = slope * H
|
||||
return round(_DEFAULT_LINEAR_SLOPE * water_level, 2)
|
||||
|
||||
|
||||
class PostgresHistory:
|
||||
def __init__(self, connection_string: Optional[str] = None, engine=None):
|
||||
connection_string = connection_string or os.getenv("POSTGRES_CONNECTION_STRING")
|
||||
if engine is None and not connection_string:
|
||||
raise RuntimeError("POSTGRES_CONNECTION_STRING is not configured")
|
||||
self.engine = engine or create_engine(connection_string, pool_pre_ping=True)
|
||||
|
||||
def station_history(
|
||||
self,
|
||||
station_code: str,
|
||||
start: datetime.datetime,
|
||||
end: datetime.datetime,
|
||||
limit: int = 2000,
|
||||
) -> List[Dict]:
|
||||
if not 1 <= limit <= 100000:
|
||||
raise ValueError("limit must be between 1 and 100000")
|
||||
if start >= end:
|
||||
raise ValueError("start must be before end")
|
||||
|
||||
query = text("""
|
||||
SELECT m.timestamp, s.station_code, m.water_level,
|
||||
m.discharge, m.discharge_percent
|
||||
FROM water_measurements m
|
||||
JOIN stations s ON m.station_id = s.id
|
||||
WHERE s.station_code = :station_code
|
||||
AND m.timestamp >= :start_time
|
||||
AND m.timestamp <= :end_time
|
||||
ORDER BY m.timestamp ASC
|
||||
LIMIT :limit
|
||||
""")
|
||||
with self.engine.connect() as connection:
|
||||
rows = connection.execute(
|
||||
query,
|
||||
{
|
||||
"station_code": station_code,
|
||||
"start_time": start,
|
||||
"end_time": end,
|
||||
"limit": limit,
|
||||
},
|
||||
)
|
||||
result = []
|
||||
for row in rows:
|
||||
timestamp = row[0]
|
||||
if isinstance(timestamp, str):
|
||||
timestamp = datetime.datetime.fromisoformat(timestamp)
|
||||
station_code = row[1]
|
||||
water_level = float(row[2]) if row[2] is not None else None
|
||||
discharge = float(row[3]) if row[3] is not None else None
|
||||
# Estimate discharge from water level if DB value is missing
|
||||
if discharge is None and water_level is not None:
|
||||
discharge = _calculate_discharge(water_level, station_code)
|
||||
result.append(
|
||||
{
|
||||
"timestamp": timestamp,
|
||||
"station_code": station_code,
|
||||
"water_level": water_level,
|
||||
"discharge": discharge,
|
||||
"discharge_percent": (
|
||||
float(row[4]) if row[4] is not None else None
|
||||
),
|
||||
}
|
||||
)
|
||||
return result
|
||||
@@ -1,182 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Rate limiting utilities for API requests
|
||||
"""
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from collections import deque
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RateLimiter:
|
||||
"""Token bucket rate limiter"""
|
||||
|
||||
def __init__(self, max_requests: int, time_window_seconds: int):
|
||||
"""
|
||||
Initialize rate limiter
|
||||
|
||||
Args:
|
||||
max_requests: Maximum number of requests allowed
|
||||
time_window_seconds: Time window in seconds
|
||||
"""
|
||||
self.max_requests = max_requests
|
||||
self.time_window = time_window_seconds
|
||||
self.requests = deque()
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def is_allowed(self) -> bool:
|
||||
"""Check if a request is allowed"""
|
||||
with self._lock:
|
||||
now = time.time()
|
||||
|
||||
# Remove old requests outside the time window
|
||||
while self.requests and self.requests[0] <= now - self.time_window:
|
||||
self.requests.popleft()
|
||||
|
||||
# Check if we can make a new request
|
||||
if len(self.requests) < self.max_requests:
|
||||
self.requests.append(now)
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def wait_time(self) -> float:
|
||||
"""Get the time to wait before next request is allowed"""
|
||||
with self._lock:
|
||||
if len(self.requests) < self.max_requests:
|
||||
return 0.0
|
||||
|
||||
# Time until the oldest request expires
|
||||
oldest_request = self.requests[0]
|
||||
return max(0.0, (oldest_request + self.time_window) - time.time())
|
||||
|
||||
def wait_if_needed(self):
|
||||
"""Block until a request is allowed"""
|
||||
wait_time = self.wait_time()
|
||||
if wait_time > 0:
|
||||
logger.info(f"Rate limit reached, waiting {wait_time:.2f} seconds")
|
||||
time.sleep(wait_time)
|
||||
|
||||
|
||||
class AdaptiveRateLimiter:
|
||||
"""Adaptive rate limiter that adjusts based on response times"""
|
||||
|
||||
def __init__(
|
||||
self, initial_rate: float = 1.0, min_rate: float = 0.1, max_rate: float = 10.0
|
||||
):
|
||||
"""
|
||||
Initialize adaptive rate limiter
|
||||
|
||||
Args:
|
||||
initial_rate: Initial requests per second
|
||||
min_rate: Minimum requests per second
|
||||
max_rate: Maximum requests per second
|
||||
"""
|
||||
self.current_rate = initial_rate
|
||||
self.min_rate = min_rate
|
||||
self.max_rate = max_rate
|
||||
self.last_request_time = 0.0
|
||||
self.response_times = deque(maxlen=10)
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def wait_and_record(self, response_time: Optional[float] = None):
|
||||
"""Wait for rate limit and record response time"""
|
||||
with self._lock:
|
||||
now = time.time()
|
||||
|
||||
# Calculate wait time based on current rate
|
||||
time_since_last = now - self.last_request_time
|
||||
min_interval = 1.0 / self.current_rate
|
||||
|
||||
if time_since_last < min_interval:
|
||||
wait_time = min_interval - time_since_last
|
||||
time.sleep(wait_time)
|
||||
now = time.time()
|
||||
|
||||
self.last_request_time = now
|
||||
|
||||
# Record response time and adjust rate
|
||||
if response_time is not None:
|
||||
self.response_times.append(response_time)
|
||||
self._adjust_rate()
|
||||
|
||||
def _adjust_rate(self):
|
||||
"""Adjust rate based on recent response times"""
|
||||
if len(self.response_times) < 3:
|
||||
return
|
||||
|
||||
avg_response_time = sum(self.response_times) / len(self.response_times)
|
||||
|
||||
# Decrease rate if responses are slow
|
||||
if avg_response_time > 5.0: # 5 seconds
|
||||
self.current_rate = max(self.min_rate, self.current_rate * 0.8)
|
||||
logger.info(
|
||||
f"Decreased rate to {self.current_rate:.2f} req/s due to slow responses"
|
||||
)
|
||||
|
||||
# Increase rate if responses are fast
|
||||
elif avg_response_time < 1.0: # 1 second
|
||||
self.current_rate = min(self.max_rate, self.current_rate * 1.1)
|
||||
logger.debug(f"Increased rate to {self.current_rate:.2f} req/s")
|
||||
|
||||
|
||||
class RequestTracker:
|
||||
"""Track API request statistics"""
|
||||
|
||||
def __init__(self):
|
||||
self.total_requests = 0
|
||||
self.successful_requests = 0
|
||||
self.failed_requests = 0
|
||||
self.total_response_time = 0.0
|
||||
self.last_request_time = None
|
||||
self.error_count_by_type = {}
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def record_request(
|
||||
self, success: bool, response_time: float, error_type: Optional[str] = None
|
||||
):
|
||||
"""Record a request"""
|
||||
with self._lock:
|
||||
self.total_requests += 1
|
||||
self.total_response_time += response_time
|
||||
self.last_request_time = datetime.now()
|
||||
|
||||
if success:
|
||||
self.successful_requests += 1
|
||||
else:
|
||||
self.failed_requests += 1
|
||||
if error_type:
|
||||
self.error_count_by_type[error_type] = (
|
||||
self.error_count_by_type.get(error_type, 0) + 1
|
||||
)
|
||||
|
||||
def get_stats(self) -> Dict[str, any]:
|
||||
"""Get request statistics"""
|
||||
with self._lock:
|
||||
if self.total_requests == 0:
|
||||
return {
|
||||
"total_requests": 0,
|
||||
"success_rate": 0.0,
|
||||
"average_response_time": 0.0,
|
||||
"last_request_time": None,
|
||||
"error_breakdown": {},
|
||||
}
|
||||
|
||||
return {
|
||||
"total_requests": self.total_requests,
|
||||
"successful_requests": self.successful_requests,
|
||||
"failed_requests": self.failed_requests,
|
||||
"success_rate": self.successful_requests / self.total_requests,
|
||||
"average_response_time": self.total_response_time / self.total_requests,
|
||||
"last_request_time": (
|
||||
self.last_request_time.isoformat()
|
||||
if self.last_request_time
|
||||
else None
|
||||
),
|
||||
"error_breakdown": dict(self.error_count_by_type),
|
||||
}
|
||||
+17
-107
@@ -1,4 +1,10 @@
|
||||
"""Collector for RID large-dam daily status (app.rid.go.th/reservoir).
|
||||
"""RID large-dam daily status (app.rid.go.th/reservoir): client, PostgreSQL
|
||||
store and archive backfill.
|
||||
|
||||
The daily collection (today + yesterday's late revisions) runs in
|
||||
ping-collector (rust/collector/src/dams.rs, same parsing and tables); this
|
||||
module is used by scripts/backfill_rid_reservoir.py, the dam features
|
||||
(src/ml/dam.py) and the tests.
|
||||
|
||||
The Royal Irrigation Department reservoir app exposes an unauthenticated
|
||||
JSON API: ``POST https://app.rid.go.th/reservoir/api/dams`` with form field
|
||||
@@ -215,19 +221,14 @@ class RidReservoirClient:
|
||||
|
||||
|
||||
class RidReservoirStore:
|
||||
"""SQL persistence for dam metadata + daily measurements.
|
||||
"""PostgreSQL persistence for dam metadata + daily measurements.
|
||||
|
||||
Shares the app's relational database; writes rid_dams (metadata) and
|
||||
rid_reservoir_daily keyed (dam_id, date) — the composite natural PK keeps
|
||||
the table TimescaleDB-hypertable compatible.
|
||||
Writes rid_dams (metadata) and rid_reservoir_daily keyed (dam_id, date) —
|
||||
the composite natural PK keeps the table TimescaleDB-hypertable
|
||||
compatible. The DDL matches rust/collector/src/db.rs.
|
||||
"""
|
||||
|
||||
def __init__(self, connection_string: str, db_type: str):
|
||||
self.db_type = db_type.lower()
|
||||
if self.db_type not in ("sqlite", "postgresql", "mysql"):
|
||||
raise ValueError(
|
||||
f"Reservoir collection requires a SQL database, got '{db_type}'"
|
||||
)
|
||||
def __init__(self, connection_string: str):
|
||||
self.connection_string = connection_string
|
||||
self.engine = None
|
||||
|
||||
@@ -272,35 +273,12 @@ class RidReservoirStore:
|
||||
PRIMARY KEY (dam_id, date)
|
||||
)
|
||||
""",
|
||||
"CREATE INDEX IF NOT EXISTS idx_rid_reservoir_date "
|
||||
"ON rid_reservoir_daily(date)",
|
||||
]
|
||||
if self.db_type != "mysql":
|
||||
ddl.append(
|
||||
"CREATE INDEX IF NOT EXISTS idx_rid_reservoir_date "
|
||||
"ON rid_reservoir_daily(date)"
|
||||
)
|
||||
with self.engine.begin() as conn:
|
||||
for statement in ddl:
|
||||
conn.execute(text(statement))
|
||||
# Widen storage_pct on tables created before 2026-08-13: the source
|
||||
# publishes junk percents (dam 100602 reports 87798%) that overflowed
|
||||
# NUMERIC(6,2) and discarded whole daily batches.
|
||||
if self.db_type == "postgresql":
|
||||
migrations = (
|
||||
"ALTER TABLE rid_reservoir_daily "
|
||||
"ALTER COLUMN storage_pct TYPE NUMERIC(8,2)",
|
||||
)
|
||||
elif self.db_type == "mysql":
|
||||
migrations = (
|
||||
"ALTER TABLE rid_reservoir_daily MODIFY storage_pct NUMERIC(8,2)",
|
||||
)
|
||||
else: # sqlite: NUMERIC is affinity only, nothing to widen
|
||||
migrations = ()
|
||||
for statement in migrations:
|
||||
try:
|
||||
with self.engine.begin() as conn:
|
||||
conn.execute(text(statement))
|
||||
except Exception as e:
|
||||
logger.warning(f"rid_reservoir_daily migration skipped: {e}")
|
||||
|
||||
def _upsert(
|
||||
self,
|
||||
@@ -319,45 +297,17 @@ class RidReservoirStore:
|
||||
col_list = ", ".join(cols)
|
||||
params = ", ".join(f":{c}" for c in cols)
|
||||
conflict = ", ".join(key_cols)
|
||||
if self.db_type == "sqlite":
|
||||
if not preserve_cols:
|
||||
return f"INSERT OR REPLACE INTO {table} ({col_list}) VALUES ({params})"
|
||||
updates = ", ".join(
|
||||
(
|
||||
f"{c} = COALESCE(excluded.{c}, {table}.{c})"
|
||||
if c in preserve_cols
|
||||
else f"{c} = excluded.{c}"
|
||||
)
|
||||
for c in value_cols
|
||||
)
|
||||
return (
|
||||
f"INSERT INTO {table} ({col_list}) VALUES ({params}) "
|
||||
f"ON CONFLICT ({conflict}) DO UPDATE SET {updates}"
|
||||
)
|
||||
if self.db_type == "postgresql":
|
||||
updates = ", ".join(
|
||||
(
|
||||
f"{c} = COALESCE(EXCLUDED.{c}, {table}.{c})"
|
||||
if c in preserve_cols
|
||||
else f"{c} = EXCLUDED.{c}"
|
||||
)
|
||||
for c in value_cols
|
||||
)
|
||||
return (
|
||||
f"INSERT INTO {table} ({col_list}) VALUES ({params}) "
|
||||
f"ON CONFLICT ({conflict}) DO UPDATE SET {updates}"
|
||||
)
|
||||
updates = ", ".join(
|
||||
(
|
||||
f"{c} = COALESCE(VALUES({c}), {c})"
|
||||
f"{c} = COALESCE(EXCLUDED.{c}, {table}.{c})"
|
||||
if c in preserve_cols
|
||||
else f"{c} = VALUES({c})"
|
||||
else f"{c} = EXCLUDED.{c}"
|
||||
)
|
||||
for c in value_cols
|
||||
)
|
||||
return (
|
||||
f"INSERT INTO {table} ({col_list}) VALUES ({params}) "
|
||||
f"ON DUPLICATE KEY UPDATE {updates}"
|
||||
f"ON CONFLICT ({conflict}) DO UPDATE SET {updates}"
|
||||
)
|
||||
|
||||
def save(self, records: List[Dict]) -> int:
|
||||
@@ -447,37 +397,12 @@ class RidReservoirStore:
|
||||
values = conn.execute(text(sql), params).fetchall()
|
||||
dates = set()
|
||||
for (value,) in values:
|
||||
if isinstance(value, str): # sqlite returns ISO strings
|
||||
value = datetime.date.fromisoformat(value[:10])
|
||||
if isinstance(value, datetime.datetime):
|
||||
value = value.date()
|
||||
dates.add(value)
|
||||
return dates
|
||||
|
||||
|
||||
class RidReservoirCollector:
|
||||
"""Fetch + persist the daily dam snapshot (today and yesterday)."""
|
||||
|
||||
def __init__(self, db_config: Dict, client: Optional[RidReservoirClient] = None):
|
||||
self.client = client or RidReservoirClient()
|
||||
self.store = RidReservoirStore(
|
||||
connection_string=db_config["connection_string"],
|
||||
db_type=db_config["type"],
|
||||
)
|
||||
|
||||
def run_cycle(self) -> int:
|
||||
"""Collect today's snapshot plus yesterday's (late daily revisions)."""
|
||||
saved = 0
|
||||
today = datetime.date.today()
|
||||
for date in (None, today - datetime.timedelta(days=1)):
|
||||
try:
|
||||
saved += self.store.save(self.client.fetch_day(date))
|
||||
except Exception as e:
|
||||
logger.error(f"RID reservoir collection failed for {date}: {e}")
|
||||
logger.info(f"RID reservoir collection: {saved} dam-day rows saved")
|
||||
return saved
|
||||
|
||||
|
||||
def backfill(
|
||||
store: RidReservoirStore,
|
||||
start: datetime.date,
|
||||
@@ -616,18 +541,3 @@ def backfill_dam(
|
||||
if chunk_start <= end:
|
||||
time.sleep(throttle_seconds)
|
||||
return total
|
||||
|
||||
|
||||
def create_collector_from_config() -> Optional[RidReservoirCollector]:
|
||||
"""Build a collector from app Config; None when disabled or non-SQL DB."""
|
||||
from .config import Config
|
||||
|
||||
if not Config.ENABLE_RESERVOIR_COLLECTION:
|
||||
return None
|
||||
db_config = Config.get_database_config()
|
||||
if db_config["type"] not in ("sqlite", "postgresql", "mysql"):
|
||||
logger.warning(
|
||||
f"Reservoir collection skipped: DB_TYPE '{db_config['type']}' is not SQL"
|
||||
)
|
||||
return None
|
||||
return RidReservoirCollector(db_config)
|
||||
@@ -1,80 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Pydantic request/response schemas for the water monitoring web API."""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class StationResponse(BaseModel):
|
||||
station_id: int
|
||||
station_code: str
|
||||
thai_name: str
|
||||
english_name: str
|
||||
latitude: Optional[float] = None
|
||||
longitude: Optional[float] = None
|
||||
geohash: Optional[str] = None
|
||||
status: str = "active"
|
||||
|
||||
|
||||
class StationCreateModel(BaseModel):
|
||||
station_code: str = Field(..., description="Station code (e.g., P.1, P.20)")
|
||||
thai_name: str = Field(..., description="Thai name of the station")
|
||||
english_name: str = Field(..., description="English name of the station")
|
||||
latitude: Optional[float] = Field(
|
||||
None, ge=-90, le=90, description="Latitude coordinate"
|
||||
)
|
||||
longitude: Optional[float] = Field(
|
||||
None, ge=-180, le=180, description="Longitude coordinate"
|
||||
)
|
||||
geohash: Optional[str] = Field(None, description="Geohash for the location")
|
||||
status: str = Field("active", description="Station status")
|
||||
|
||||
|
||||
class StationUpdateModel(BaseModel):
|
||||
thai_name: Optional[str] = Field(None, description="Thai name of the station")
|
||||
english_name: Optional[str] = Field(None, description="English name of the station")
|
||||
latitude: Optional[float] = Field(
|
||||
None, ge=-90, le=90, description="Latitude coordinate"
|
||||
)
|
||||
longitude: Optional[float] = Field(
|
||||
None, ge=-180, le=180, description="Longitude coordinate"
|
||||
)
|
||||
geohash: Optional[str] = Field(None, description="Geohash for the location")
|
||||
status: Optional[str] = Field(None, description="Station status")
|
||||
|
||||
|
||||
class MeasurementResponse(BaseModel):
|
||||
timestamp: datetime
|
||||
station_code: str
|
||||
station_name_en: str
|
||||
station_name_th: str
|
||||
water_level: float
|
||||
discharge: Optional[float] = None
|
||||
discharge_percent: Optional[float] = None
|
||||
status: str = "active"
|
||||
|
||||
|
||||
class HealthResponse(BaseModel):
|
||||
overall_status: str
|
||||
timestamp: str
|
||||
checks: Dict[str, Dict[str, Any]]
|
||||
|
||||
|
||||
class MetricsResponse(BaseModel):
|
||||
counters: Dict[str, float]
|
||||
gauges: Dict[str, float]
|
||||
histograms: Dict[str, Dict[str, float]]
|
||||
|
||||
|
||||
class ScrapingStatusResponse(BaseModel):
|
||||
is_running: bool
|
||||
last_run: Optional[datetime] = None
|
||||
next_run: Optional[datetime] = None
|
||||
total_runs: int = 0
|
||||
successful_runs: int = 0 # cycles that found the current hour published
|
||||
failed_runs: int = 0 # cycles behind (RID late) or failed
|
||||
feed_current: Optional[bool] = None # last cycle had the current hour
|
||||
latest_reading: Optional[datetime] = None # newest RID reading stored
|
||||
readings_saved: int = 0 # readings upserted by the last cycle
|
||||
@@ -1,61 +0,0 @@
|
||||
"""Client for ThaiWater's public water-level sensor feed."""
|
||||
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
class ThaiWaterClient:
|
||||
API_URL = "https://twa-api-public.thaiwater.net/v2/waterlevel"
|
||||
|
||||
def __init__(self, session=None, api_key: Optional[str] = None, timeout: int = 30):
|
||||
self.session = session or requests.Session()
|
||||
self.api_key = api_key
|
||||
self.timeout = timeout
|
||||
|
||||
def fetch_ping_sensors(self) -> List[Dict]:
|
||||
if not self.api_key:
|
||||
raise RuntimeError("THAIWATER_API_KEY is not configured")
|
||||
|
||||
response = self.session.get(
|
||||
self.API_URL,
|
||||
headers={"Accept-Language": "en", "x-api-key": self.api_key},
|
||||
timeout=self.timeout,
|
||||
)
|
||||
response.raise_for_status()
|
||||
return self._parse_ping_features(response.json())
|
||||
|
||||
@staticmethod
|
||||
def _parse_ping_features(payload: Dict) -> List[Dict]:
|
||||
sensors = []
|
||||
for collection in payload.get("data", {}).values():
|
||||
for feature in collection.get("features", []):
|
||||
properties = feature.get("properties") or {}
|
||||
basin = properties.get("basin") or {}
|
||||
if basin.get("basin") != "Ping":
|
||||
continue
|
||||
|
||||
geometry = feature.get("geometry") or {}
|
||||
coordinates = geometry.get("coordinates") or []
|
||||
if len(coordinates) < 2:
|
||||
continue
|
||||
|
||||
station = properties.get("station") or {}
|
||||
station_code = station.get("stationCode", "")
|
||||
sensors.append(
|
||||
{
|
||||
"id": f"thaiwater:{properties.get('id')}",
|
||||
"station_code": station_code.split("-", 1)[-1],
|
||||
"station_name": station.get("station"),
|
||||
"latitude": coordinates[1],
|
||||
"longitude": coordinates[0],
|
||||
"timestamp": properties.get("waterlevelDatetime"),
|
||||
"water_level_msl": properties.get("waterlevelMsl"),
|
||||
"bank_percent": properties.get("storagePercent"),
|
||||
"distance_to_bank": properties.get("diffWlBank"),
|
||||
"river_name": properties.get("riverName"),
|
||||
"agency": (properties.get("agency") or {}).get("agencyShort"),
|
||||
"source": "ThaiWater",
|
||||
}
|
||||
)
|
||||
return sensors
|
||||
Loaded 100 of 126 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user