diff --git a/.env.example b/.env.example index cf53a1f..a8a0b61 100644 --- a/.env.example +++ b/.env.example @@ -1,141 +1,85 @@ -# Northern Thailand Ping River Monitor Configuration -# Copy this file to .env and customize for your environment +# Ping River Monitor configuration: copy to .env and fill in. +# +# Read by ping-collector (the service; rust/collector/src/config.rs) and by the +# Python training tools (src/config.py). Under systemd the variables come from +# EnvironmentFile= (systemd's parser); for manual runs both programs load +# ./.env with python-dotenv's rules. Unset keys take the defaults shown. -# Database Configuration -DB_TYPE=postgresql -# Options: sqlite, mysql, postgresql, influxdb, victoriametrics - -# SQLite Configuration (default) -WATER_DB_PATH=water_levels.db - -# VictoriaMetrics Configuration -VM_HOST=localhost -VM_PORT=8428 -VM_URL= - -# InfluxDB Configuration -INFLUX_HOST=localhost -INFLUX_PORT=8086 -INFLUX_DATABASE=ping_river_monitoring -INFLUX_USERNAME= -INFLUX_PASSWORD= - -# PostgreSQL Configuration (Remote Server) -# Option 1: Full connection string (URL encode special characters in password) -POSTGRES_CONNECTION_STRING=postgresql://username:url_encoded_password@your-postgres-host:5432/water_monitoring - -# Option 2: Individual components (password will be automatically URL encoded) -POSTGRES_HOST=your-postgres-host +# --- Database (PostgreSQL only) --------------------------------------------- +# Either a full connection string (URL-encode special characters in the +# password: : %3A @ %40 # %23 ? %3F & %26 / %2F % %25) ... +POSTGRES_CONNECTION_STRING= +# ... or the parts (the password is encoded for you) +POSTGRES_HOST=localhost POSTGRES_PORT=5432 POSTGRES_DB=water_monitoring -POSTGRES_USER=username -POSTGRES_PASSWORD=your:password@with!special#chars +POSTGRES_USER=postgres +POSTGRES_PASSWORD= -# Examples for connection string: -# - Local: postgresql://postgres:password@localhost:5432/water_monitoring -# - Remote: postgresql://user:pass@192.168.1.100:5432/water_monitoring -# - With special chars: postgresql://user:my%3Apass%40word@host:5432/db -# - With SSL: postgresql://user:pass@host:port/db?sslmode=require -# - Connection pooling: postgresql://user:pass@host:port/db?pool_size=20&max_overflow=0 +# --- What the service runs --------------------------------------------------- +# Website and API: the address to listen on (empty = no website) +WEB_LISTEN=0.0.0.0:8000 +# Forecast + ntfy/Matrix notifications after each collection (false = collect only) +FORECAST_IN_COLLECTOR=true +# Trained models (models/flood_.json.gz from src/ml/export.py) +# MODELS_DIR=models -# Special character URL encoding: -# : → %3A @ → %40 # → %23 ? → %3F & → %26 / → %2F % → %25 - -# MySQL Configuration -MYSQL_CONNECTION_STRING=mysql://user:password@localhost:3306/ping_river_monitoring - -# API Configuration -API_HOST=0.0.0.0 -API_PORT=8000 -API_WORKERS=1 -# Public ThaiWater API key used to add Ping-basin water-level sensors. -THAIWATER_API_KEY= - -# Data Collection Settings -SCRAPING_INTERVAL_HOURS=1 -# RID poll minute past the hour; retry interval while RID is behind +# --- Collection ---------------------------------------------------------------- +# RID is polled this many minutes past each hour and retried every +# RID_RETRY_MINUTES while the current hour is not published yet RID_POLL_OFFSET_MINUTES=5 RID_RETRY_MINUTES=10 -# Days of RID holes the leader re-fetches at startup (0 = off) +# RID reports with holes re-fetched at startup (0 = off); older: `ping-collector fill-gaps all` GAP_FILL_DAYS_ON_START=14 -# false when the Rust ping-collector service collects (rust/collector); -# share the status file between both services for /scraping/status -COLLECTION_ENABLED=true -# SCRAPING_STATUS_FILE=/opt/thailand-water-monitor/collector-status.json -# true: ping-collector also forecasts (models/flood_*.json.gz, rust/forecast) -# and sends the ntfy notifications; this app serves its stored forecast and -# computes itself only when that is older than FORECAST_MAX_AGE_SECONDS. -# Set by scripts/install_collector.sh --cutover-forecast. -FORECAST_IN_COLLECTOR=false -# FORECAST_MAX_AGE_SECONDS=7200 -# MODELS_DIR=models -REQUEST_TIMEOUT=30 -MAX_RETRIES=3 -RETRY_DELAY_SECONDS=60 - -# Data Retention -DATA_RETENTION_DAYS=365 - -# Logging Configuration +# ENABLE_HII_COLLECTION=true +# ENABLE_RESERVOIR_COLLECTION=true +# ENABLE_RAIN_COLLECTION=true +# HII_BASIN_CODE=6 +# REQUEST_TIMEOUT=30 +# Runtime station list written by the admin API; falls back to src/data/stations.json +# STATION_CONFIG_PATH=stations.json LOG_LEVEL=INFO -LOG_FILE=water_monitor.log -# Security (for production) -SECRET_KEY=your-secret-key-here -API_KEY=your-api-key-here - -# Monitoring -ENABLE_METRICS=true -ENABLE_HEALTH_CHECKS=true - -# Geographic Settings -TIMEZONE=Asia/Bangkok -DEFAULT_LATITUDE=18.7875 -DEFAULT_LONGITUDE=99.0045 - -# External Services -NOTIFICATION_EMAIL= -SMTP_SERVER= -SMTP_PORT=587 -SMTP_USERNAME= -SMTP_PASSWORD= - -# Public push notifications via self-hosted ntfy (https://ntfy.sh, single binary). -# Leave NTFY_SERVER empty to disable. Topics published: --warning, -# --danger, -warning, -danger, -p1-outlook, -# -status. See docs/NOTIFICATIONS.md. +# --- Notifications (docs/NOTIFICATIONS.md) ------------------------------------ +# Public push alerts via self-hosted ntfy; empty NTFY_SERVER = off. Topics: +# --warning|danger, -warning|danger, +# -p1-outlook, -status. NTFY_SERVER= -# Where the monitor POSTs (defaults to NTFY_SERVER). Use the local ntfy -# address (loopback or Tailscale IP) so publishing does not depend on -# DNS / the reverse proxy being up. +# Where the service POSTs (default NTFY_SERVER): the local ntfy address, so +# publishing does not depend on DNS or the reverse proxy NTFY_PUBLISH_URL= NTFY_TOPIC_PREFIX=ping NTFY_TOKEN= PUBLIC_URL=https://water.buildfor.life/ -# ping-collector notifier: report RID lagging this many hours while ThaiWater -# telemetry is current (0 = never) +# Report RID lagging this many hours while ThaiWater telemetry is current (0 = never) RID_STALE_ALERT_HOURS=18 - -# Matrix: ping-collector sends the ntfy transitions to this room too -# (docs/MATRIX_QUICK_START.md); off while token or room is empty +# Matrix room for a team: the same transitions (docs/MATRIX_QUICK_START.md); +# off while token or room is empty MATRIX_HOMESERVER=https://matrix.org MATRIX_ACCESS_TOKEN= MATRIX_ROOM_ID= -# Website and API served by ping-collector (rust/collector/src/web): the -# address to listen on; empty = the Python app serves (port 8000). Set by -# scripts/install_collector.sh --cutover-api. -WEB_LISTEN= +# --- Website and API ------------------------------------------------------------- +# X-API-Key for station editing, /config and POST /scrape/trigger (empty = those are off) +ADMIN_API_KEY= +# Comma-separated origins allowed cross-site with credentials (empty: any origin, no credentials) +CORS_ALLOW_ORIGINS= +# ThaiWater API key for /sensors/thaiwater (extra Ping-basin sensors on the map) +THAIWATER_API_KEY= +# Self-hosted Umami: server-side tracking of API calls +# UMAMI_API_URL=https://stats.buildfor.life/api/send +# UMAMI_WEBSITE_ID=00b2be73-8f5f-4400-9029-3be852eb08f7 +# UMAMI_TRACK_API=true +# Response cache TTLs (seconds) +# LATEST_CACHE_TTL_SECONDS=45 +# HII_CACHE_TTL_SECONDS=120 +# HEALTH_CACHE_TTL_SECONDS=30 +# A stored forecast older than this is served marked X-Data-Stale: true +# FORECAST_MAX_AGE_SECONDS=7200 # STATIC_DIR=src/static -# ADMIN_API_KEY= # X-API-Key for station editing, /config, /scrape/trigger +# Only for `ping-collector serve` running separately from `run`: the status file +# SCRAPING_STATUS_FILE=collector-status.json -# Grafana Integration -GRAFANA_URL=http://localhost:3000 - -# Alert Configuration -ALERT_MAX_AGE_HOURS=2 -ALERT_CHECK_INTERVAL_MINUTES=15 - -# Development Settings -DEBUG=false -DEVELOPMENT_MODE=false \ No newline at end of file +# --- Python tools ---------------------------------------------------------------- +# Training/evaluation read the database above; this overrides it for them only +# FLOOD_ML_DB_URL= diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 1b8c30e..2c7f88d 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -1,13 +1,13 @@ name: CI -# What this checks, on every push and PR to master: +# The Python side (src/ml training/evaluation stack, archive backfills), on +# every push and PR to master: # 1. formatting contract (black + isort, config in pyproject.toml) # 2. flake8 hard-error gate (syntax, undefined names) -# 3. the pytest suite (synthetic data, no DB/network; ~1 min) -# Docker build / staging / production / perf jobs from the original template -# were removed: there is no registry, no staging host, and production is a -# systemd unit deployed by `git pull` on the server (docs/FLOOD_FORECASTING.md -# section 6, scripts/install.sh). Re-add a job when the thing it deploys exists. +# 3. the pytest suite: synthetic data, no network; the database tests run +# against a throwaway PostgreSQL service (TEST_DATABASE_URL, as in rust.yml) +# The service itself (ping-collector) is checked by rust.yml. Production is a +# systemd unit deployed by `git pull` + scripts/install.sh on the server. on: push: @@ -15,7 +15,7 @@ on: pull_request: branches: [master] schedule: - # daily, catches dependency drift / upstream API changes in the tests + # daily, catches dependency drift - cron: "0 2 * * *" workflow_dispatch: @@ -41,17 +41,30 @@ jobs: pip install --root-user-action=ignore black==26.5.1 isort==5.12.0 flake8==6.1.0 - name: black - run: black --check --diff src/ *.py + run: black --check --diff src/ scripts/ tests/ rust/forecast/testdata rust/collector/testdata - name: isort - run: isort --check-only --diff src/ *.py + run: isort --check-only --diff src/ scripts/ tests/ - name: flake8 (errors only) - run: flake8 src/ --count --select=E9,F63,F7,F82 --show-source --statistics + run: flake8 src/ scripts/ tests/ --count --select=E9,F63,F7,F82 --show-source --statistics test: name: Test suite runs-on: ubuntu-latest + services: + postgres: + image: postgres:16 + env: + POSTGRES_PASSWORD: ping + POSTGRES_DB: ping + options: >- + --health-cmd pg_isready + --health-interval 5s + --health-timeout 5s + --health-retries 20 + env: + TEST_DATABASE_URL: postgres://postgres:ping@postgres:5432/ping steps: - uses: actions/checkout@v4 @@ -66,10 +79,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip --root-user-action=ignore - pip install --root-user-action=ignore -r requirements.txt - pip install --root-user-action=ignore pytest==9.1.1 pytest-asyncio==0.21.1 + pip install --root-user-action=ignore -r requirements.txt pytest==9.1.1 - name: pytest - env: - DB_TYPE: sqlite - run: pytest -q -p no:cacheprovider + run: pytest -q -p no:cacheprovider -rs diff --git a/.gitea/workflows/docs.yml b/.gitea/workflows/docs.yml index fb5d413..1a802e2 100644 --- a/.gitea/workflows/docs.yml +++ b/.gitea/workflows/docs.yml @@ -1,36 +1,23 @@ name: Docs -# Checks that the documentation the project actually ships stays consistent: -# - every relative link / image path in docs/*.md and README.md resolves -# inside the repo (external URLs are NOT fetched: localhost examples, -# rate-limited hosts and the Tailscale-era links made that gate permanently -# red, and a 200 on a curl --head proves nothing about a doc anyway) -# - the FastAPI app imports and its OpenAPI schema is exportable (that is -# the reference at https://water.buildfor.life/docs) -# The previous Sphinx/apidoc jobs produced artifacts nobody read and were -# removed. Reference docs live in docs/*.md; the public overview is at -# https://buildfor.life/docs/tooling/ping-river-monitor/. +# Every relative link / image path in the shipped Markdown (README, CLAUDE, +# CONTRIBUTING, rust/README, docs/) resolves inside the repo. External URLs +# are NOT fetched: localhost examples and rate-limited hosts made that gate +# permanently red, and a 200 proves nothing about a doc anyway. The API +# reference (/docs, /openapi.json) is generated by ping-collector from its +# route table and tested in rust/collector/src/web/tests.rs. on: push: branches: [master, develop] paths: - - "docs/**" - - "README.md" - - "CONTRIBUTING.md" - - "src/web_api.py" - - "src/schemas.py" + - "**.md" - ".gitea/workflows/docs.yml" pull_request: paths: - - "docs/**" - - "README.md" - - "CONTRIBUTING.md" + - "**.md" workflow_dispatch: -env: - PYTHON_VERSION: "3.11" - jobs: docs: name: Validate documentation @@ -43,7 +30,8 @@ jobs: python3 - <<'PY' import re, sys, pathlib root = pathlib.Path(".") - files = [root / "README.md", root / "CONTRIBUTING.md", *root.glob("docs/**/*.md")] + files = [root / "README.md", root / "CLAUDE.md", root / "CONTRIBUTING.md", + root / "rust" / "README.md", *root.glob("docs/**/*.md")] link = re.compile(r"!?\[[^\]]*\]\(([^)\s]+)(?:\s+\"[^\"]*\")?\)") bad = [] for md in files: @@ -65,35 +53,3 @@ jobs: sys.exit(1) print(f"checked {len(files)} files, all relative links resolve") PY - - - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - cache: pip - cache-dependency-path: requirements.txt - - - name: Install dependencies - run: | - python -m pip install --upgrade pip --root-user-action=ignore - pip install --root-user-action=ignore -r requirements.txt - - - name: OpenAPI schema exports - env: - DB_TYPE: sqlite - run: | - python - <<'PY' - import json - from src.web_api import app - spec = app.openapi() - paths = sorted(spec["paths"]) - required = {"/forecast", "/measurements/latest", "/measurements/history/{station_code}", "/stations", "/api/stats", "/health"} - missing = required - set(paths) - assert not missing, f"documented endpoints missing from the app: {missing}" - json.dump(spec, open("openapi.json", "w"), indent=1) - print(f"{len(paths)} paths; schema written to openapi.json") - PY - - - uses: actions/upload-artifact@v3 - with: - name: openapi-${{ github.run_number }} - path: openapi.json diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml deleted file mode 100644 index 9c13dec..0000000 --- a/.gitea/workflows/release.yml +++ /dev/null @@ -1,314 +0,0 @@ -name: Release - Northern Thailand Ping River Monitor - -on: - push: - tags: - - "v*.*.*" - workflow_dispatch: - inputs: - version: - description: "Release version (e.g., v3.1.3)" - required: true - type: string - -env: - PYTHON_VERSION: "3.11" - REGISTRY: git.b4l.co.th - IMAGE_NAME: b4l/northern-thailand-ping-river-monitor - # GitHub token for better rate limits and authentication - GH_TOKEN: ${{ secrets.GH_TOKEN }} - -jobs: - # Create release - create-release: - name: Create Release - runs-on: ubuntu-latest - outputs: - version: ${{ steps.version.outputs.version }} - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - token: ${{ secrets.GITEA_TOKEN }} - fetch-depth: 0 - - - name: Get version - id: version - run: | - if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then - echo "version=${{ github.event.inputs.version }}" >> $GITHUB_OUTPUT - else - echo "version=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT - fi - - - name: Generate changelog - id: changelog - run: | - # Generate changelog from git commits - echo "## Changes" > CHANGELOG.md - git log --pretty=format:"- %s" $(git describe --tags --abbrev=0 HEAD^)..HEAD >> CHANGELOG.md || echo "- Initial release" >> CHANGELOG.md - echo "" >> CHANGELOG.md - echo "## Docker Images" >> CHANGELOG.md - echo "- \`${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.version.outputs.version }}\`" >> CHANGELOG.md - echo "- \`${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest\`" >> CHANGELOG.md - - - name: Create Release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITEA_TOKEN }} - with: - tag_name: ${{ steps.version.outputs.version }} - release_name: Northern Thailand Ping River Monitor ${{ steps.version.outputs.version }} - body_path: CHANGELOG.md - draft: false - prerelease: false - - # Build and test for release - test-release: - name: Test Release Build - runs-on: ubuntu-latest - needs: create-release - strategy: - matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - token: ${{ secrets.GITEA_TOKEN }} - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip --root-user-action=ignore - pip install --root-user-action=ignore -r requirements.txt - pip install --root-user-action=ignore -r requirements-dev.txt - - - name: Run full test suite - run: | - python tests/test_integration.py - python tests/test_station_management.py - python run.py --test - - - name: Build Python package - run: | - pip install --root-user-action=ignore build - python -m build - - - name: Upload Python package - uses: actions/upload-artifact@v3 - with: - name: python-package-${{ matrix.python-version }} - path: dist/ - - # Build release Docker images - build-release: - name: Build Release Images - runs-on: ubuntu-latest - needs: [create-release, test-release] - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - token: ${{ secrets.GITEA_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to Container Registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ vars.WORKER_USERNAME}} - password: ${{ secrets.CI_BOT_TOKEN }} - - - name: Build and push release images - uses: docker/build-push-action@v5 - with: - context: . - platforms: linux/amd64,linux/arm64 - push: true - tags: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.create-release.outputs.version }} - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest - labels: | - org.opencontainers.image.title=Northern Thailand Ping River Monitor - org.opencontainers.image.description=Real-time water level monitoring for Ping River Basin - org.opencontainers.image.version=${{ needs.create-release.outputs.version }} - org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} - org.opencontainers.image.revision=${{ github.sha }} - cache-from: type=gha - cache-to: type=gha,mode=max - env: - GITHUB_TOKEN: ${{ secrets.GITEA_TOKEN }} - - # Security scan for release - security-scan: - name: Security Scan - runs-on: ubuntu-latest - needs: build-release - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - token: ${{ secrets.GITEA_TOKEN}} - - # Test release deployment locally - deploy-release: - name: Test Release Deployment - runs-on: ubuntu-latest - needs: [create-release, build-release, security-scan] - environment: - name: testing - url: http://localhost:8080 - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - token: ${{ secrets.GITEA_TOKEN }} - - - name: Log in to Container Registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ vars.WORKER_USERNAME}} - password: ${{ secrets.CI_BOT_TOKEN }} - - - name: Deploy to production (Local Test) - run: | - set -euo pipefail - echo "🚀 Testing ${{ needs.create-release.outputs.version }} deployment locally..." - - # Create a dedicated network so we can resolve by container name - docker network create ci_net || true - - # Pull the built image - docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.create-release.outputs.version }} - - # Stop & remove any existing container - docker rm -f ping-river-monitor-test 2>/dev/null || true - - # Start the container on the user-defined network - docker run -d \ - --name ping-river-monitor-test \ - --network ci_net \ - -p 8080:8000 \ - -e LOG_LEVEL=INFO \ - -e DB_TYPE=sqlite \ - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.create-release.outputs.version }} - - echo "✅ Container started for testing" - - - name: Health check after deployment - run: | - set -euo pipefail - echo "⏳ Waiting for application to start..." - - # Pull a curl-only image for probing (keeps your app image slim) - docker pull curlimages/curl:8.10.1 - - # Helper: curl via a sibling container on the SAME Docker network - probe() { - local url="$1" - docker run --rm --network ci_net curlimages/curl:8.10.1 \ - -sS --max-time 5 --connect-timeout 3 -w "HTTP_CODE:%{http_code}" "$url" || true - } - - # Wait for /health (up to ~3m 45s) - for i in {1..15}; do - echo "🔍 Attempt $i/15: checking http://ping-river-monitor-test:8000/health" - resp="$(probe http://ping-river-monitor-test:8000/health)" - code="$(echo "$resp" | sed -n 's/.*HTTP_CODE:\([0-9]\+\).*/\1/p')" - body="$(echo "$resp" | sed 's/HTTP_CODE:[0-9]*$//')" - - echo "HTTP: ${code:-} | Body: ${body:-}" - - if [ "${code:-}" = "200" ] && [ -n "${body:-}" ]; then - echo "✅ Health endpoint responding successfully" - break - fi - - echo "❌ Not ready yet. Showing recent logs…" - docker logs --tail 20 ping-river-monitor-test || true - sleep 15 - - if [ "$i" -eq 15 ]; then - echo "❌ Health never reached 200. Failing." - exit 1 - fi - done - - echo "🧪 Testing API endpoints…" - endpoints=("health" "docs" "stations" "metrics") - for ep in "${endpoints[@]}"; do - url="http://ping-river-monitor-test:8000/$ep" - resp="$(probe "$url")" - code="$(echo "$resp" | sed -n 's/.*HTTP_CODE:\([0-9]\+\).*/\1/p')" - - if [ "${code:-}" = "200" ]; then - echo "✅ /$ep: OK" - else - echo "❌ /$ep: FAILED (HTTP ${code:-})" - echo "Response: $(echo "$resp" | sed 's/HTTP_CODE:[0-9]*$//')" - exit 1 - fi - done - - echo "✅ All health checks passed!" - - - name: Container logs and cleanup - if: always() - run: | - echo "📋 Container logs:" - docker logs ping-river-monitor-test || true - - echo "🧹 Cleaning up test container..." - docker stop ping-river-monitor-test || true - docker rm ping-river-monitor-test || true - - echo "📊 Deployment Test Summary:" - echo "Version: ${{ needs.create-release.outputs.version }}" - echo "Image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.create-release.outputs.version }}" - echo "Status: Container tested successfully" - echo "Ready for production deployment" - - # Notify stakeholders - notify: - name: Notify Release - runs-on: ubuntu-latest - needs: [create-release, deploy-release] - if: always() - - steps: - - name: Notify success - if: needs.deploy-release.result == 'success' - run: | - echo "🎉 Release ${{ needs.create-release.outputs.version }} tested successfully!" - echo "🧪 Local Test: Passed all health checks" - echo "� GDocker Image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.create-release.outputs.version }}" - echo "✅ Ready for production deployment" - - # Add notification to Slack, Discord, email, etc. - # curl -X POST -H 'Content-type: application/json' \ - # --data '{"text":"🎉 Northern Thailand Ping River Monitor ${{ needs.create-release.outputs.version }} tested and ready for deployment!"}' \ - # ${{ secrets.SLACK_WEBHOOK_URL }} - - - name: Notify failure - if: needs.deploy-release.result == 'failure' - run: | - echo "❌ Release ${{ needs.create-release.outputs.version }} testing failed!" - echo "Please check the logs and fix issues before production deployment." - - # Add failure notification - # curl -X POST -H 'Content-type: application/json' \ - # --data '{"text":"❌ Northern Thailand Ping River Monitor ${{ needs.create-release.outputs.version }} testing failed!"}' \ - # ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/CLAUDE.md b/CLAUDE.md index 9b44b34..52a96ae 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,16 +5,20 @@ Guidance for AI coding agents working in this repository. ## What this is Flood monitoring and forecasting for the Ping River, Chiang Mai. Public dashboard and -API at https://water.buildfor.life/ (never publish the server's private/Tailscale IP). -Production: one systemd unit on a small VPS, `/opt/thailand-water-monitor`, user -`water-monitor`, interpreter `.venv/bin/python` (uv-managed), updated by `git pull`. +API at https://water.buildfor.life/ (never publish the server's private/mesh IP). +Production: one systemd unit, `ping-collector.service` (the Rust binary in `rust/`), +on a small VPS in `/opt/thailand-water-monitor` (a git checkout), user `water-monitor`; +plus the monthly `water-monitor-retrain.timer` (Python, `.venv`, uv-managed). Updated by +`git pull`, `cargo build --release`, `sudo bash scripts/install.sh`. ## Rules - Python 3.11 only. `uv sync --python 3.11`; run everything as `uv run ...`. -- `make format` (black 88 / isort black profile, config in pyproject.toml) before - committing; CI fails on formatting. `make test` must stay green — tests are - synthetic-data only, never add one that needs the DB or network. +- `make format` (black 88 / isort black profile, config in pyproject.toml; cargo fmt) + before committing; CI fails on formatting. `make test` must stay green. Tests use + synthetic data, never the network; database tests (both languages) run against a + throwaway PostgreSQL schema from `TEST_DATABASE_URL` and skip without it. +- PostgreSQL only. No other database backends. - Timestamps everywhere are Asia/Bangkok wall-clock with no offset. The dashboard parses them with `parseTs()` and renders with `timeZone: TZ`; keep it that way. - Model changes go through the rolling-origin harness (`scripts/evaluate_variants.py`) @@ -27,37 +31,41 @@ Production: one systemd unit on a small VPS, `/opt/thailand-water-monitor`, user before. Stage files by name. - Do not add Co-Authored-By trailers. - The dashboard is a single file, `src/static/dashboard.html`, EN + TH via the `t()` - table: every user-visible string needs both languages. + table: every user-visible string needs both languages. Every path it fetches must be + a route in `rust/collector/src/web/mod.rs` (tests/test_dashboard.py checks). - Rust (`rust/`, see rust/README.md): `cargo fmt --all`, `cargo clippy --all-targets - -- -D warnings`, `cargo test` (database tests need `TEST_DATABASE_URL`). The Rust - collector writes the same tables as the Python collectors: a change to a table, - upsert or parser on one side must be made on the other until the Python side is - retired (check with a live run of both into two databases). -- The forecast is served by Rust (`rust/forecast`) from models trained in Python. - A change to `src/ml/features.py`, `predict.py`, the HII fill or the dam frame - must be ported in the same change: regenerate - `rust/forecast/testdata/gen_forecast_fixtures.py` and keep - `cargo test -p ping-forecast` green. Changing thresholds, stages, leads or - notify constants: `uv run python -m src.ml.export --config` (a test fails - otherwise) and rebuild the collector. Notify texts: - `rust/collector/testdata/gen_notify_cases.py`. -- API routes live in `rust/collector/src/web` (phase 3). Until the FastAPI app is - removed, a route change goes into both, and `scripts/api_parity.py` (both - servers on one database) must report no differences. A route's parameters are - declared once (`web/params.rs`); that list validates and documents (/docs). + -- -D warnings`, `cargo test`; minimum version 1.88 + (`CARGO_TARGET_DIR=target/msrv cargo +1.88 check --all-targets --locked`). Read + `.env` only through `rust/collector/src/envfile.rs` (python-dotenv semantics, never + dotenvy: the Python tools read the same file). +- The forecast is served by Rust (`rust/forecast`) from models trained in Python; + `src/ml/predict.py` is the reference it is tested against. A change to + `src/ml/features.py`, `predict.py`, the HII fill or the dam frame must be ported in + the same change: regenerate `rust/forecast/testdata/gen_forecast_fixtures.py` and keep + `cargo test -p ping-forecast` green. Changing thresholds, stages, leads or the + notifier settings (`src/ml/export.py`): `uv run python -m src.ml.export --config` + (a test fails otherwise) and rebuild the collector. +- Notifier texts are pinned by `rust/collector/testdata/notify_python_cases.json.gz`; + a deliberate wording change updates that fixture in the same commit. +- API routes live in `rust/collector/src/web`. A route's parameters are declared once + (`web/params.rs`); that list validates and documents (/docs). Keep response formats + stable: external clients use them. +- The Python backfill stores (`src/hii_collector.py`, `src/rid_reservoir.py`, + `src/ml/rain.py`) write the same tables as `rust/collector/src/db.rs`: a table or + upsert change goes into both. ## Where things are -- `src/web_api.py` FastAPI app; `src/water_scraper_v3.py` RID collector; - `src/hii_collector.py` ThaiWater/HII; `src/ml/` features/train/evaluate/predict, - `rain.py` (Open-Meteo), `dam.py`, `hii_rain.py`. -- `scripts/retrain.sh` + `water-monitor-retrain.timer`: monthly retrain with staged - promote. `scripts/dev_proxy.py`: serve the working-copy dashboard against the live API. -- `rust/collector` (`ping-collector`): Rust data collector; with it deployed the - Python app runs `COLLECTION_ENABLED=false`. With `FORECAST_IN_COLLECTOR=true` it - also forecasts (`rust/forecast`) and sends ntfy + Matrix; with `WEB_LISTEN` it - serves the website and API (`rust/collector/src/web`, the port of - `src/web_api.py`; the dashboard stays `src/static/dashboard.html`). Rewrite - plan: `docs/RUST_REWRITE.md`. +- `rust/collector` (`ping-collector`): collection (`rid.rs`, `hii.rs`, `dams.rs`, + `rain.rs`), forecast run (`forecasting.rs`), ntfy/Matrix (`notify.rs`), website/API + (`web/`), tables (`db.rs`). `rust/forecast`: inference. +- `src/ml/`: features/train/evaluate/predict/export, `rain.py` (Open-Meteo), `dam.py`, + `hii_rain.py`. `src/hii_backfill.py`, `src/hii_collector.py`, `src/rid_reservoir.py`: + archive backfills (`scripts/backfill_*.py`). +- `scripts/install.sh`: install/update on the server. `scripts/retrain.sh` + + `water-monitor-retrain.timer`: monthly retrain with staged promote. + `scripts/forecast_parity.py`: Rust vs Python forecast on a live database. + `scripts/dev_proxy.py`: serve the working-copy dashboard against the live API. - `docs/FLOOD_FORECASTING.md` is the authoritative model write-up; `docs/DATA_SOURCES.md` - the source catalog. + the source catalog; `docs/NOTIFICATIONS.md` the alerts; `docs/RUST_REWRITE.md` the + history of the Python-to-Rust move. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5a4fab3..1593e94 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,277 +1,51 @@ -# Contributing to Northern Thailand Ping River Monitor +# Contributing -Thank you for your interest in contributing to the Northern Thailand Ping River Monitor! This document provides guidelines and information for contributors. +Issues and merge requests on +[git.b4l.co.th](https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor). +The repository has two halves: the service in Rust (`rust/`, see +[rust/README.md](rust/README.md)) and the model training/evaluation stack in Python +(`src/ml`, see [docs/FLOOD_FORECASTING.md](docs/FLOOD_FORECASTING.md)). -## 🚀 Getting Started +## Setup -### Prerequisites -- Python 3.9 or higher -- Git -- Basic knowledge of water monitoring systems -- Familiarity with FastAPI and time-series databases (optional) +- Python 3.11 and [uv](https://docs.astral.sh/uv/) (3.12+ has no pandas 2.0.3 wheels) +- Rust >= 1.88 from [rustup](https://rustup.rs), plus `pkg-config` and `libssl-dev` on Linux +- Docker (or any PostgreSQL 16) for the database tests -### Development Setup +```bash +git clone https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor.git +cd Northern-Thailand-Ping-River-Monitor +make install # uv sync + pre-commit hooks +docker run -d --name ping-pg -e POSTGRES_PASSWORD=ping -e POSTGRES_DB=ping -p 55432:5432 postgres:16 +export TEST_DATABASE_URL=postgres://postgres:ping@localhost:55432/ping +make check # format check, flake8, clippy, pytest, cargo test, Rust 1.88 +``` -1. **Clone the repository:** - ```bash - git clone https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor.git - cd Northern-Thailand-Ping-River-Monitor - ``` +Without `TEST_DATABASE_URL` the database tests are skipped, the rest still run. -2. **Create a virtual environment:** - ```bash - python -m venv venv - source venv/bin/activate # On Windows: venv\Scripts\activate - ``` +## Before a merge request -3. **Install dependencies:** - ```bash - pip install -r requirements.txt - ``` +- `make format`, then `make check` green. CI runs the same (`.gitea/workflows/`). +- One logical change per commit, imperative subject (`fix(collector): ...`, + `feat(web): ...`, `docs: ...`). +- Dashboard changes (`src/static/dashboard.html`): every visible string in English + and Thai (`t()` table); a screenshot in the merge request. +- API changes: keep existing fields and formats; external clients read them. New + parameters go into the route's declaration in `rust/collector/src/web/params.rs`, + which also documents them at `/docs`. +- Model changes: evaluate with `scripts/evaluate_variants.py` (rolling origin, judged + on first-alert lead and false alarms) and record the result in + `docs/FLOOD_FORECASTING.md` section 5, whether it helped or not. A feature change + must be ported to `rust/forecast` in the same change (see CLAUDE.md for the + fixture and config regeneration steps). +- Never commit credentials, `.env`, trained models (`models/*.joblib`, + `models/*.json.gz`) or the server's private addresses. -4. **Run tests:** - ```bash - python test_integration.py - ``` +## Useful commands -5. **Start the development server:** - ```bash - python run.py --web-api - ``` - -## 🔧 Development Guidelines - -### Code Style -- Follow PEP 8 Python style guidelines -- Use type hints for all function parameters and return values -- Write descriptive docstrings for all classes and functions -- Keep functions focused and under 50 lines when possible - -### Testing -- Write tests for new features -- Ensure all existing tests pass before submitting -- Test both success and error cases -- Include integration tests for API endpoints - -### Documentation -- Update README.md for new features -- Add docstrings to all new functions and classes -- Update API documentation for new endpoints -- Include examples in documentation - -## 📝 Contribution Process - -### 1. Issue Creation -- Check existing issues before creating new ones -- Use clear, descriptive titles -- Provide detailed descriptions with steps to reproduce (for bugs) -- Include system information and logs when relevant - -### 2. Pull Request Process -1. **Fork the repository** -2. **Create a feature branch:** - ```bash - git checkout -b feature/your-feature-name - ``` -3. **Make your changes** -4. **Test thoroughly:** - ```bash - python test_integration.py - python test_station_management.py - ``` -5. **Commit with clear messages:** - ```bash - git commit -m "Add: New station validation feature" - ``` -6. **Push to your fork:** - ```bash - git push origin feature/your-feature-name - ``` -7. **Create a pull request** - -### Pull Request Guidelines -- Use clear, descriptive titles -- Provide detailed descriptions of changes -- Reference related issues -- Include screenshots for UI changes -- Ensure all tests pass -- Update documentation as needed - -## 🎯 Areas for Contribution - -### High Priority -- **Additional Database Support** - MongoDB, TimescaleDB -- **Authentication System** - User management and API keys -- **Real-time Alerts** - Email/SMS notifications for critical conditions -- **Mobile App** - React Native or Flutter interface -- **Advanced Analytics** - Trend analysis and forecasting - -### Medium Priority -- **Bulk Station Operations** - Import/export multiple stations -- **Map Interface** - Visual station management -- **Data Export** - CSV, Excel format support -- **Performance Optimization** - Query optimization and caching -- **Internationalization** - Multi-language support - -### Low Priority -- **Additional Visualizations** - Custom chart types -- **Report Generation** - Automated PDF reports -- **Integration APIs** - Third-party service connections -- **Advanced Logging** - Structured logging with ELK stack - -## 🐛 Bug Reports - -When reporting bugs, please include: - -1. **Environment Information:** - - Operating system and version - - Python version - - Database type and version - - Browser (for web interface issues) - -2. **Steps to Reproduce:** - - Clear, numbered steps - - Expected vs actual behavior - - Screenshots or error messages - -3. **Additional Context:** - - Log files (sanitized of sensitive data) - - Configuration details - - Recent changes to the system - -## 💡 Feature Requests - -For feature requests, please provide: - -1. **Use Case:** Why is this feature needed? -2. **Description:** What should the feature do? -3. **Acceptance Criteria:** How do we know it's complete? -4. **Priority:** How important is this feature? -5. **Alternatives:** Are there workarounds available? - -## 🔒 Security - -### Reporting Security Issues -- **DO NOT** create public issues for security vulnerabilities -- Email security issues to: [security contact] -- Include detailed information about the vulnerability -- Allow time for fixes before public disclosure - -### Security Guidelines -- Never commit sensitive data (passwords, API keys) -- Use environment variables for configuration -- Validate all user inputs -- Follow OWASP security guidelines -- Keep dependencies updated - -## 📊 Database Contributions - -### Adding New Database Support -1. Create adapter in `src/database_adapters.py` -2. Implement all required methods -3. Add configuration options in `src/config.py` -4. Update documentation -5. Add integration tests - -### Database Schema Changes -- Provide migration scripts -- Test with existing data -- Document breaking changes -- Update all database adapters - -## 🌐 API Contributions - -### Adding New Endpoints -1. Define Pydantic models in `src/models.py` -2. Implement endpoint in `src/web_api.py` -3. Add input validation -4. Include error handling -5. Write comprehensive tests -6. Update API documentation - -### API Guidelines -- Use RESTful conventions -- Include proper HTTP status codes -- Provide clear error messages -- Support pagination for large datasets -- Include rate limiting considerations - -## 📈 Performance Guidelines - -### Optimization Areas -- Database query efficiency -- API response times -- Memory usage -- Concurrent request handling -- Data processing speed - -### Monitoring -- Use built-in metrics collection -- Monitor response times -- Track error rates -- Watch resource usage -- Set up alerts for issues - -## 🧪 Testing Guidelines - -### Test Types -- **Unit Tests:** Individual function testing -- **Integration Tests:** Component interaction testing -- **API Tests:** Endpoint functionality testing -- **Performance Tests:** Load and stress testing - -### Test Coverage -- Aim for >80% code coverage -- Test both success and failure paths -- Include edge cases -- Test with different database backends - -## 📚 Documentation Standards - -### Code Documentation -- Use clear, descriptive variable names -- Write comprehensive docstrings -- Include type hints -- Add inline comments for complex logic - -### User Documentation -- Keep README.md updated -- Provide clear installation instructions -- Include usage examples -- Document configuration options - -## 🤝 Community Guidelines - -### Communication -- Be respectful and professional -- Help newcomers get started -- Share knowledge and experiences -- Provide constructive feedback - -### Code Reviews -- Review code thoroughly -- Provide specific, actionable feedback -- Be open to suggestions -- Focus on code quality and maintainability - -## 📞 Getting Help - -### Resources -- **Documentation:** README.md and docs/ -- **API Reference:** http://localhost:8000/docs -- **Issues:** GitHub Issues section -- **Discussions:** GitHub Discussions - -### Contact -- **General Questions:** Create a GitHub issue -- **Security Issues:** [security contact] -- **Maintainer:** [maintainer contact] - -## 🎉 Recognition - -Contributors will be recognized in: -- README.md contributors section -- Release notes for significant contributions -- Special recognition for major features - -Thank you for contributing to the Northern Thailand Ping River Monitor! Your contributions help improve water resource monitoring and management in the region. \ No newline at end of file +```bash +make build && make serve # the website/API from your database on 127.0.0.1:8001 +uv run python scripts/dev_proxy.py # the working-copy dashboard against the live API +uv run python scripts/evaluate_variants.py --stations P.1 +uv run python -m src.ml.export --config # after changing thresholds or notifier settings +``` diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index d498e29..0000000 --- a/Dockerfile +++ /dev/null @@ -1,58 +0,0 @@ -# Multi-stage Docker build for Northern Thailand Ping River Monitor - -# Build stage -FROM python:3.11-slim as builder - -# Set working directory -WORKDIR /app - -# Install build dependencies -RUN apt-get update && apt-get install -y \ - build-essential \ - gcc \ - && rm -rf /var/lib/apt/lists/* - -# Copy requirements and install Python dependencies -COPY requirements.txt . -RUN pip install --no-cache-dir --user -r requirements.txt - -# Production stage -FROM python:3.11-slim - -# Set working directory -WORKDIR /app - -# Install runtime dependencies and create user -RUN apt-get update && apt-get install -y \ - wget \ - curl \ - && rm -rf /var/lib/apt/lists/* \ - && groupadd -r appuser && useradd -r -g appuser appuser \ - && mkdir -p /home/appuser/.local - -# Copy Python packages from builder stage -COPY --from=builder /root/.local /home/appuser/.local - -# Copy application code -COPY . . - -# Create logs directory and set permissions -RUN mkdir -p logs && chown -R appuser:appuser /app /home/appuser/.local - -# Set environment variables -ENV PYTHONUNBUFFERED=1 -ENV TZ=Asia/Bangkok -ENV PATH=/home/appuser/.local/bin:$PATH - -# Switch to non-root user -USER appuser - -# Health check for API mode -HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ - CMD python -c "import requests; requests.get('http://localhost:8000/health', timeout=5)" 2>/dev/null || exit 1 - -# Expose port for API -EXPOSE 8000 - -# Default command -CMD ["python", "run.py", "--web-api"] diff --git a/MIGRATION_TO_UV.md b/MIGRATION_TO_UV.md deleted file mode 100644 index f6653e5..0000000 --- a/MIGRATION_TO_UV.md +++ /dev/null @@ -1,165 +0,0 @@ -# Migration to uv - -This document describes the migration from traditional Python package management (pip + requirements.txt) to [uv](https://docs.astral.sh/uv/), a fast Python package installer and resolver. - -## What Changed - -### Files Added -- `pyproject.toml` - Modern Python project configuration combining dependencies and metadata -- `.python-version` - Specifies Python version for uv -- `scripts/setup_uv.sh` - Unix setup script for uv environment -- `scripts/setup_uv.bat` - Windows setup script for uv environment -- This migration guide - -### Files Modified -- `Makefile` - Updated all commands to use `uv run` instead of direct Python execution - -### Files That Can Be Removed (Optional) -- `requirements.txt` - Dependencies now in pyproject.toml -- `requirements-dev.txt` - Dev dependencies now in pyproject.toml -- `setup.py` - Configuration now in pyproject.toml - -## Installation - -### Install uv - -**Unix/macOS:** -```bash -curl -LsSf https://astral.sh/uv/install.sh | sh -``` - -**Windows (PowerShell):** -```powershell -powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex" -``` - -### Setup Project - -**Unix/macOS:** -```bash -# Run the setup script -chmod +x scripts/setup_uv.sh -./scripts/setup_uv.sh - -# Or manually: -uv sync -uv run pre-commit install -``` - -**Windows:** -```batch -REM Run the setup script -scripts\setup_uv.bat - -REM Or manually: -uv sync -uv run pre-commit install -``` - -## New Workflow - -### Common Commands - -| Old Command | New Command | Description | -|-------------|-------------|-------------| -| `pip install -r requirements.txt` | `uv sync --no-dev` | Install production dependencies | -| `pip install -r requirements-dev.txt` | `uv sync` | Install all dependencies (including dev) | -| `python run.py` | `uv run python run.py` | Run the application | -| `pytest` | `uv run pytest` | Run tests | -| `black src/` | `uv run black src/` | Format code | - -### Using the Makefile - -The Makefile has been updated to use uv, so all existing commands work the same: - -```bash -make install-dev # Install dev dependencies with uv -make test # Run tests with uv -make run-api # Start API server with uv -make lint # Lint code with uv -make format # Format code with uv -``` - -### Adding Dependencies - -**Production dependency:** -```bash -uv add requests -``` - -**Development dependency:** -```bash -uv add --dev pytest -``` - -**Specific version:** -```bash -uv add "fastapi==0.104.1" -``` - -### Managing Python Versions - -uv can automatically manage Python versions: - -```bash -# Install and use Python 3.11 -uv python install 3.11 -uv sync - -# Use specific Python version -uv sync --python 3.11 -``` - -## Benefits of uv - -1. **Speed** - 10-100x faster than pip -2. **Reliability** - Better dependency resolution -3. **Simplicity** - Single tool for packages and Python versions -4. **Reproducibility** - Lock file ensures consistent environments -5. **Modern** - Built-in support for pyproject.toml - -## Troubleshooting - -### Command not found -Make sure uv is in your PATH after installation. Restart your terminal or run: -```bash -source ~/.bashrc # or ~/.zshrc -``` - -### Lock file conflicts -If you encounter lock file issues: -```bash -rm uv.lock -uv sync -``` - -### Python version issues -Ensure the Python version in `.python-version` is available: -```bash -uv python list -uv python install 3.11 # if needed -``` - -## Rollback (if needed) - -If you need to rollback to the old system: - -1. Use the original requirements files: - ```bash - pip install -r requirements.txt - pip install -r requirements-dev.txt - ``` - -2. Revert the Makefile changes to use `python` instead of `uv run python` - -3. Remove uv-specific files: - ```bash - rm pyproject.toml .python-version uv.lock - rm -rf .venv # if created by uv - ``` - -## Additional Resources - -- [uv Documentation](https://docs.astral.sh/uv/) -- [Migration Guide](https://docs.astral.sh/uv/guides/projects/) -- [pyproject.toml Reference](https://packaging.python.org/en/latest/specifications/pyproject-toml/) \ No newline at end of file diff --git a/Makefile b/Makefile index ee6b4a4..81bc2af 100644 --- a/Makefile +++ b/Makefile @@ -1,181 +1,70 @@ -# Northern Thailand Ping River Monitor - Makefile +# Ping River Monitor. The service is Rust (rust/, ping-collector); Python is +# the model training/evaluation stack and the archive backfills (src/). +# Database tests (both sides) need TEST_DATABASE_URL, e.g. +# docker run -d --name ping-pg -e POSTGRES_PASSWORD=ping -e POSTGRES_DB=ping -p 55432:5432 postgres:16 +# export TEST_DATABASE_URL=postgres://postgres:ping@localhost:55432/ping -.PHONY: help install install-dev test test-coverage lint format clean run run-api docker-build docker-run docs +PY_SRC := src/ scripts/ tests/ rust/forecast/testdata rust/collector/testdata +CARGO := cargo +MANIFEST := --manifest-path rust/Cargo.toml + +.PHONY: help install format lint test test-py test-rust build check serve train evaluate export clean -# Default target help: - @echo "Northern Thailand Ping River Monitor - Available Commands:" - @echo "" - @echo "Setup:" - @echo " install Install production dependencies" - @echo " install-dev Install development dependencies" - @echo "" - @echo "Development:" - @echo " test Run all tests" - @echo " test-cov Run tests with coverage report" - @echo " lint Run code linting" - @echo " format Format code with black and isort" - @echo " clean Clean up temporary files" - @echo "" - @echo "Running:" - @echo " run Run the monitor in continuous mode" - @echo " run-api Run the web API server" - @echo " run-test Run a single test cycle" - @echo " run-status Show system status" - @echo "" - @echo "Alerting:" - @echo " alert-check Check water levels and send alerts" - @echo " alert-test Send test Matrix message" - @echo "" - @echo "Distribution:" - @echo " build-exe Build standalone executable" - @echo " package Build and create distribution package" - @echo "" - @echo "Docker:" - @echo " docker-build Build Docker image" - @echo " docker-run Run with Docker Compose" - @echo " docker-stop Stop Docker services" - @echo "" - @echo "Database:" - @echo " setup-postgres Setup PostgreSQL database" - @echo " test-postgres Test PostgreSQL connection" - @echo " encode-password URL encode password for connection string" - @echo " migrate-sqlite Migrate SQLite data to PostgreSQL" - @echo " migrate-fast Fast migration with 10K batch size" - @echo " analyze-sqlite Analyze SQLite database structure (dry run)" - @echo "" - @echo "Documentation:" - @echo " docs Generate documentation" + @echo "make install uv sync (Python 3.11) + pre-commit hooks" + @echo "make format black + isort (Python), cargo fmt (Rust)" + @echo "make lint the CI checks without tests (format, flake8, clippy)" + @echo "make test pytest + cargo test" + @echo "make build release build of ping-collector (rust/target/release)" + @echo "make check lint + test + the Rust 1.88 minimum-version check" + @echo "make serve website/API from the local database on 127.0.0.1:8001" + @echo "make train retrain every station's model into models/" + @echo "make evaluate rolling-origin harness on P.1 (see scripts/evaluate_variants.py)" + @echo "make export export trained models + flood_config.json for the Rust forecaster" -# Installation install: - uv sync --no-dev - -install-dev: - uv sync + uv sync --python 3.11 uv run pre-commit install -# Testing -test: +format: + uv run black $(PY_SRC) + uv run isort src/ scripts/ tests/ + $(CARGO) fmt --all $(MANIFEST) + +lint: + uv run black --check $(PY_SRC) + uv run isort --check-only src/ scripts/ tests/ + uv run flake8 src/ scripts/ tests/ --count --select=E9,F63,F7,F82 --show-source --statistics + $(CARGO) fmt --all --check $(MANIFEST) + $(CARGO) clippy --all-targets --locked $(MANIFEST) -- -D warnings + +test: test-py test-rust + +test-py: uv run pytest -q -test-cov: - uv run pytest --cov=src --cov-report=html --cov-report=term +test-rust: + $(CARGO) test --locked $(MANIFEST) -# Code quality -lint: - uv run flake8 src/ --max-line-length=100 - uv run mypy src/ +build: + $(CARGO) build --release --locked $(MANIFEST) -format: - uv run black src/ *.py - uv run isort src/ *.py +check: lint test + CARGO_TARGET_DIR=rust/target/msrv $(CARGO) +1.88 check --all-targets --locked $(MANIFEST) + +serve: + rust/target/release/ping-collector serve --listen 127.0.0.1:8001 --no-analytics + +train: + uv run python scripts/train_flood_model.py --stations all + +evaluate: + uv run python scripts/evaluate_variants.py --stations P.1 + +export: + uv run python -m src.ml.export + uv run python -m src.ml.export --config -# Cleanup clean: - find . -type f -name "*.pyc" -delete - find . -type d -name "__pycache__" -delete - find . -type f -name "*.log" -delete - rm -rf .pytest_cache/ - rm -rf .mypy_cache/ - rm -rf htmlcov/ - rm -rf dist/ - rm -rf build/ - rm -rf *.egg-info/ - -# Running -run: - uv run python run.py - -run-api: - uv run python run.py --web-api - -run-test: - uv run python run.py --test - -run-status: - uv run python run.py --status - -# Alerting -alert-check: - uv run python run.py --alert-check - -alert-test: - uv run python run.py --alert-test - -# Docker -docker-build: - docker build -t ping-river-monitor . - -docker-run: - docker-compose -f docker-compose.victoriametrics.yml up -d - -docker-stop: - docker-compose -f docker-compose.victoriametrics.yml down - -docker-logs: - docker-compose -f docker-compose.victoriametrics.yml logs -f - -# Documentation -docs: - cd docs && make html - -# Monitoring -health-check: - curl -f http://localhost:8000/health || exit 1 - -metrics: - curl -s http://localhost:8000/metrics | jq . - -# Development helpers -dev-setup: install-dev - cp .env.example .env - @echo "Development environment set up!" - @echo "Edit .env file with your configuration" - -# Production deployment -deploy-check: - uv run python run.py --test - @echo "Deployment check passed!" - -# Database management -setup-postgres: - uv run python scripts/setup_postgres.py - -test-postgres: - uv run python -c "from scripts.setup_postgres import test_postgres_connection; from src.config import Config; config = Config.get_database_config(); test_postgres_connection(config['connection_string'])" - -migrate-sqlite: - uv run python scripts/migrate_sqlite_to_postgres.py - -migrate-fast: - uv run python scripts/migrate_sqlite_to_postgres.py --fast - -analyze-sqlite: - uv run python scripts/migrate_sqlite_to_postgres.py --dry-run - -# Git helpers -git-setup: - git remote add origin https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor.git - @echo "Git remote configured!" - -# Quick start -quick-start: install dev-setup run-test - @echo "Quick start completed!" - @echo "Run 'make run-api' to start the web interface" - -# Gitea Actions -validate-workflows: - @echo "Validating Gitea Actions workflows..." - @for file in .gitea/workflows/*.yml; do \ - echo "Checking $$file..."; \ - uv run python -c "import yaml; yaml.safe_load(open('$$file', encoding='utf-8'))" || exit 1; \ - done - @echo "✅ All workflows are valid" - -workflow-test: - @echo "Testing workflow components locally..." - make test - make lint - make format - @echo "✅ Workflow test completed" \ No newline at end of file + find . -type d -name "__pycache__" -prune -exec rm -rf {} + + rm -rf .pytest_cache/ htmlcov/ dist/ build/ *.egg-info/ diff --git a/POSTGRESQL_SETUP.md b/POSTGRESQL_SETUP.md deleted file mode 100644 index 814abb4..0000000 --- a/POSTGRESQL_SETUP.md +++ /dev/null @@ -1,287 +0,0 @@ -# PostgreSQL Setup for Northern Thailand Ping River Monitor - -This guide helps you configure PostgreSQL as the database backend for the water monitoring system. - -## Prerequisites - -- PostgreSQL server running on a remote machine (already available) -- Network connectivity to the PostgreSQL server -- Database credentials (username, password, host, port) - -## Quick Setup - -### 1. Configure Environment - -Copy the example environment file and configure it: - -```bash -cp .env.example .env -``` - -Edit `.env` and update the PostgreSQL configuration: - -```bash -# Database Configuration -DB_TYPE=postgresql - -# PostgreSQL Configuration (Remote Server) -POSTGRES_CONNECTION_STRING=postgresql://username:password@your-postgres-host:5432/water_monitoring -``` - -### 2. Run Setup Script - -Use the interactive setup script: - -```bash -# Using uv -uv run python scripts/setup_postgres.py - -# Or using make -make setup-postgres -``` - -The script will: -- Test your database connection -- Create the database if it doesn't exist -- Initialize the required tables and indexes -- Set up sample monitoring stations - -### 3. Test Connection - -Test your PostgreSQL connection: - -```bash -make test-postgres -``` - -### 4. Run the Application - -Start collecting data: - -```bash -# Run a test cycle -make run-test - -# Start the web API -make run-api -``` - -## Manual Configuration - -If you prefer manual setup, here's what you need: - -### Connection String Format - -``` -postgresql://username:password@host:port/database -``` - -**Examples:** -- Basic: `postgresql://postgres:mypassword@192.168.1.100:5432/water_monitoring` -- With SSL: `postgresql://user:pass@host:5432/db?sslmode=require` -- With connection pooling: `postgresql://user:pass@host:5432/db?pool_size=20&max_overflow=0` - -### Environment Variables - -| Variable | Description | Example | -|----------|-------------|---------| -| `DB_TYPE` | Database type | `postgresql` | -| `POSTGRES_CONNECTION_STRING` | Full connection string | See above | - -### Database Schema - -The application uses these main tables: - -1. **stations** - Monitoring station information -2. **water_measurements** - Time series water level data -3. **alert_thresholds** - Warning/danger level definitions -4. **data_quality_log** - Data collection issue tracking - -See `sql/init_postgres.sql` for the complete schema. - -## Connection Options - -### SSL Connection - -For secure connections, add SSL parameters: - -```bash -POSTGRES_CONNECTION_STRING=postgresql://user:pass@host:5432/db?sslmode=require -``` - -SSL modes: -- `disable` - No SSL -- `require` - Require SSL -- `prefer` - Use SSL if available -- `verify-ca` - Verify certificate authority -- `verify-full` - Full certificate verification - -### Connection Pooling - -For high-performance applications, configure connection pooling: - -```bash -POSTGRES_CONNECTION_STRING=postgresql://user:pass@host:5432/db?pool_size=20&max_overflow=0 -``` - -Parameters: -- `pool_size` - Number of connections to maintain -- `max_overflow` - Additional connections allowed -- `pool_timeout` - Seconds to wait for connection -- `pool_recycle` - Seconds before connection refresh - -## Troubleshooting - -### Common Issues - -**1. Connection Refused** -``` -psycopg2.OperationalError: could not connect to server -``` -- Check if PostgreSQL server is running -- Verify host/port in connection string -- Check firewall settings - -**2. Authentication Failed** -``` -psycopg2.OperationalError: FATAL: password authentication failed -``` -- Verify username/password in connection string -- Check PostgreSQL pg_hba.conf configuration -- Ensure user has database access permissions - -**3. Database Does Not Exist** -``` -psycopg2.OperationalError: FATAL: database "water_monitoring" does not exist -``` -- Run the setup script to create the database -- Or manually create: `CREATE DATABASE water_monitoring;` - -**4. Permission Denied** -``` -psycopg2.ProgrammingError: permission denied for table -``` -- Ensure user has appropriate permissions -- Grant access: `GRANT ALL PRIVILEGES ON DATABASE water_monitoring TO username;` - -### Network Configuration - -For remote PostgreSQL servers, ensure: - -1. **PostgreSQL allows remote connections** (`postgresql.conf`): - ``` - listen_addresses = '*' - port = 5432 - ``` - -2. **Client authentication is configured** (`pg_hba.conf`): - ``` - # Allow connections from your application server - host water_monitoring username your.app.ip/32 md5 - ``` - -3. **Firewall allows PostgreSQL port**: - ```bash - # On PostgreSQL server - sudo ufw allow 5432/tcp - ``` - -### Performance Tuning - -For optimal performance with time series data: - -1. **Increase work_mem** for sorting operations -2. **Tune shared_buffers** for caching -3. **Configure maintenance_work_mem** for indexing -4. **Set up regular VACUUM and ANALYZE** for statistics - -Example PostgreSQL configuration additions: -``` -# postgresql.conf -shared_buffers = 256MB -work_mem = 16MB -maintenance_work_mem = 256MB -effective_cache_size = 1GB -``` - -## Monitoring - -### Check Application Status - -```bash -# View current configuration -uv run python -c "from src.config import Config; Config.print_settings()" - -# Test database connection -make test-postgres - -# Check latest data -psql "postgresql://user:pass@host:5432/water_monitoring" -c "SELECT COUNT(*) FROM water_measurements;" -``` - -### PostgreSQL Monitoring - -Connect directly to check database status: - -```bash -# Connect to database -psql "postgresql://username:password@host:5432/water_monitoring" - -# Check table sizes -\dt+ - -# View latest measurements -SELECT * FROM latest_measurements LIMIT 10; - -# Check data quality -SELECT issue_type, COUNT(*) FROM data_quality_log -WHERE created_at > NOW() - INTERVAL '24 hours' -GROUP BY issue_type; -``` - -## Backup and Maintenance - -### Backup Database - -```bash -# Full backup -pg_dump "postgresql://user:pass@host:5432/water_monitoring" > backup.sql - -# Data only -pg_dump --data-only "postgresql://user:pass@host:5432/water_monitoring" > data_backup.sql -``` - -### Restore Database - -```bash -# Restore full backup -psql "postgresql://user:pass@host:5432/water_monitoring" < backup.sql - -# Restore data only -psql "postgresql://user:pass@host:5432/water_monitoring" < data_backup.sql -``` - -### Regular Maintenance - -Set up regular maintenance tasks: - -```sql --- Update table statistics (run weekly) -ANALYZE; - --- Reclaim disk space (run monthly) -VACUUM; - --- Reindex tables (run quarterly) -REINDEX DATABASE water_monitoring; -``` - -## Next Steps - -1. Set up monitoring and alerting -2. Configure data retention policies -3. Set up automated backups -4. Implement connection pooling if needed -5. Configure SSL for production use - -For more advanced configuration, see the [PostgreSQL documentation](https://www.postgresql.org/docs/). \ No newline at end of file diff --git a/SQLITE_MIGRATION.md b/SQLITE_MIGRATION.md deleted file mode 100644 index d49d074..0000000 --- a/SQLITE_MIGRATION.md +++ /dev/null @@ -1,278 +0,0 @@ -# SQLite to PostgreSQL Migration Guide - -This guide helps you migrate your existing SQLite water monitoring data to PostgreSQL. - -## Quick Migration - -### 1. Analyze Your SQLite Database (Optional) - -First, check what's in your SQLite database: - -```bash -# Analyze without migrating -make analyze-sqlite - -# Or specify a specific SQLite file -uv run python scripts/migrate_sqlite_to_postgres.py --dry-run /path/to/your/database.db -``` - -### 2. Run the Migration - -```bash -# Auto-detect SQLite file and migrate -make migrate-sqlite - -# Or specify a specific SQLite file -uv run python scripts/migrate_sqlite_to_postgres.py /path/to/your/database.db -``` - -The migration tool will: -- ✅ Connect to both databases -- ✅ Analyze your SQLite schema automatically -- ✅ Migrate station information -- ✅ Migrate all measurement data in batches -- ✅ Handle different SQLite table structures -- ✅ Verify the migration results -- ✅ Generate a detailed log file - -## What Gets Migrated - -### Station Data -- Station IDs and codes -- Thai and English names -- Coordinates (latitude/longitude) -- Geohash data (if available) -- Creation/update timestamps - -### Measurement Data -- Water level readings -- Discharge measurements -- Discharge percentages -- Timestamps -- Station associations -- Data quality status - -## Supported SQLite Schemas - -The migration tool automatically detects and handles various SQLite table structures: - -### Modern Schema -```sql --- Stations -stations: id, station_code, station_name_th, station_name_en, latitude, longitude, geohash - --- Measurements -water_measurements: timestamp, station_id, water_level, discharge, discharge_percent, status -``` - -### Legacy Schema -```sql --- Stations -water_stations: station_id, station_code, station_name, lat, lon - --- Measurements -measurements: timestamp, station_id, water_level, discharge, discharge_percent -``` - -### Simple Schema -```sql --- Any table with basic water level data --- The tool will adapt and map columns automatically -``` - -## Migration Process - -### Step 1: Database Connection -- Connects to your SQLite database -- Verifies PostgreSQL connection -- Validates configuration - -### Step 2: Schema Analysis -- Scans SQLite tables and columns -- Reports data counts -- Identifies table structures - -### Step 3: Station Migration -- Extracts station metadata -- Maps to PostgreSQL format -- Handles missing data gracefully - -### Step 4: Measurement Migration -- Processes data in batches (1000 records at a time) -- Converts timestamps correctly -- Preserves all measurement values -- Shows progress during migration - -### Step 5: Verification -- Compares record counts -- Validates data integrity -- Reports migration statistics - -## Command Options - -```bash -# Basic migration (auto-detects SQLite file) -uv run python scripts/migrate_sqlite_to_postgres.py - -# Specify SQLite database path -uv run python scripts/migrate_sqlite_to_postgres.py /path/to/database.db - -# Dry run (analyze only, no migration) -uv run python scripts/migrate_sqlite_to_postgres.py --dry-run - -# Custom batch size for large databases -uv run python scripts/migrate_sqlite_to_postgres.py --batch-size 5000 -``` - -## Auto-Detection - -The tool automatically searches for SQLite files in common locations: -- `water_levels.db` -- `water_monitoring.db` -- `database.db` -- `../water_levels.db` - -## Migration Output - -The tool provides detailed logging: - -``` -======================================== -SQLite to PostgreSQL Migration Tool -======================================== -SQLite database: water_levels.db -PostgreSQL: postgresql - -Step 1: Connecting to databases... -Connected to SQLite database: water_levels.db -Connected to PostgreSQL database - -Step 2: Analyzing SQLite database structure... -Table 'stations': 8 columns, 25 rows -Table 'water_measurements': 7 columns, 15420 rows - -Step 3: Migrating station data... -Migrated 25 stations - -Step 4: Migrating measurement data... -Found 15420 measurements to migrate -Migrated 1000/15420 measurements -Migrated 2000/15420 measurements -... -Successfully migrated 15420 measurements - -Step 5: Verifying migration... -SQLite stations: 25 -SQLite measurements: 15420 -PostgreSQL measurements retrieved: 15420 -Migrated stations: 25 -Migrated measurements: 15420 - -======================================== -MIGRATION COMPLETED -======================================== -Duration: 0:02:15 -Stations migrated: 25 -Measurements migrated: 15420 -No errors encountered -``` - -## Error Handling - -The migration tool is robust and handles: -- **Missing tables** - Tries alternative table names -- **Different column names** - Maps common variations -- **Missing data** - Uses sensible defaults -- **Invalid timestamps** - Attempts multiple date formats -- **Connection issues** - Provides clear error messages -- **Large datasets** - Processes in batches to avoid memory issues - -## Log Files - -Migration creates a detailed log file: -- `migration.log` - Complete migration log -- Shows all operations, errors, and statistics -- Useful for troubleshooting - -## Troubleshooting - -### Common Issues - -**1. SQLite file not found** -``` -SQLite database file not found. Please specify the path: - python migrate_sqlite_to_postgres.py /path/to/database.db -``` -**Solution**: Specify the correct path to your SQLite file - -**2. PostgreSQL not configured** -``` -Error: PostgreSQL not configured. Set DB_TYPE=postgresql in your .env file -``` -**Solution**: Ensure your .env file has `DB_TYPE=postgresql` - -**3. Connection failed** -``` -Database connection error: connection refused -``` -**Solution**: Check your PostgreSQL connection settings - -**4. No tables found** -``` -Could not analyze SQLite database structure -``` -**Solution**: Verify your SQLite file contains water monitoring data - -### Performance Tips - -- **Large databases**: Use `--batch-size 5000` for faster processing -- **Slow networks**: Reduce batch size to `--batch-size 100` -- **Memory issues**: Process smaller batches - -## After Migration - -Once migration is complete: - -1. **Verify data**: - ```bash - make run-test - make run-api - ``` - -2. **Check the web interface**: Latest readings should show your migrated data - -3. **Backup your SQLite**: Keep the original file as backup - -4. **Update configurations**: Remove SQLite references from configs - -## Rollback - -If you need to rollback: - -1. **Clear PostgreSQL data**: - ```sql - DELETE FROM water_measurements; - DELETE FROM stations; - ``` - -2. **Switch back to SQLite**: - ```bash - # In .env file - DB_TYPE=sqlite - WATER_DB_PATH=water_levels.db - ``` - -3. **Test the rollback**: - ```bash - make run-test - ``` - -The migration tool is designed to be safe and can be run multiple times - it handles duplicates appropriately. - -## Next Steps - -After successful migration: -- Set up automated backups for PostgreSQL -- Configure monitoring and alerting -- Consider data retention policies -- Update documentation references \ No newline at end of file diff --git a/docker-compose.victoriametrics.yml b/docker-compose.victoriametrics.yml deleted file mode 100644 index e8d6364..0000000 --- a/docker-compose.victoriametrics.yml +++ /dev/null @@ -1,106 +0,0 @@ -version: '3.8' - -services: - victoriametrics: - image: victoriametrics/victoria-metrics:latest - container_name: ping_river_victoriametrics - ports: - - "8428:8428" - volumes: - - vm_data:/victoria-metrics-data - - ./vm-config:/etc/victoriametrics - command: - - '--storageDataPath=/victoria-metrics-data' - - '--retentionPeriod=2y' - - '--httpListenAddr=:8428' - - '--maxConcurrentInserts=32' - - '--search.maxQueryDuration=60s' - - '--search.maxConcurrentRequests=16' - - '--dedup.minScrapeInterval=30s' - - '--memory.allowedPercent=80' - - '--loggerLevel=INFO' - - '--loggerFormat=json' - - '--search.maxSeries=1000000' - - '--search.maxPointsPerTimeseries=100000' - restart: unless-stopped - environment: - - TZ=Asia/Bangkok - healthcheck: - test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost:8428/health"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 40s - labels: - - "com.victoriametrics.service=ping-river-monitoring" - - grafana: - image: grafana/grafana:latest - container_name: ping_river_grafana - ports: - - "3000:3000" - volumes: - - grafana_data:/var/lib/grafana - - ./grafana/provisioning:/etc/grafana/provisioning - - ./grafana/dashboards:/var/lib/grafana/dashboards - environment: - - GF_SECURITY_ADMIN_PASSWORD=admin_password - - GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-worldmap-panel - - GF_USERS_ALLOW_SIGN_UP=false - - GF_SECURITY_ALLOW_EMBEDDING=true - - GF_AUTH_ANONYMOUS_ENABLED=false - - TZ=Asia/Bangkok - restart: unless-stopped - depends_on: - victoriametrics: - condition: service_healthy - labels: - - "com.victoriametrics.service=ping-river-monitoring" - - ping-river-monitor: - build: . - container_name: ping_river_monitor - environment: - - DB_TYPE=victoriametrics - - VM_HOST=victoriametrics - - VM_PORT=8428 - - TZ=Asia/Bangkok - restart: unless-stopped - depends_on: - victoriametrics: - condition: service_healthy - volumes: - - ./logs:/app/logs - labels: - - "com.victoriametrics.service=ping-river-monitoring" - - ping-river-api: - build: . - container_name: ping_river_api - ports: - - "8000:8000" - environment: - - DB_TYPE=victoriametrics - - VM_HOST=victoriametrics - - VM_PORT=8428 - - TZ=Asia/Bangkok - restart: unless-stopped - depends_on: - victoriametrics: - condition: service_healthy - volumes: - - ./logs:/app/logs - command: ["python", "-m", "src.main", "--web-api"] - labels: - - "com.victoriametrics.service=ping-river-monitoring" - -volumes: - vm_data: - driver: local - grafana_data: - driver: local - -networks: - default: - name: ping-river-monitoring - driver: bridge diff --git a/docs/DATABASE_DEPLOYMENT_GUIDE.md b/docs/DATABASE_DEPLOYMENT_GUIDE.md deleted file mode 100644 index 1861fa3..0000000 --- a/docs/DATABASE_DEPLOYMENT_GUIDE.md +++ /dev/null @@ -1,447 +0,0 @@ -# Database Deployment Guide for Thailand Water Monitor - -This guide covers deployment options for storing water monitoring data in production environments. - -## 🏆 Recommendation Summary - -| Database | Best For | Performance | Complexity | Cost | -|----------|----------|-------------|------------|------| -| **InfluxDB** | Time-series data, dashboards | ⭐⭐⭐⭐⭐ | ⭐⭐⭐ | ⭐⭐⭐ | -| **VictoriaMetrics** | High-performance metrics | ⭐⭐⭐⭐⭐ | ⭐⭐⭐⭐ | ⭐⭐⭐⭐ | -| **PostgreSQL** | Complex queries, reliability | ⭐⭐⭐⭐ | ⭐⭐ | ⭐⭐⭐⭐ | -| **MySQL** | Familiar, existing infrastructure | ⭐⭐⭐ | ⭐⭐ | ⭐⭐⭐⭐ | - -## 1. InfluxDB Deployment (Recommended for Time-Series) - -### Why InfluxDB? -- **Purpose-built** for time-series data -- **Excellent compression** (10:1 typical ratio) -- **Built-in retention policies** and downsampling -- **Great Grafana integration** for dashboards -- **High write throughput** (100k+ points/second) - -### Docker Deployment - -```yaml -# docker-compose.yml -version: '3.8' -services: - influxdb: - image: influxdb:1.8 - container_name: water_influxdb - ports: - - "8086:8086" - volumes: - - influxdb_data:/var/lib/influxdb - - ./influxdb.conf:/etc/influxdb/influxdb.conf:ro - environment: - - INFLUXDB_DB=water_monitoring - - INFLUXDB_ADMIN_USER=admin - - INFLUXDB_ADMIN_PASSWORD=your_secure_password - - INFLUXDB_USER=water_user - - INFLUXDB_USER_PASSWORD=water_password - restart: unless-stopped - - grafana: - image: grafana/grafana:latest - container_name: water_grafana - ports: - - "3000:3000" - volumes: - - grafana_data:/var/lib/grafana - environment: - - GF_SECURITY_ADMIN_PASSWORD=admin_password - restart: unless-stopped - -volumes: - influxdb_data: - grafana_data: -``` - -### Environment Variables -```bash -# .env file -DB_TYPE=influxdb -INFLUX_HOST=localhost -INFLUX_PORT=8086 -INFLUX_DATABASE=water_monitoring -INFLUX_USERNAME=water_user -INFLUX_PASSWORD=water_password -``` - -### InfluxDB Configuration -```toml -# influxdb.conf -[meta] - dir = "/var/lib/influxdb/meta" - -[data] - dir = "/var/lib/influxdb/data" - wal-dir = "/var/lib/influxdb/wal" - - # Optimize for time-series data - cache-max-memory-size = "1g" - cache-snapshot-memory-size = "25m" - cache-snapshot-write-cold-duration = "10m" - - # Retention and compression - compact-full-write-cold-duration = "4h" - max-series-per-database = 1000000 - max-values-per-tag = 100000 - -[coordinator] - write-timeout = "10s" - max-concurrent-queries = 0 - query-timeout = "0s" - -[retention] - enabled = true - check-interval = "30m" - -[http] - enabled = true - bind-address = ":8086" - auth-enabled = true - max-body-size = "25000000" - max-concurrent-requests = 0 - max-enqueued-requests = 0 -``` - -### Production Setup Commands -```bash -# Start services -docker-compose up -d - -# Create retention policies -docker exec -it water_influxdb influx -username admin -password your_secure_password -execute " -CREATE RETENTION POLICY \"raw_data\" ON \"water_monitoring\" DURATION 90d REPLICATION 1 DEFAULT; -CREATE RETENTION POLICY \"downsampled\" ON \"water_monitoring\" DURATION 730d REPLICATION 1; -" - -# Create continuous queries for downsampling -docker exec -it water_influxdb influx -username admin -password your_secure_password -execute " -CREATE CONTINUOUS QUERY \"downsample_hourly\" ON \"water_monitoring\" -BEGIN - SELECT mean(water_level) AS water_level, mean(discharge) AS discharge, mean(discharge_percent) AS discharge_percent - INTO \"downsampled\".\"water_data_hourly\" - FROM \"water_data\" - GROUP BY time(1h), station_code, station_name_en, station_name_th -END -" -``` - -## 2. VictoriaMetrics Deployment (High Performance) - -### Why VictoriaMetrics? -- **Extremely fast** and resource-efficient -- **Better compression** than InfluxDB -- **Prometheus-compatible** API -- **Lower memory usage** -- **Built-in clustering** - -### Docker Deployment -```yaml -# docker-compose.yml -version: '3.8' -services: - victoriametrics: - image: victoriametrics/victoria-metrics:latest - container_name: water_victoriametrics - ports: - - "8428:8428" - volumes: - - vm_data:/victoria-metrics-data - command: - - '--storageDataPath=/victoria-metrics-data' - - '--retentionPeriod=2y' - - '--httpListenAddr=:8428' - - '--maxConcurrentInserts=16' - restart: unless-stopped - -volumes: - vm_data: -``` - -### Environment Variables -```bash -# .env file -DB_TYPE=victoriametrics -VM_HOST=localhost -VM_PORT=8428 -``` - -## 3. PostgreSQL Deployment (Relational + Time-Series) - -### Why PostgreSQL? -- **Mature and reliable** -- **Excellent for complex queries** -- **TimescaleDB extension** for time-series optimization -- **Strong consistency guarantees** -- **Rich ecosystem** - -### Docker Deployment with TimescaleDB -```yaml -# docker-compose.yml -version: '3.8' -services: - postgres: - image: timescale/timescaledb:latest-pg14 - container_name: water_postgres - ports: - - "5432:5432" - volumes: - - postgres_data:/var/lib/postgresql/data - - ./init.sql:/docker-entrypoint-initdb.d/init.sql - environment: - - POSTGRES_DB=water_monitoring - - POSTGRES_USER=water_user - - POSTGRES_PASSWORD=secure_password - restart: unless-stopped - -volumes: - postgres_data: -``` - -### Database Initialization -```sql --- init.sql -CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE; - --- Create hypertable for time-series optimization -CREATE TABLE water_measurements ( - id BIGSERIAL PRIMARY KEY, - timestamp TIMESTAMPTZ NOT NULL, - station_id INT NOT NULL, - water_level NUMERIC(10,3), - discharge NUMERIC(10,2), - discharge_percent NUMERIC(5,2), - status VARCHAR(20) DEFAULT 'active', - created_at TIMESTAMPTZ DEFAULT NOW() -); - --- Convert to hypertable (TimescaleDB) -SELECT create_hypertable('water_measurements', 'timestamp', chunk_time_interval => INTERVAL '1 day'); - --- Create indexes -CREATE INDEX idx_water_measurements_station_time ON water_measurements (station_id, timestamp DESC); -CREATE INDEX idx_water_measurements_timestamp ON water_measurements (timestamp DESC); - --- Create retention policy (keep raw data for 2 years) -SELECT add_retention_policy('water_measurements', INTERVAL '2 years'); - --- Create continuous aggregates for performance -CREATE MATERIALIZED VIEW water_measurements_hourly -WITH (timescaledb.continuous) AS -SELECT - time_bucket('1 hour', timestamp) AS bucket, - station_id, - AVG(water_level) as avg_water_level, - MAX(water_level) as max_water_level, - MIN(water_level) as min_water_level, - AVG(discharge) as avg_discharge, - MAX(discharge) as max_discharge, - MIN(discharge) as min_discharge, - AVG(discharge_percent) as avg_discharge_percent -FROM water_measurements -GROUP BY bucket, station_id; - --- Refresh policy for continuous aggregates -SELECT add_continuous_aggregate_policy('water_measurements_hourly', - start_offset => INTERVAL '1 day', - end_offset => INTERVAL '1 hour', - schedule_interval => INTERVAL '1 hour'); -``` - -### Environment Variables -```bash -# .env file -DB_TYPE=postgresql -POSTGRES_CONNECTION_STRING=postgresql://water_user:secure_password@localhost:5432/water_monitoring -``` - -## 4. MySQL Deployment (Traditional Relational) - -### Docker Deployment -```yaml -# docker-compose.yml -version: '3.8' -services: - mysql: - image: mysql:8.0 - container_name: water_mysql - ports: - - "3306:3306" - volumes: - - mysql_data:/var/lib/mysql - - ./mysql.cnf:/etc/mysql/conf.d/mysql.cnf - - ./init.sql:/docker-entrypoint-initdb.d/init.sql - environment: - - MYSQL_ROOT_PASSWORD=root_password - - MYSQL_DATABASE=water_monitoring - - MYSQL_USER=water_user - - MYSQL_PASSWORD=water_password - restart: unless-stopped - -volumes: - mysql_data: -``` - -### MySQL Configuration -```ini -# mysql.cnf -[mysqld] -# Optimize for time-series data -innodb_buffer_pool_size = 1G -innodb_log_file_size = 256M -innodb_flush_log_at_trx_commit = 2 -innodb_flush_method = O_DIRECT - -# Partitioning support -partition = ON - -# Query cache -query_cache_type = 1 -query_cache_size = 128M - -# Connection settings -max_connections = 200 -connect_timeout = 10 -wait_timeout = 600 -``` - -### Environment Variables -```bash -# .env file -DB_TYPE=mysql -MYSQL_CONNECTION_STRING=mysql://water_user:water_password@localhost:3306/water_monitoring -``` - -## 5. Installation and Dependencies - -### Required Python Packages -```bash -# Base requirements -pip install requests schedule - -# Database-specific packages -pip install influxdb # For InfluxDB -pip install sqlalchemy pymysql # For MySQL -pip install sqlalchemy psycopg2-binary # For PostgreSQL -# VictoriaMetrics uses HTTP API (no extra packages needed) -``` - -### Updated requirements.txt -```txt -requests>=2.28.0 -schedule>=1.2.0 -pandas>=1.5.0 - -# Database adapters (install as needed) -influxdb>=5.3.1 -sqlalchemy>=1.4.0 -pymysql>=1.0.2 -psycopg2-binary>=2.9.0 -``` - -## 6. Production Deployment Examples - -### Using InfluxDB (Recommended) -```bash -# Set environment variables -export DB_TYPE=influxdb -export INFLUX_HOST=your-influx-server.com -export INFLUX_PORT=8086 -export INFLUX_DATABASE=water_monitoring -export INFLUX_USERNAME=water_user -export INFLUX_PASSWORD=your_secure_password - -# Run the scraper -python water_scraper_v3.py -``` - -### Using PostgreSQL with TimescaleDB -```bash -# Set environment variables -export DB_TYPE=postgresql -export POSTGRES_CONNECTION_STRING=postgresql://water_user:password@your-postgres-server.com:5432/water_monitoring - -# Run the scraper -python water_scraper_v3.py -``` - -### Using VictoriaMetrics -```bash -# Set environment variables -export DB_TYPE=victoriametrics -export VM_HOST=your-vm-server.com -export VM_PORT=8428 - -# Run the scraper -python water_scraper_v3.py -``` - -## 7. Monitoring and Alerting - -### Grafana Dashboard Setup -1. **Add Data Source**: Configure your database as a Grafana data source -2. **Import Dashboard**: Use pre-built water monitoring dashboards -3. **Set Alerts**: Configure alerts for abnormal water levels or discharge rates - -### Example Grafana Queries - -#### InfluxDB Queries -```sql --- Current water levels -SELECT last("water_level") FROM "water_data" GROUP BY "station_code" - --- Discharge trends (last 24h) -SELECT mean("discharge") FROM "water_data" WHERE time >= now() - 24h GROUP BY time(1h), "station_code" -``` - -#### PostgreSQL/TimescaleDB Queries -```sql --- Current water levels -SELECT DISTINCT ON (station_id) - station_id, water_level, discharge, timestamp -FROM water_measurements -ORDER BY station_id, timestamp DESC; - --- Hourly averages (last 24h) -SELECT - time_bucket('1 hour', timestamp) as hour, - station_id, - AVG(water_level) as avg_level, - AVG(discharge) as avg_discharge -FROM water_measurements -WHERE timestamp >= NOW() - INTERVAL '24 hours' -GROUP BY hour, station_id -ORDER BY hour DESC; -``` - -## 8. Performance Optimization Tips - -### For All Databases -- **Batch inserts**: Insert multiple measurements at once -- **Connection pooling**: Reuse database connections -- **Indexing**: Ensure proper indexes on timestamp and station_id -- **Retention policies**: Automatically delete old data - -### InfluxDB Specific -- Use **tags** for metadata (station codes, names) -- Use **fields** for numeric values (water levels, discharge) -- Configure **retention policies** and **continuous queries** -- Enable **compression** for long-term storage - -### PostgreSQL/TimescaleDB Specific -- Use **hypertables** for automatic partitioning -- Create **continuous aggregates** for common queries -- Configure **compression** for older chunks -- Use **parallel queries** for large datasets - -### VictoriaMetrics Specific -- Use **labels** efficiently (similar to Prometheus) -- Configure **retention periods** appropriately -- Use **downsampling** for long-term storage -- Enable **deduplication** if needed - -This deployment guide provides production-ready configurations for all supported database backends. Choose the one that best fits your infrastructure and requirements. diff --git a/docs/DATA_SOURCES.md b/docs/DATA_SOURCES.md index a98b058..2dece3c 100644 --- a/docs/DATA_SOURCES.md +++ b/docs/DATA_SOURCES.md @@ -23,7 +23,7 @@ The only source persisted to the database and used by the ML pipeline. | Params | `DW[UtokID]=1`, `DW[BasinID]=6` (Ping), `DW[TimeCurrent]=`, `rows=100` | | Variables | Water level (m, gauge datum), discharge (m³/s, `'***'` = malformed), discharge % of channel capacity | | Stations | 16 P-series gauges (P.1 anchor at Nawarat Bridge; see `src/data/stations.json`) | -| Client | `src/water_scraper_v3.py` | +| Client | `rust/collector/src/rid.rs` (`ping-collector`; `fill-gaps` re-fetches old reports) | Human-facing page: @@ -117,9 +117,9 @@ GET .../rain_24h_graph?station_type=tele_rainfall&station_id=418&start_date=...& ### 2.2 `twa-api-public.thaiwater.net` — auth-gated (x-api-key / session) 🔒 -The layer our existing `src/thaiwater.py` client uses -(`GET /v2/waterlevel` with `x-api-key: $THAIWATER_API_KEY`; wired to -`GET /sensors/thaiwater` in the web API, display-only, never persisted). +The layer the website's `GET /sensors/thaiwater` uses (`GET /v2/waterlevel` +with an `x-api-key` header from `THAIWATER_API_KEY`; +`rust/collector/src/web/hii.rs`, display-only, never persisted). Without a key: HTTP 401/500. No public key-registration page was found — obtain a sanctioned key from HII (). @@ -242,21 +242,22 @@ Oct 2024 flood). Mae Kuang Udom Thara is the second upstream reservoir. | Source | Status | Action | |---|---|---| | RID hourly gauges | ✅ Ingested (hourly → PostgreSQL) | — | -| ThaiWater `/v2/waterlevel` | 🟡 Display-only (`src/thaiwater.py`, needs `THAIWATER_API_KEY`, never persisted) | Optionally persist | -| HII `rain_24h` | ✅ Ingested hourly via `src/hii_collector.py` → `hii_rain_stations` + `hii_rainfall` (Ping-filtered; ~300 stations) | — | -| HII `waterlevel_load` | ✅ Ingested hourly via `src/hii_collector.py` → `hii_wl_stations` + `hii_waterlevel` (125 Ping stations, m MSL; `rid_code` column maps mirrors like `ridhydro_P.1` → `P.1`, `offset_msl` converts MSL → gauge datum) | — | +| ThaiWater `/v2/waterlevel` | 🟡 Display-only (`/sensors/thaiwater`, needs `THAIWATER_API_KEY`, never persisted) | Optionally persist | +| HII `rain_24h` | ✅ Ingested hourly by ping-collector (`rust/collector/src/hii.rs`) → `hii_rain_stations` + `hii_rainfall` (Ping-filtered; ~300 stations) | — | +| HII `waterlevel_load` | ✅ Ingested hourly by ping-collector (`rust/collector/src/hii.rs`) → `hii_wl_stations` + `hii_waterlevel` (125 Ping stations, m MSL; `rid_code` column maps mirrors like `ridhydro_P.1` → `P.1`, `offset_msl` converts MSL → gauge datum) | — | | HII `waterlevel_graph` | ✅ Backfill via `scripts/backfill_hii_waterlevel.py` → `hii_waterlevel` (hourly MSL + discharge, archive ≥2019; full-year windows per request; upserts never overwrite live-snapshot columns) | Run once on the box: `python scripts/backfill_hii_waterlevel.py` (defaults: 2019-01-01 → today, RID-mirror + key stations; `--stations P.1,P.67`, `--all` for every Ping station) | | Mae Ngat reservoir (app.rid.go.th) | ✅ Ingested (2026-08-13) | Daily storage/inflow/outflow for all large dams → `rid_reservoir_daily`; candidate model features for next retrain | | Satellite QPE (GSMaP/IMERG) | ❌ | Backfill training rainfall (GEE) | -| Open-Meteo forecasts | ❌ | Add forecast features (live + 2021 archive for training) | +| Open-Meteo rain | ✅ Hourly by ping-collector → `openmeteo_rain` (5 catchment points, past 5 days + 48 h forecast); the 2021+ archive trains the hgb-v3 rain features | — | | HII API key (dams, forecasts) | ❌ | Contact HII for sanctioned access | -**Collector configuration** (`src/hii_collector.py`): runs automatically every -scraping cycle (hourly cadence, even while the RID scraper is in 1-minute retry -mode) from both `--web-api` and continuous-monitoring modes; one-shot via -`python -m src.main --collect-hii`. Env vars: `ENABLE_HII_COLLECTION` -(default `true`), `HII_BASIN_CODE` (default `6` = Ping). Requires a SQL -`DB_TYPE` (sqlite/postgresql/mysql); tables are created automatically. +**Collector configuration** (ping-collector): HII, the dams and Open-Meteo are +fetched once per clock hour, also while RID is being retried; one-shot via +`ping-collector once`. Env vars: `ENABLE_HII_COLLECTION`, +`ENABLE_RESERVOIR_COLLECTION`, `ENABLE_RAIN_COLLECTION` (default `true`), +`HII_BASIN_CODE` (default `6` = Ping). Tables are created automatically +(PostgreSQL). The archive backfills stay in Python (`scripts/backfill_*.py`, +using `src/hii_collector.py` and `src/rid_reservoir.py`). **Caveats**: `api-v3` is an undocumented backend — no SLA, no ToS, may change without notice. Poll hourly at most, cache aggressively, and pursue official diff --git a/docs/DEBIAN_TROUBLESHOOTING.md b/docs/DEBIAN_TROUBLESHOOTING.md deleted file mode 100644 index 24f7c39..0000000 --- a/docs/DEBIAN_TROUBLESHOOTING.md +++ /dev/null @@ -1,329 +0,0 @@ -# Debian/Linux Troubleshooting Guide - -This guide addresses common issues when running the Thailand Water Monitor on Debian and other Linux distributions. - -## Fixed Issues - -### SQLAlchemy Connection Error (RESOLVED) - -**Error Message:** -``` -2025-07-24 19:48:31,920 - ERROR - Failed to connect to SQLITE: 'Connection' object has no attribute 'commit' -2025-07-24 19:48:32,740 - ERROR - Error saving to SQLITE: 'Connection' object has no attribute 'commit' -``` - -**Root Cause:** -This error occurred due to incompatibility between the database adapter code and newer versions of SQLAlchemy. The code was calling `conn.commit()` on a connection object that doesn't have a `commit()` method in newer SQLAlchemy versions. - -**Solution Applied:** -Changed from `engine.connect()` to `engine.begin()` context manager, which automatically handles transactions: - -```python -# OLD (problematic) code: -with self.engine.connect() as conn: - conn.execute(text(sql)) - conn.commit() # This fails in newer SQLAlchemy - -# NEW (fixed) code: -with self.engine.begin() as conn: - conn.execute(text(sql)) - # Transaction automatically committed when context exits -``` - -**Status:** ✅ **FIXED** - The issue has been resolved in the current version. - -## Installation on Debian/Ubuntu - -### System Requirements - -```bash -# Update package list -sudo apt update - -# Install Python and pip -sudo apt install python3 python3-pip python3-venv - -# Install system dependencies for database drivers -sudo apt install build-essential python3-dev - -# For MySQL support (optional) -sudo apt install default-libmysqlclient-dev - -# For PostgreSQL support (optional) -sudo apt install libpq-dev -``` - -### Python Environment Setup - -```bash -# Create virtual environment -python3 -m venv water_monitor_env - -# Activate virtual environment -source water_monitor_env/bin/activate - -# Install requirements -pip install -r requirements.txt -``` - -### Running the Monitor - -```bash -# Test run -python water_scraper_v3.py --test - -# Run with specific database -export DB_TYPE=sqlite -python water_scraper_v3.py - -# Run demo -python demo_databases.py -``` - -## Common Linux Issues - -### 1. Permission Errors - -**Error:** -``` -PermissionError: [Errno 13] Permission denied: 'water_levels.db' -``` - -**Solution:** -```bash -# Check current directory permissions -ls -la - -# Create data directory with proper permissions -mkdir -p data -chmod 755 data - -# Set database path to data directory -export WATER_DB_PATH=data/water_levels.db -``` - -### 2. Missing System Dependencies - -**Error:** -``` -ImportError: No module named '_sqlite3' -``` - -**Solution:** -```bash -# Install SQLite development headers -sudo apt install libsqlite3-dev - -# Reinstall Python if needed -sudo apt install python3-sqlite -``` - -### 3. Network/Firewall Issues - -**Error:** -``` -requests.exceptions.ConnectionError: HTTPSConnectionPool -``` - -**Solution:** -```bash -# Test network connectivity -curl -I https://hyd-app-db.rid.go.th/hydro1h.html - -# Check firewall rules -sudo ufw status - -# Allow outbound HTTPS if needed -sudo ufw allow out 443 -``` - -### 4. Systemd Service Setup - -Create service file `/etc/systemd/system/water-monitor.service`: - -```ini -[Unit] -Description=Thailand Water Level Monitor -After=network.target - -[Service] -Type=simple -User=water-monitor -Group=water-monitor -WorkingDirectory=/opt/water_level_monitor -Environment=PATH=/opt/water_level_monitor/venv/bin -Environment=DB_TYPE=sqlite -Environment=WATER_DB_PATH=/opt/water_level_monitor/data/water_levels.db -ExecStart=/opt/water_level_monitor/venv/bin/python water_scraper_v3.py -Restart=always -RestartSec=60 - -# Security settings -NoNewPrivileges=true -PrivateTmp=true -ProtectSystem=strict -ProtectHome=true -ReadWritePaths=/opt/water_level_monitor/data -ReadWritePaths=/opt/water_level_monitor/logs - -[Install] -WantedBy=multi-user.target -``` - -Enable and start: -```bash -sudo systemctl daemon-reload -sudo systemctl enable water-monitor.service -sudo systemctl start water-monitor.service -sudo systemctl status water-monitor.service -``` - -### 5. Log Rotation - -Create `/etc/logrotate.d/water-monitor`: - -``` -/opt/water_level_monitor/water_monitor.log { - daily - missingok - rotate 30 - compress - delaycompress - notifempty - create 644 water-monitor water-monitor - postrotate - systemctl reload water-monitor.service - endscript -} -``` - -## Database-Specific Issues - -### SQLite - -**Issue:** Database locked -```bash -# Check for processes using the database -sudo lsof /path/to/water_levels.db - -# Kill processes if needed -sudo pkill -f water_scraper_v3.py -``` - -### VictoriaMetrics with HTTPS - -**Configuration:** -```bash -export DB_TYPE=victoriametrics -export VM_HOST=https://your-vm-server.com -export VM_PORT=443 -``` - -**Test connection:** -```bash -curl -k https://your-vm-server.com/health -``` - -## Performance Optimization - -### 1. System Tuning - -```bash -# Increase file descriptor limits -echo "* soft nofile 65536" >> /etc/security/limits.conf -echo "* hard nofile 65536" >> /etc/security/limits.conf - -# Optimize network settings -echo "net.core.rmem_max = 16777216" >> /etc/sysctl.conf -echo "net.core.wmem_max = 16777216" >> /etc/sysctl.conf -sysctl -p -``` - -### 2. Database Optimization - -```bash -# For SQLite -export SQLITE_CACHE_SIZE=10000 -export SQLITE_SYNCHRONOUS=NORMAL - -# Monitor database size -du -h data/water_levels.db -``` - -## Monitoring and Maintenance - -### Health Check Script - -Create `health_check.sh`: - -```bash -#!/bin/bash -LOG_FILE="/opt/water_level_monitor/water_monitor.log" -SERVICE_NAME="water-monitor" - -# Check if service is running -if ! systemctl is-active --quiet $SERVICE_NAME; then - echo "ERROR: $SERVICE_NAME is not running" - systemctl restart $SERVICE_NAME - exit 1 -fi - -# Check recent log entries -RECENT_ERRORS=$(tail -n 100 $LOG_FILE | grep -c "ERROR") -if [ $RECENT_ERRORS -gt 5 ]; then - echo "WARNING: $RECENT_ERRORS errors found in recent logs" - exit 1 -fi - -echo "OK: Service is healthy" -exit 0 -``` - -### Cron Job for Health Checks - -```bash -# Add to crontab -*/5 * * * * /opt/water_level_monitor/health_check.sh >> /var/log/water-monitor-health.log 2>&1 -``` - -## Getting Help - -### Debug Information - -```bash -# System information -uname -a -python3 --version -pip list | grep -E "(sqlalchemy|requests|influxdb)" - -# Service logs -journalctl -u water-monitor.service -f - -# Application logs -tail -f water_monitor.log - -# Database information -sqlite3 water_levels.db ".schema" -sqlite3 water_levels.db "SELECT COUNT(*) FROM water_measurements;" -``` - -### Common Commands - -```bash -# Restart service -sudo systemctl restart water-monitor.service - -# View logs -sudo journalctl -u water-monitor.service --since "1 hour ago" - -# Test configuration -python config.py - -# Test database connection -python demo_databases.py - -# Manual data fetch -python water_scraper_v3.py --test -``` - -This troubleshooting guide should help resolve most common issues encountered when running the Thailand Water Monitor on Debian and other Linux distributions. diff --git a/docs/FLOOD_FORECASTING.md b/docs/FLOOD_FORECASTING.md index 53af623..0bb1c0a 100644 --- a/docs/FLOOD_FORECASTING.md +++ b/docs/FLOOD_FORECASTING.md @@ -62,8 +62,8 @@ P.5 before 2022-01-01; the rest are handled by the per-row coverage gates. **How `src/ml/data.py` loads it.** `resolve_db_url()` picks a connection string in priority order: an explicit `--db-url` argument, then the `FLOOD_ML_DB_URL` -environment variable, then `Config.get_database_config()` when `DB_TYPE` is -`postgresql`, else `None`. `load_measurements()` then tries three tiers: +environment variable, then the `POSTGRES_*` settings (`src/config.py`, the same +`.env` the service reads), else `None`. `load_measurements()` then tries three tiers: 1. **PostgreSQL** (`_fetch_from_db`) — the primary path. NULL discharge stays NULL, which matters because the models must learn from the real missingness @@ -614,32 +614,28 @@ fix, the served features match what this section evaluates. ## 6. Deployment -### API +### Serving -With `FORECAST_IN_COLLECTOR=true` the forecast is computed by `ping-collector` -(`rust/forecast`, a parity-tested port of this serving path) from +The forecast is computed by `ping-collector` (`rust/forecast`, a parity-tested +port of `src/ml/predict.py`, which stays as the reference implementation) from `models/flood_.json.gz`, which `train_all()` exports next to each -joblib bundle; `/forecast` serves its `forecast_current` row and computes -in-process only when that is older than `FORECAST_MAX_AGE_SECONDS` (2 h). See -`rust/README.md`. The rest of this section describes the in-process path. +joblib bundle. It runs after every collection cycle that brought newer +readings and at least hourly, stores the rows in `forecast_history` (section 9's +skill tracking) and `forecast_current`, and `GET /forecast` serves the stored +row: one JSON row per station × horizon with the fields listed in section 1. +A stored forecast older than `FORECAST_MAX_AGE_SECONDS` (2 h, i.e. the service +has stopped forecasting) is still served, marked `X-Data-Stale: true`. See +`rust/README.md`. -`GET /forecast` (`src/web_api.py`) returns one JSON row per station × horizon with -the fields listed in section 1. Results are cached in-process for -`FORECAST_TTL = 900` seconds (15 minutes), which matches the data cadence — the -underlying readings do not update faster than hourly. Inference runs in a thread -via `asyncio.to_thread` so it never blocks the event loop. +A station without a model bundle, or whose bundle needs a feature the Rust side +cannot build, gets the heuristic, as in Python. So **before the first training +run `/forecast` returns 200 with an all-heuristic payload**, provided there is +recent gauge data; you get an empty `200 []` only when there is no recent data +at all. Judge deployment state by the `source` field, not the status code. -Failure modes: **503** if the `src.ml` package cannot be imported (missing -scikit-learn, say), **502** on any other exception. - -One behaviour worth knowing, because the code comments suggest otherwise: the -endpoint's `FileNotFoundError` ("No trained flood models found") and `RuntimeError` -handlers are unreachable — nothing in `src/ml/` raises either, and -`predict._forecast_station` checks `bundle_path.exists()` and falls back to the -heuristic instead. So **before the first training run `/forecast` returns 200 with -an all-heuristic payload**, not a 503, provided there is recent gauge data; you -get an empty `200 []` only when there is no recent data at all. Judge deployment -state by the `source` field, not the status code. +The Python serving path (`predict.get_latest_forecasts`) is not called by the +service any more; `scripts/forecast_parity.py` runs both on the live database +and `scripts/install.sh` does that on every install. ### Dashboard @@ -661,13 +657,14 @@ The server already has the PostgreSQL connection configured, so no host override is needed: ```bash -cd /path/to/Northern-Thailand-Ping-River-Monitor -python scripts/train_flood_model.py --stations all +cd /opt/thailand-water-monitor +uv run python scripts/train_flood_model.py --stations all ``` -`resolve_db_url()` picks up `Config.get_database_config()` automatically when -`DB_TYPE=postgresql`. The run writes fifteen `models/flood_{station}.joblib` -bundles plus `models/metrics.json`. +`resolve_db_url()` picks up the `POSTGRES_*` settings from `.env`. The run writes +fifteen `models/flood_{station}.joblib` bundles, their `.json.gz` exports for the +service, and `models/metrics.json`. The monthly timer does the same through +`scripts/retrain.sh` with a staged promote (section 7). ### Artifacts and dependencies @@ -733,7 +730,10 @@ gap between fetch and first model could not be reconstructed from the surviving artifacts. Do not attribute it to per-station training cost. Either way the conclusion holds: **retraining is minutes, not tens of minutes.** -**Inference** (15 bundles, 16 stations × 3 horizons = 48 rows): +**Inference in Python** (15 bundles, 16 stations × 3 horizons = 48 rows; the +retired in-process serving path, measured before the Rust forecaster took +over: there the whole run takes milliseconds, 0.25 ms per station against +158 ms, and the 15 exported models need about 70 MB resident): | Measurement | Value | |---|---| @@ -753,9 +753,9 @@ one-time lazy `sklearn.ensemble` import — the remaining fourteen average 159 m Of the ~640 ms warm compute, model prediction is ~525 ms, the hourly grid 47 ms, and feature building 71 ms across all sixteen stations. -**Live-endpoint measurements** (a second, independent benchmark run against a real -uvicorn instance of the app, same day, same workstation, RSS summed over the -process tree): +**Live-endpoint measurements of the retired Python API** (a second, independent +benchmark run against a real uvicorn instance of the app, same day, same +workstation, RSS summed over the process tree): | Measurement | Value | |---|---| @@ -775,14 +775,11 @@ Training shows the opposite (2.3× slower single-threaded), so set the variable per process, never globally. **Server sizing, in plain terms:** this is a small workload and almost any server -runs it. **RAM is the binding constraint, not CPU.** Budget about **1 GB for the -API process** so the ~335 MB steady state has headroom on top of the rest of the -app; training peaks at only ~210–315 MB and can share the same box. No GPU -anywhere. Pin thread counts per process — `OMP_NUM_THREADS=1` in the serving -unit, `OMP_NUM_THREADS=4` for retraining so it cannot monopolise every core while -the API is serving. And since a cold call costs seconds against a 2.4 ms cache -hit, consider warming `/forecast` once at startup rather than letting a user -absorb it. +runs it. The service (`ping-collector`, Rust) stays under 100 MB resident with +every model loaded (its unit caps it at 512 MB); the monthly retrain peaks at +~210–315 MB and shares the same box. No GPU anywhere. The retrain unit sets +`OMP_NUM_THREADS=4` so it cannot monopolise every core while the site is +serving. ## 7. Retraining policy @@ -821,15 +818,15 @@ fires `water-monitor-retrain.service` on the 1st of every month at 03:30 server time (`Persistent=true`, so a missed run catches up at boot). The unit runs `scripts/retrain.sh` as the service user with `OMP_NUM_THREADS=4`, `Nice=15`: -1. trains all stations into `models/.staging/` (the API keeps serving the old - bundles throughout); +1. trains all stations into `models/.staging/` (the service keeps serving the + old bundles throughout); 2. refuses to promote unless `metrics.json` reports a `hgb-v3+` version and at least 14 trained stations (exit 3, staging discarded, old models untouched); 3. renames the new bundles into `models/`, moving the previous generation to `models/.previous/` for rollback. -No API restart: `predict.py` reloads bundles by mtime on the next hourly -precompute. `systemctl list-timers water-monitor-retrain.timer` shows the next +No restart: ping-collector reloads changed `.json.gz` exports by mtime on its +next forecast run. `systemctl list-timers water-monitor-retrain.timer` shows the next run; `sudo systemctl start water-monitor-retrain.service` runs it now (after a flood, say); `journalctl -u water-monitor-retrain` has the log. The installer (`scripts/install.sh`) enables the timer. @@ -912,9 +909,10 @@ non-model-backed, and all of them are visible in the payload: horizon's head is in the bundle's `skipped_heads`, almost always because the station had fewer than 30 positive examples for that threshold. - A whole station flipping to heuristic after a code change — the feature-name - check in `_model_forecast` caught a mismatch between the live feature builder - and the stored `feature_names`. The fix is to retrain; the log line names the - missing columns. + check caught a mismatch between the feature builder and the stored + `feature_names` (in Python `_model_forecast`; in Rust a bundle needing a + feature `rust/forecast` cannot build). The fix is to retrain or to port the + feature; the log line names the missing columns. A live example from the 2026-08-10 cache: of 48 forecast rows, 42 came from models and 6 were heuristic — three for P.4A, which has no bundle by design, and diff --git a/docs/GAP_FILLING_GUIDE.md b/docs/GAP_FILLING_GUIDE.md deleted file mode 100644 index c93d262..0000000 --- a/docs/GAP_FILLING_GUIDE.md +++ /dev/null @@ -1,275 +0,0 @@ -# Gap Filling and Data Integrity Guide - -This guide explains the enhanced gap-filling functionality that addresses data gaps and missing timestamps in the Thailand Water Monitor. - -## ✅ **Issues Resolved** - -### **1. Data Gaps Problem** -- **Before**: Tool only fetched current day data, leaving gaps in historical records -- **After**: Automatically detects and fills missing timestamps for the last 7 days - -### **2. Missing Midnight Timestamps** -- **Before**: Jump from 23:00 to 01:00 (missing 00:00 midnight data) -- **After**: Specifically checks for and fills midnight hour gaps - -### **3. Changed Values** -- **Before**: No mechanism to update existing data if values changed on the server -- **After**: Compares existing data with fresh API data and updates changed values - -## 🔧 **New Features** - -### **Command Line Interface** -```bash -# Check for missing data gaps -python water_scraper_v3.py --check-gaps [days] - -# Fill missing data gaps -python water_scraper_v3.py --fill-gaps [days] - -# Update existing data with latest values -python water_scraper_v3.py --update-data [days] - -# Run single test cycle -python water_scraper_v3.py --test - -# Show help -python water_scraper_v3.py --help -``` - -### **Automatic Gap Detection** -The system now automatically: -- Generates expected hourly timestamps for the specified time range -- Compares with existing database records -- Identifies missing timestamps -- Groups missing data by date for efficient API calls - -### **Intelligent Gap Filling** -- **Historical Data Fetching**: Retrieves data for specific dates to fill gaps -- **Selective Insertion**: Only inserts data for actually missing timestamps -- **API Rate Limiting**: Includes delays between API calls to be respectful -- **Error Handling**: Continues processing even if some dates fail - -### **Data Update Mechanism** -- **Change Detection**: Compares water levels, discharge rates, and percentages -- **Precision Checking**: Uses appropriate thresholds (0.001m for water level, 0.1 cms for discharge) -- **Selective Updates**: Only updates records where values have actually changed - -## 📊 **Test Results** - -### **Before Enhancement** -``` -Found 22 missing timestamps in the last 2 days: - 2025-07-23: Missing hours [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23] - 2025-07-24: Missing hours [0, 20, 21, 22, 23] - 2025-07-25: Missing hours [0, 9] -``` - -### **After Gap Filling** -``` -Gap filling completed. Filled 96 missing data points - -Remaining gaps: - 2025-07-24: Missing hours [10] - 2025-07-25: Missing hours [0, 10] -``` - -**Improvement**: Reduced from 22 missing timestamps to 3 (86% improvement) - -## 🚀 **Enhanced Scraping Cycle** - -The regular scraping cycle now includes three phases: - -### **Phase 1: Current Data Collection** -```python -# Fetch and save current data -water_data = self.fetch_water_data() -success = self.save_to_database(water_data) -``` - -### **Phase 2: Gap Filling (Last 7 Days)** -```python -# Check for and fill missing data -filled_count = self.fill_data_gaps(days_back=7) -``` - -### **Phase 3: Data Updates (Last 2 Days)** -```python -# Update existing data with latest values -updated_count = self.update_existing_data(days_back=2) -``` - -## 🔧 **Technical Improvements** - -### **Database Connection Handling** -- **SQLite Optimization**: Added timeout and thread safety parameters -- **Retry Logic**: Exponential backoff for database lock errors -- **Transaction Management**: Proper use of `engine.begin()` for automatic commits - -### **Error Recovery** -```python -# Retry logic with exponential backoff -for attempt in range(max_retries): - try: - success = self.db_adapter.save_measurements(water_data) - if success: - return True - except Exception as e: - if "database is locked" in str(e).lower(): - time.sleep(2 ** attempt) # 1s, 2s, 4s delays - continue -``` - -### **Memory Efficiency** -- **Selective Data Processing**: Only processes data for missing timestamps -- **Batch Processing**: Groups operations by date to minimize API calls -- **Resource Management**: Proper cleanup and connection handling - -## 📋 **Usage Examples** - -### **Daily Maintenance** -```bash -# Check for gaps in the last week -python water_scraper_v3.py --check-gaps 7 - -# Fill any found gaps -python water_scraper_v3.py --fill-gaps 7 - -# Update recent data for accuracy -python water_scraper_v3.py --update-data 2 -``` - -### **Historical Data Recovery** -```bash -# Check for gaps in the last month -python water_scraper_v3.py --check-gaps 30 - -# Fill gaps for the last month (be patient, this takes time) -python water_scraper_v3.py --fill-gaps 30 -``` - -### **Production Monitoring** -```bash -# Quick test to ensure system is working -python water_scraper_v3.py --test - -# Check for recent gaps -python water_scraper_v3.py --check-gaps 1 -``` - -## 🔍 **Monitoring and Alerts** - -### **Gap Detection Output** -``` -Found 22 missing timestamps: - 2025-07-23: Missing hours [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23] - 2025-07-24: Missing hours [0, 20, 21, 22, 23] - 2025-07-25: Missing hours [0, 9] -``` - -### **Gap Filling Progress** -``` -Fetching data for 2025-07-24 to fill 5 missing timestamps -Successfully fetched 368 data points from API for 2025-07-24 -Filled 80 data points for 2025-07-24 -Gap filling completed. Filled 96 missing data points -``` - -### **Update Detection** -``` -Checking for updates on 2025-07-24 -Update needed for P.1 at 2025-07-24 15:00:00 -Updated 5 measurements for 2025-07-24 -Data update completed. Updated 5 measurements -``` - -## ⚙️ **Configuration Options** - -### **Environment Variables** -```bash -# Database configuration -export DB_TYPE=sqlite -export WATER_DB_PATH=water_monitoring.db - -# Gap filling settings (can be added to config.py) -export GAP_FILL_DAYS=7 # Days to check for gaps -export UPDATE_DAYS=2 # Days to check for updates -export API_DELAY=1 # Seconds between API calls -export MAX_RETRIES=3 # Database retry attempts -``` - -### **Customizable Parameters** -- **Gap Check Period**: Default 7 days, configurable via command line -- **Update Period**: Default 2 days, configurable via command line -- **API Rate Limiting**: 1-second delay between calls (configurable) -- **Retry Logic**: 3 attempts with exponential backoff (configurable) - -## 🛠️ **Troubleshooting** - -### **Common Issues** - -#### **Database Locked Errors** -``` -ERROR - Error saving to SQLITE: database is locked -``` -**Solution**: The retry logic now handles this automatically with exponential backoff. - -#### **API Rate Limiting** -``` -WARNING - Too many requests to API -``` -**Solution**: Increase delay between API calls or reduce the number of days processed at once. - -#### **Missing Data Still Present** -``` -Found X missing timestamps after gap filling -``` -**Possible Causes**: -- Data not available on the Thai government server for those timestamps -- Network issues during API calls -- API returned empty data for those specific times - -### **Debug Commands** -```bash -# Enable debug logging -export LOG_LEVEL=DEBUG -python water_scraper_v3.py --check-gaps 1 - -# Test specific date range -python water_scraper_v3.py --fill-gaps 1 - -# Check database directly -sqlite3 water_monitoring.db "SELECT COUNT(*) FROM water_measurements;" -sqlite3 water_monitoring.db "SELECT timestamp, COUNT(*) FROM water_measurements GROUP BY timestamp ORDER BY timestamp DESC LIMIT 10;" -``` - -## 📈 **Performance Metrics** - -### **Gap Filling Efficiency** -- **API Calls**: Grouped by date to minimize requests -- **Processing Speed**: ~100-400 data points per API call -- **Success Rate**: 86% gap reduction in test case -- **Resource Usage**: Minimal memory footprint with selective processing - -### **Database Performance** -- **SQLite Optimization**: Connection pooling and timeout handling -- **Transaction Efficiency**: Batch inserts with proper transaction management -- **Retry Success**: Automatic recovery from temporary lock conditions - -## 🎯 **Best Practices** - -### **Regular Maintenance** -1. **Daily**: Run `--check-gaps 1` to monitor recent data quality -2. **Weekly**: Run `--fill-gaps 7` to catch any missed data -3. **Monthly**: Run `--update-data 7` to ensure data accuracy - -### **Production Deployment** -1. **Automated Scheduling**: Use cron or systemd timers for regular gap checks -2. **Monitoring**: Set up alerts for excessive missing data -3. **Backup**: Regular database backups before major gap-filling operations - -### **Data Quality Assurance** -1. **Validation**: Check for reasonable value ranges after gap filling -2. **Comparison**: Compare filled data with nearby timestamps for consistency -3. **Documentation**: Log all gap-filling activities for audit trails - -This enhanced gap-filling system ensures comprehensive and accurate water level monitoring with minimal data loss and automatic recovery capabilities. diff --git a/docs/GITEA_WORKFLOWS.md b/docs/GITEA_WORKFLOWS.md deleted file mode 100644 index f4a327e..0000000 --- a/docs/GITEA_WORKFLOWS.md +++ /dev/null @@ -1,300 +0,0 @@ -# 🔄 Gitea Actions Workflows - Northern Thailand Ping River Monitor - -## 📋 Overview - -This document describes the Gitea Actions workflows configured for the Northern Thailand Ping River Monitor project. These workflows provide comprehensive CI/CD, security scanning, and documentation generation. - -## 🚀 Available Workflows - -### 1. **CI/CD Pipeline** (`.gitea/workflows/ci.yml`) - -**Triggers:** -- Push to `main` or `develop` branches -- Pull requests to `main` -- Daily scheduled runs at 2 AM UTC - -**Jobs:** -- **Test Suite**: Multi-version Python testing (3.9-3.12) -- **Code Quality**: Linting, formatting, and type checking -- **Build**: Docker image creation and testing -- **Integration Test**: Testing with VictoriaMetrics service -- **Deploy Staging**: Automatic deployment to staging (develop branch) -- **Deploy Production**: Manual deployment to production (main branch) -- **Performance Test**: Load testing after production deployment - -**Key Features:** -- ✅ Multi-Python version testing -- ✅ Docker multi-architecture builds (amd64, arm64) -- ✅ Service integration testing -- ✅ Automatic staging deployment -- ✅ Manual production approval -- ✅ Performance validation - -### 2. **Security & Dependency Updates** (`.gitea/workflows/security.yml`) - -**Triggers:** -- Daily scheduled runs at 3 AM UTC -- Manual dispatch -- Changes to requirements files or Dockerfile - -**Jobs:** -- **Dependency Scan**: Safety, Bandit, Semgrep security scans -- **Docker Security**: Trivy vulnerability scanning -- **License Check**: License compliance verification -- **Dependency Updates**: Automated update detection -- **Code Quality**: Complexity and maintainability analysis - -**Key Features:** -- 🔒 Daily security scans -- 📦 Dependency vulnerability detection -- 📄 License compliance checking -- 🔄 Automated update notifications -- 📊 Code quality metrics - -### 3. **Release Workflow** (`.gitea/workflows/release.yml`) - -**Triggers:** -- Git tags matching `v*.*.*` pattern -- Manual dispatch with version input - -**Jobs:** -- **Create Release**: Automated release creation with changelog -- **Test Release**: Comprehensive testing across Python versions -- **Build Release**: Multi-architecture Docker images with proper tags -- **Security Scan**: Trivy security scanning of release images -- **Deploy Release**: Production deployment with health checks -- **Validate Release**: Post-deployment validation and testing - -**Key Features:** -- 🏷️ Automated release creation -- 📝 Changelog generation -- 🐳 Multi-architecture Docker builds -- 🔒 Security scanning -- ✅ Comprehensive validation - -### 4. **Documentation** (`.gitea/workflows/docs.yml`) - -**Triggers:** -- Changes to documentation files -- Changes to Python source files -- Manual dispatch - -**Jobs:** -- **Validate Docs**: Link checking and structure validation -- **Generate API Docs**: OpenAPI specification generation -- **Build Sphinx Docs**: Comprehensive API documentation -- **Documentation Summary**: Build status and artifact summary - -**Key Features:** -- 📚 Automated API documentation -- 🔗 Link validation -- 📖 Sphinx documentation generation -- ✅ Documentation completeness checking - -## 🔧 Workflow Configuration - -### **Required Secrets** - -Configure these secrets in your Gitea repository settings: - -```bash -GITEA_TOKEN # Gitea access token for container registry -GITHUB_TOKEN # GitHub personal access token for better rate limits -SLACK_WEBHOOK_URL # Optional: Slack notifications -STAGING_WEBHOOK_URL # Optional: Staging deployment webhook -PRODUCTION_WEBHOOK_URL # Optional: Production deployment webhook -``` - -**Setting up GitHub Token:** -1. Go to GitHub → Settings → Developer settings → Personal access tokens -2. Generate a new token with `public_repo` scope (for public repos) or `repo` scope (for private repos) -3. Copy the token -4. In your Gitea repository → Settings → Secrets → Add `GITHUB_TOKEN` with the copied value - -### **Environment Variables** - -Key environment variables used across workflows: - -```yaml -PYTHON_VERSION: '3.11' # Default Python version -REGISTRY: git.b4l.co.th # Container registry -IMAGE_NAME: grabowski/northern-thailand-ping-river-monitor -``` - -## 📊 Workflow Status - -### **CI/CD Pipeline Status** -- **Test Coverage**: Multi-version Python testing -- **Code Quality**: Automated linting and formatting -- **Security**: Integrated security scanning -- **Deployment**: Automated staging, manual production - -### **Security Monitoring** -- **Daily Scans**: Automated vulnerability detection -- **Dependency Updates**: Proactive update notifications -- **License Compliance**: Automated license checking -- **Code Quality**: Continuous quality monitoring - -### **Release Management** -- **Automated Releases**: Tag-based release creation -- **Multi-Architecture**: Support for amd64 and arm64 -- **Security Validation**: Pre-deployment security checks -- **Health Monitoring**: Post-deployment validation - -## 🚀 Usage Examples - -### **Triggering Workflows** - -**Manual CI/CD Run:** -```bash -# Push to trigger CI/CD -git push origin main - -# Create pull request to trigger testing -git checkout -b feature/new-feature -git push origin feature/new-feature -# Create PR in Gitea UI -``` - -**Manual Security Scan:** -```bash -# Trigger via Gitea Actions UI -# Go to Actions → Security & Dependency Updates → Run workflow -``` - -**Creating a Release:** -```bash -# Create and push a tag -git tag v3.1.1 -git push origin v3.1.1 - -# Or use manual dispatch in Gitea Actions UI -``` - -### **Monitoring Workflow Results** - -**Check Workflow Status:** -1. Navigate to your repository in Gitea -2. Click on "Actions" tab -3. View workflow runs and their status - -**Download Artifacts:** -1. Click on a completed workflow run -2. Scroll to "Artifacts" section -3. Download reports and logs - -**View Security Reports:** -1. Go to Security workflow runs -2. Download security-reports artifacts -3. Review JSON reports for vulnerabilities - -## 🔍 Troubleshooting - -### **Common Issues** - -**Workflow Fails on Dependencies:** -```bash -# Check requirements.txt for version conflicts -pip-compile requirements.in -``` - -**Docker Build Fails:** -```bash -# Test Docker build locally -make docker-build -docker run --rm ping-river-monitor python run.py --test -``` - -**Security Scan Failures:** -```bash -# Run security scans locally -safety check -r requirements.txt -bandit -r src/ -``` - -**Test Failures:** -```bash -# Run tests locally -make test -python tests/test_integration.py -``` - -### **Debugging Workflows** - -**Enable Debug Logging:** -Add to workflow file: -```yaml -env: - ACTIONS_STEP_DEBUG: true - ACTIONS_RUNNER_DEBUG: true -``` - -**Check Workflow Logs:** -1. Go to failed workflow run -2. Click on failed job -3. Expand failed step to see detailed logs - -**Validate Workflow Syntax:** -```bash -# Validate YAML syntax -make validate-workflows -``` - -## 📈 Performance Optimization - -### **Caching Strategy** -- **Pip Cache**: Cached across workflow runs -- **Docker Layer Cache**: GitHub Actions cache for faster builds -- **Dependency Cache**: Cached based on requirements.txt hash - -### **Parallel Execution** -- **Matrix Builds**: Multiple Python versions tested in parallel -- **Independent Jobs**: Security scans run independently of tests -- **Conditional Execution**: Jobs skip when not needed - -### **Resource Management** -- **Timeout Settings**: Prevent hanging workflows -- **Resource Limits**: Appropriate runner sizing -- **Artifact Cleanup**: Automatic cleanup of old artifacts - -## 🔒 Security Best Practices - -### **Secret Management** -- Use Gitea repository secrets for sensitive data -- Never commit secrets to repository -- Rotate secrets regularly -- Use least-privilege access tokens - -### **Container Security** -- Multi-stage Docker builds for smaller images -- Non-root user in containers -- Regular base image updates -- Vulnerability scanning before deployment - -### **Code Security** -- Automated security scanning in CI/CD -- Dependency vulnerability monitoring -- License compliance checking -- Code quality enforcement - -## 📚 Additional Resources - -### **Gitea Actions Documentation** -- [Gitea Actions Overview](https://docs.gitea.io/en-us/usage/actions/) -- [Workflow Syntax](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions) -- [Available Actions](https://github.com/marketplace?type=actions) - -### **Project-Specific Resources** -- [Contributing Guide](../CONTRIBUTING.md) - -### **Monitoring and Alerts** -- Workflow status badges in README -- Email notifications for failures -- Slack/Discord integration for team updates -- Grafana dashboards for deployment metrics - ---- - -**Workflow Version**: v3.1.3 -**Last Updated**: 2025-08-12 -**Maintained By**: Ping River Monitor Team \ No newline at end of file diff --git a/docs/MATRIX_QUICK_START.md b/docs/MATRIX_QUICK_START.md index 2e72d17..8ef4436 100644 --- a/docs/MATRIX_QUICK_START.md +++ b/docs/MATRIX_QUICK_START.md @@ -32,13 +32,9 @@ curl -X POST https://matrix.org/_matrix/client/v3/login \ ## Step 4: Configure .env File Add these to your `.env` file: ```bash -# Matrix Alerting Configuration MATRIX_HOMESERVER=https://matrix.org MATRIX_ACCESS_TOKEN=syt_your_access_token_here MATRIX_ROOM_ID=!your_room_id:matrix.org - -# Grafana Integration (optional) -GRAFANA_URL=http://localhost:3000 ``` ## Step 5: Restart and check @@ -71,8 +67,8 @@ so a Matrix outage is retried on Matrix only and does not re-send ntfy. The old `run.py --alert-check` cron job (every 15 minutes, repeating an alert for as long as a level stayed high) is replaced by this; -`scripts/install_collector.sh` comments it out of the crontab once -ping-collector sends Matrix, keeping a backup next to the app. +`scripts/install.sh` comments it out of the crontab if it is still there, +keeping a backup next to the app. ## Troubleshooting - **HTTP 401/403** in `journalctl -u ping-collector`: the access token is invalid diff --git a/docs/NOTIFICATIONS.md b/docs/NOTIFICATIONS.md index 57d9db5..e423183 100644 --- a/docs/NOTIFICATIONS.md +++ b/docs/NOTIFICATIONS.md @@ -36,12 +36,11 @@ the public dashboard. ## How it runs -Since phase 2 of the Rust rewrite the notifier runs in `ping-collector` -(`rust/collector/src/notify.rs`) when `FORECAST_IN_COLLECTOR=true`, after each -forecast run. It is a port of `src/notify.py` with the same topics, texts and -state keys (checked word for word against Python on 751 generated messages), -so switching between the two sends nothing twice. One difference: the Rust -notifier also uses ThaiWater (HII) telemetry. For the stations whose HII twin +The notifier runs in `ping-collector` (`rust/collector/src/notify.rs`) when +`FORECAST_IN_COLLECTOR=true`, after each forecast run. It is a port of the +Python notifier it replaced in 2026-10, with the same topics, texts and state +keys (checked word for word on 751 generated messages), so the switch sent +nothing twice. One difference: it also uses ThaiWater (HII) telemetry. For the stations whose HII twin is the same sensor (P.1, P.103, P.20, P.4A, P.67, P.75, P.82, P.84, P.92, and P.81 bias-corrected), a reading newer than RID's drives the level alerts; the message then says "ThaiWater telemetry; the RID report is not in yet". RID's @@ -52,12 +51,12 @@ once after `RID_STALE_ALERT_HOURS` (18, 0 = off). With `MATRIX_*` set, the same transitions also go to a Matrix room, each once (per station, outlook, feed; not the per-topic summaries), with their own `matrix:` state keys. -With `FORECAST_IN_COLLECTOR=false`, `src/notify.py` is called once per collection cycle inside the API process -(leader only), right after the forecast precompute, so it sees exactly the -readings and forecasts the dashboard shows. Per-key last-sent state is stored -in the `notification_state` table of the monitor's own database, so a restart -or redeploy never re-sends and never misses a crossing that happened while -the service was down (the next cycle compares against the persisted state). +It runs in the same process that serves the dashboard, right after the +forecast, so it sees exactly the readings and forecasts the dashboard shows. +Per-key last-sent state is stored in the `notification_state` table of the +monitor's own database, so a restart or redeploy never re-sends and never +misses a crossing that happened while the service was down (the next cycle +compares against the persisted state). If ntfy is unreachable the transition is **not** recorded, so it is retried on the next cycle rather than silently lost. Any other failure in the notify @@ -86,8 +85,8 @@ token, and appends `NTFY_SERVER` (public URL for subscribers), and `NTFY_TOKEN` to `.env` if they are not there yet. Then: ```bash -systemctl restart water-monitor -journalctl -u water-monitor -n 20 | grep ntfy # "ntfy notifications: https://... topics ping-*" +systemctl restart ping-collector +journalctl -u ping-collector -n 20 | grep ntfy # "ntfy notifications: publish to ..., topics ping-*" curl -s 'https://ntfy.buildfor.life/ping-status/json?poll=1' # anonymous read works ``` @@ -120,21 +119,30 @@ holds back an alert. Test the pipeline before the domain is live with | `NTFY_TOKEN` | *(empty)* | bearer token if the server requires auth to publish (it does, see above) | | `PUBLIC_URL` | `https://water.buildfor.life/` | click-through target in messages | -Tunables in `src/notify.py`: `CLEAR_MARGIN_M` (0.10), `OUTLOOK_ON` / `OUTLOOK_OFF` -(0.50 / 0.25), stale feed threshold (3 h, argument to `evaluate`). +| `RID_STALE_ALERT_HOURS` | `18` | report RID lagging this long while ThaiWater is current (0 = never) | +| `MATRIX_HOMESERVER` / `MATRIX_ACCESS_TOKEN` / `MATRIX_ROOM_ID` | *(empty = off)* | the team room (docs/MATRIX_QUICK_START.md) | + +Tunables in `src/ml/export.py` (compiled into ping-collector through +`rust/forecast/flood_config.json`: regenerate with +`uv run python -m src.ml.export --config` and rebuild): `CLEAR_MARGIN_M` +(0.10), `OUTLOOK_ON` / `OUTLOOK_OFF` (0.50 / 0.25), `FEED_STALE_AFTER_H` (3), +the capacity guard and the station names in the messages. ## Testing -`tests/test_notify.py` covers the state machine: quiet river sends nothing; -crossing once, then silence while above, then all-clear; hysteresis on the way -down; escalation to danger and back; basin digest grouping; outlook on/off; -heuristic forecasts ignored; stale feed and recovery; state survives a restart -through sqlite; a failed publish is retried next cycle. +The tests in `rust/collector/src/notify.rs` cover the state machine: quiet +river sends nothing; crossing once, then silence while above, then all-clear; +hysteresis on the way down; escalation to danger and back; outlook on/off with +heuristic forecasts ignored; HII readings replacing a lagging RID reading; +stale feed only when both sources are; RID lagging reported once; a failed +publish retried next cycle; the Matrix rendering; and the 652-step replay of +the Python notifier's messages. `notification_state` round-trips against +PostgreSQL in `db_tests.rs`. -To exercise the real path against a real ntfy locally: run `ntfy serve` (any -platform, same binary), set `NTFY_SERVER`/`NTFY_TOKEN`, seed readings, and -poll the topic JSON. `scripts/e2e_notify.py` does exactly that if you want a -template. +`ping-collector notify --dry-run` shows the readings judged and what would be +sent right now, changing nothing. To exercise the real path against a real +ntfy locally: run `ntfy serve` (any platform, same binary), point +`NTFY_SERVER`/`NTFY_TOKEN` at it and poll the topic JSON. ## Why ntfy and not … diff --git a/docs/RUST_REWRITE.md b/docs/RUST_REWRITE.md index b975c4f..d7f1871 100644 --- a/docs/RUST_REWRITE.md +++ b/docs/RUST_REWRITE.md @@ -1,9 +1,17 @@ -# A Rust rewrite: assessment +# The Rust rewrite -Status: **in progress.** Phase 1 (collector) is in production since 2026-10-17, -phase 2 (forecast + ntfy) and phase 3 (website/API + Matrix) since -2026-10-18; see -[rust/README.md](../rust/README.md). Spike: [spikes/rust-forecast](../spikes/rust-forecast/README.md). +Status: **done** (2026-10-18). The service is `ping-collector` +([rust/README.md](../rust/README.md)); Python keeps model training, +evaluation and the archive backfills. Phases 1-3 replaced the Python +collector, the forecast + notifier and the website/API one at a time, each +checked against the Python side on production data before the switch; +phase 4 removed the Python service code. The migration tooling +(`scripts/collector_parity.py`, `scripts/api_parity.py`, +`scripts/install_collector.sh` with its cutover/rollback modes, the +`spikes/rust-forecast` spike) is in git history before the cleanup commit. + +The assessment below was written before phase 1 and is kept as the record of +why and how. ## Short version @@ -87,7 +95,7 @@ This slows model iteration, which is the most active part of the repo. | 1 | `ping-collector`: RID, HII, dams, Open-Meteo | in production since 2026-10-17 23:09 (Python: `COLLECTION_ENABLED=false`) | | 2 | Forecast crate + ntfy | in production since 2026-10-18 15:08 (`FORECAST_IN_COLLECTOR=true`); live check on production data: 48/48 rows identical, 96/96 probabilities bit-identical | | 3 | Website + read API in `ping-collector`, Matrix like ntfy | in production since 2026-10-18 16:40 (`WEB_LISTEN`, `water-monitor.service` disabled); on production data 53 of 53 compared routes identical to FastAPI (`scripts/api_parity.py`) | -| 4 | Cleanup | | +| 4 | Cleanup: Python service code removed | done 2026-10-18 | 0. **Spike: done.** P.1 features + HGB inference, bit-identical. 1. **Collector daemon** (`rust/collector`, ~1.5k LOC incl. tests): RID with the @@ -96,7 +104,7 @@ This slows model iteration, which is the most active part of the repo. API runs with `COLLECTION_ENABLED=false` and re-forecasts when the newest RID reading in the database advances. ntfy moved to phase 2: the outlook alert needs the forecasts, which are still Python. Cutover and rollback - are one command each (`scripts/install_collector.sh`). + were one command each (`scripts/install_collector.sh`, since removed). 2. **Forecast crate + ntfy** (`rust/forecast`, run by `ping-collector` with `FORECAST_IN_COLLECTOR=true`): all stations, HII/rain/dam inputs, heuristic fallback, probabilities and stages, `forecast_history`, `forecast_current` @@ -130,9 +138,16 @@ This slows model iteration, which is the most active part of the repo. their own `matrix:` state keys seeded from ntfy's, replacing the `run.py --alert-check` cron job that repeated alerts every 15 minutes. Python keeps training, evaluation and the monthly retrain timer. -4. **Cleanup:** drop FastAPI/uvicorn, the Python collector, notifier, CLI and - the multi-backend adapters (SQLite, MySQL, InfluxDB, VictoriaMetrics) and - demo databases; the Python environment shrinks to the training stack. +4. **Cleanup** (done): FastAPI/uvicorn, the Python collector, notifier, Matrix + alerting, CLI (`run.py`), the multi-backend adapters (SQLite, MySQL, + InfluxDB, VictoriaMetrics), demo databases, Docker/Grafana files and the + migration scripts are gone, about 17k lines. The Python environment is the + training stack (pandas, scikit-learn, SQLAlchemy/psycopg2): `src/ml`, the + backfill clients/stores (`src/hii_collector.py`, `src/rid_reservoir.py`) + and `src/ml/predict.py` as the reference implementation the Rust forecaster + is tested against. PostgreSQL only; the Python database tests run against + a throwaway schema (`TEST_DATABASE_URL`, as the Rust ones). One installer, + `scripts/install.sh`. Decided before phase 3 (2026-10-18): Matrix stays and behaves like ntfy; `/docs` stays; SQLite dev mode goes; the monthly retrain stays on the same diff --git a/docs/STATION_MANAGEMENT_GUIDE.md b/docs/STATION_MANAGEMENT_GUIDE.md deleted file mode 100644 index 9439836..0000000 --- a/docs/STATION_MANAGEMENT_GUIDE.md +++ /dev/null @@ -1,241 +0,0 @@ -# 🏔️ Station Management Guide - Northern Thailand Ping River Monitor - -## 🎯 **Overview** - -The Northern Thailand Ping River Monitor now includes comprehensive station management capabilities, allowing you to dynamically add, update, and remove monitoring stations through the web API. - -## 🌊 **Current Coverage** - -The system currently monitors **16 water stations** along the Ping River Basin: - -### **Upper Ping River (Chiang Mai Province)** -- **P.20** - Ban Chiang Dao (บ้านเชียงดาว) -- **P.75** - Ban Chai Lat (บ้านช่อแล) -- **P.92** - Ban Muang Aut (บ้านเมืองกึ๊ด) -- **P.4A** - Ban Mae Taeng (บ้านแม่แตง) -- **P.67** - Ban Tae (บ้านแม่แต) -- **P.21** - Ban Rim Tai (บ้านริมใต้) -- **P.103** - Ring Bridge 3 (สะพานวงแหวนรอบ 3) - -### **Middle Ping River** -- **P.1** - Nawarat Bridge (สะพานนวรัฐ) - *Main reference station* -- **P.82** - Ban Sob win (บ้านสบวิน) -- **P.84** - Ban Panton (บ้านพันตน) -- **P.81** - Ban Pong (บ้านโป่ง) -- **P.5** - Tha Nang Bridge (สะพานท่านาง) - -### **Lower Ping River** -- **P.77** - Baan Sop Mae Sapuord (บ้านสบแม่สะป๊วด) -- **P.87** - Ban Pa Sang (บ้านป่าซาง) -- **P.76** - Banb Mae I Hai (บ้านแม่อีไฮ) -- **P.85** - Baan Lai Kaew (บ้านหล่ายแก้ว) - -## 🔧 **Station Management API** - -### **List All Stations** -```bash -GET /stations -``` - -**Response:** -```json -[ - { - "station_id": 1, - "station_code": "P.20", - "thai_name": "บ้านเชียงดาว", - "english_name": "Ban Chiang Dao", - "latitude": 19.36731448032191, - "longitude": 98.9688487015384, - "geohash": null, - "status": "active" - } -] -``` - -### **Get Specific Station** -```bash -GET /stations/{station_id} -``` - -### **Add New Station** -```bash -POST /stations -Content-Type: application/json - -{ - "station_code": "P.NEW", - "thai_name": "สถานีใหม่", - "english_name": "New Station", - "latitude": 18.7875, - "longitude": 99.0045, - "geohash": "w5q6uuhvfcfp25", - "status": "active" -} -``` - -### **Update Station Information** -```bash -PUT /stations/{station_id} -Content-Type: application/json - -{ - "thai_name": "ชื่อใหม่", - "english_name": "Updated Name", - "latitude": 18.8000, - "longitude": 99.0100 -} -``` - -### **Delete Station** -```bash -DELETE /stations/{station_id} -``` - -## 🧪 **Testing Station Management** - -Use the provided test script to verify station management functionality: - -```bash -# Test all station management endpoints -python test_station_management.py -``` - -This will: -1. List existing stations -2. Create a test station -3. Retrieve station details -4. Update station information -5. Verify changes -6. Delete the test station -7. Confirm deletion - -## 📊 **Station Data Model** - -### **Required Fields** -- `station_code`: Unique identifier (e.g., "P.1", "P.20") -- `thai_name`: Thai language name -- `english_name`: English language name - -### **Optional Fields** -- `latitude`: GPS latitude coordinate (-90 to 90) -- `longitude`: GPS longitude coordinate (-180 to 180) -- `geohash`: Geohash string for location -- `status`: Station status ("active", "inactive", "maintenance", "error") - -### **Validation Rules** -- Station codes must be unique -- Latitude must be between -90 and 90 -- Longitude must be between -180 and 180 -- Names cannot be empty -- Status must be valid enum value - -## 🌐 **Web Interface** - -Access the station management interface through the web dashboard: - -1. **Start the API server:** - ```bash - python run.py --web-api - ``` - -2. **Open your browser:** - - Dashboard: http://localhost:8000 - - API Documentation: http://localhost:8000/docs - -3. **Use the interactive API docs** to test station management endpoints - -## 🔄 **Integration with Data Collection** - -- **Dynamic Station Discovery**: New stations are automatically included in data collection -- **Real-time Updates**: Station information changes are reflected immediately -- **Data Continuity**: Historical data is preserved when updating station details -- **Error Handling**: Invalid stations are skipped during data collection - -## 📍 **Geographic Coverage** - -The Ping River Basin monitoring network covers: - -- **Total Distance**: ~400 km from Chiang Dao to Nakhon Sawan -- **Elevation Range**: 300m to 1,200m above sea level -- **Catchment Area**: ~25,000 km² -- **Major Cities**: Chiang Mai, Lamphun, Tak, Nakhon Sawan - -## 🚀 **Usage Examples** - -### **Add a New Upstream Station** -```bash -curl -X POST "http://localhost:8000/stations" \ - -H "Content-Type: application/json" \ - -d '{ - "station_code": "P.UPSTREAM", - "thai_name": "สถานีต้นน้ำ", - "english_name": "Upstream Station", - "latitude": 19.5000, - "longitude": 98.9000, - "status": "active" - }' -``` - -### **Update Station Coordinates** -```bash -curl -X PUT "http://localhost:8000/stations/1" \ - -H "Content-Type: application/json" \ - -d '{ - "latitude": 19.3700, - "longitude": 98.9700 - }' -``` - -### **Mark Station for Maintenance** -```bash -curl -X PUT "http://localhost:8000/stations/5" \ - -H "Content-Type: application/json" \ - -d '{ - "status": "maintenance" - }' -``` - -## 🔒 **Best Practices** - -### **Station Naming** -- Use consistent code format (P.XX) -- Include both Thai and English names -- Use descriptive location names - -### **Coordinate Accuracy** -- Use high-precision GPS coordinates (6+ decimal places) -- Verify coordinates match actual station location -- Include geohash for efficient spatial queries - -### **Status Management** -- Set status to "maintenance" during repairs -- Use "inactive" for temporarily offline stations -- Use "error" for stations with data quality issues - -### **Data Integrity** -- Test new stations before adding to production -- Backup station configuration before major changes -- Monitor data quality after station updates - -## 🎯 **Future Enhancements** - -Planned improvements for station management: - -1. **Bulk Operations** - Import/export multiple stations -2. **Station Groups** - Organize stations by river section -3. **Automated Validation** - GPS coordinate verification -4. **Historical Tracking** - Track station configuration changes -5. **Alert Integration** - Notifications for station status changes -6. **Map Interface** - Visual station management on interactive map - -## 📞 **Support** - -For station management issues: - -1. Check the API documentation at `/docs` -2. Run the test script: `python test_station_management.py` -3. Review logs for error details -4. Verify station data format and validation rules - -The station management system provides flexible control over your monitoring network while maintaining data integrity and system reliability. \ No newline at end of file diff --git a/docs/VICTORIAMETRICS_SETUP.md b/docs/VICTORIAMETRICS_SETUP.md deleted file mode 100644 index 293d0e6..0000000 --- a/docs/VICTORIAMETRICS_SETUP.md +++ /dev/null @@ -1,443 +0,0 @@ -# VictoriaMetrics Setup Guide for Thailand Water Monitor - -This guide provides comprehensive instructions for setting up VictoriaMetrics as the time-series database backend for the Thailand Water Monitor. - -## Why VictoriaMetrics? - -VictoriaMetrics is an excellent choice for water monitoring data because: - -- **High Performance**: Up to 10x faster than InfluxDB -- **Low Resource Usage**: Uses 10x less RAM than Prometheus -- **Better Compression**: 70x better compression than Prometheus -- **Prometheus Compatible**: Drop-in replacement for Prometheus -- **Easy to Deploy**: Single binary, no dependencies -- **Cost Effective**: Open source with commercial support available - -## Quick Start - -### 1. Environment Variables - -Set these environment variables to configure VictoriaMetrics: - -```bash -# Windows (PowerShell) -$env:DB_TYPE="victoriametrics" -$env:VM_HOST="localhost" -$env:VM_PORT="8428" - -# Linux/Mac -export DB_TYPE=victoriametrics -export VM_HOST=localhost -export VM_PORT=8428 -``` - -### 2. Start VictoriaMetrics with Docker - -```bash -# Simple setup -docker run -d \ - --name victoriametrics \ - -p 8428:8428 \ - -v victoria-metrics-data:/victoria-metrics-data \ - victoriametrics/victoria-metrics:latest \ - --storageDataPath=/victoria-metrics-data \ - --retentionPeriod=2y \ - --httpListenAddr=:8428 - -# Verify it's running -curl http://localhost:8428/health -``` - -### 3. Run the Water Monitor - -```bash -python water_scraper_v3.py -``` - -### 4. Access Grafana Dashboard - -```bash -# Start with Docker Compose (includes Grafana) -docker-compose -f docker-compose.victoriametrics.yml up -d - -# Access Grafana at http://localhost:3000 -# Username: admin -# Password: admin_password -``` - -## Production Setup - -### Docker Compose Configuration - -Use the provided `docker-compose.victoriametrics.yml` file: - -```bash -# Start the complete stack -docker-compose -f docker-compose.victoriametrics.yml up -d - -# Check status -docker-compose -f docker-compose.victoriametrics.yml ps - -# View logs -docker-compose -f docker-compose.victoriametrics.yml logs -f -``` - -### Manual VictoriaMetrics Configuration - -#### High-Performance Configuration - -```bash -docker run -d \ - --name victoriametrics \ - -p 8428:8428 \ - -v victoria-metrics-data:/victoria-metrics-data \ - victoriametrics/victoria-metrics:latest \ - --storageDataPath=/victoria-metrics-data \ - --retentionPeriod=2y \ - --httpListenAddr=:8428 \ - --maxConcurrentInserts=32 \ - --search.maxQueryDuration=60s \ - --search.maxConcurrentRequests=16 \ - --dedup.minScrapeInterval=30s \ - --memory.allowedPercent=80 \ - --loggerLevel=INFO \ - --loggerFormat=json \ - --search.maxSeries=1000000 \ - --search.maxPointsPerTimeseries=100000 -``` - -#### Configuration Parameters Explained - -| Parameter | Description | Recommended Value | -|-----------|-------------|-------------------| -| `--storageDataPath` | Data storage directory | `/victoria-metrics-data` | -| `--retentionPeriod` | How long to keep data | `2y` (2 years) | -| `--httpListenAddr` | HTTP listen address | `:8428` | -| `--maxConcurrentInserts` | Max concurrent inserts | `32` | -| `--search.maxQueryDuration` | Max query duration | `60s` | -| `--search.maxConcurrentRequests` | Max concurrent queries | `16` | -| `--dedup.minScrapeInterval` | Deduplication interval | `30s` | -| `--memory.allowedPercent` | Max memory usage | `80` | -| `--loggerLevel` | Log level | `INFO` | -| `--search.maxSeries` | Max time series | `1000000` | - -### Kubernetes Deployment - -```yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: victoriametrics -spec: - replicas: 1 - selector: - matchLabels: - app: victoriametrics - template: - metadata: - labels: - app: victoriametrics - spec: - containers: - - name: victoriametrics - image: victoriametrics/victoria-metrics:latest - ports: - - containerPort: 8428 - args: - - --storageDataPath=/victoria-metrics-data - - --retentionPeriod=2y - - --httpListenAddr=:8428 - - --maxConcurrentInserts=32 - volumeMounts: - - name: storage - mountPath: /victoria-metrics-data - resources: - requests: - memory: "512Mi" - cpu: "500m" - limits: - memory: "2Gi" - cpu: "2000m" - volumes: - - name: storage - persistentVolumeClaim: - claimName: victoriametrics-pvc ---- -apiVersion: v1 -kind: Service -metadata: - name: victoriametrics -spec: - selector: - app: victoriametrics - ports: - - port: 8428 - targetPort: 8428 - type: ClusterIP -``` - -## Data Queries - -### HTTP API Queries - -VictoriaMetrics provides a Prometheus-compatible HTTP API: - -```bash -# Current water levels for all stations -curl "http://localhost:8428/api/v1/query?query=water_level" - -# Water levels for specific station -curl "http://localhost:8428/api/v1/query?query=water_level{station_code=\"P.1\"}" - -# Average discharge over last hour -curl "http://localhost:8428/api/v1/query?query=avg_over_time(water_discharge[1h])" - -# High discharge alerts (>80%) -curl "http://localhost:8428/api/v1/query?query=water_discharge_percent>80" - -# Time range query (last 6 hours) -START=$(date -d '6 hours ago' +%s) -END=$(date +%s) -curl "http://localhost:8428/api/v1/query_range?query=water_level&start=${START}&end=${END}&step=300" -``` - -### PromQL Examples - -```promql -# Current water levels -water_level - -# Water level trends (last 24h) -water_level[24h] - -# Discharge rates by station -water_discharge{station_code="P.1"} - -# Average discharge across all stations -avg(water_discharge) - -# Stations with high discharge (>80%) -water_discharge_percent > 80 - -# Rate of change in water level -rate(water_level[5m]) - -# Maximum water level in last hour -max_over_time(water_level[1h]) - -# Stations with increasing water levels -increase(water_level[1h]) > 0 -``` - -## Grafana Integration - -### Data Source Configuration - -1. **Add VictoriaMetrics as Prometheus Data Source**: - - URL: `http://localhost:8428` (or `http://victoriametrics:8428` in Docker) - - Access: Server (default) - - HTTP Method: POST - -2. **Import Dashboard**: - - Use the provided `water-monitoring-dashboard.json` - - Or create custom dashboards with the queries above - -### Dashboard Panels - -The included dashboard provides: - -- **Time Series**: Water levels and discharge over time -- **Table**: Current status of all stations -- **Pie Chart**: Discharge percentage distribution -- **Gauge**: Average discharge percentage -- **Variables**: Filter by station - -## Monitoring and Maintenance - -### Health Checks - -```bash -# Check VictoriaMetrics health -curl http://localhost:8428/health - -# Check metrics endpoint -curl http://localhost:8428/metrics - -# Check configuration -curl http://localhost:8428/api/v1/status/config -``` - -### Performance Monitoring - -```bash -# Query performance stats -curl http://localhost:8428/api/v1/status/tsdb - -# Memory usage -curl http://localhost:8428/api/v1/status/runtime - -# Active queries -curl http://localhost:8428/api/v1/status/active_queries -``` - -### Backup and Restore - -```bash -# Create backup -docker exec victoriametrics /usr/bin/vmbackup \ - -storageDataPath=/victoria-metrics-data \ - -dst=fs:///backup/$(date +%Y%m%d) - -# Restore from backup -docker exec victoriametrics /usr/bin/vmrestore \ - -src=fs:///backup/20250724 \ - -storageDataPath=/victoria-metrics-data -``` - -### Log Analysis - -```bash -# View logs -docker logs victoriametrics - -# Follow logs -docker logs -f victoriametrics - -# Search for errors -docker logs victoriametrics 2>&1 | grep ERROR -``` - -## Troubleshooting - -### Common Issues - -1. **Connection Refused**: - ```bash - # Check if VictoriaMetrics is running - docker ps | grep victoriametrics - - # Check port binding - netstat -tlnp | grep 8428 - ``` - -2. **High Memory Usage**: - ```bash - # Reduce memory limit - docker run ... --memory.allowedPercent=60 ... - ``` - -3. **Slow Queries**: - ```bash - # Increase query timeout - docker run ... --search.maxQueryDuration=120s ... - ``` - -4. **Data Not Appearing**: - ```bash - # Check if data is being written - curl "http://localhost:8428/api/v1/query?query=up" - - # Check water monitor logs - tail -f water_monitor.log - ``` - -### Performance Tuning - -1. **For High Write Load**: - ```bash - --maxConcurrentInserts=64 - --insert.maxQueueDuration=60s - ``` - -2. **For High Query Load**: - ```bash - --search.maxConcurrentRequests=32 - --search.maxQueryDuration=120s - ``` - -3. **For Large Datasets**: - ```bash - --search.maxSeries=10000000 - --search.maxPointsPerTimeseries=1000000 - ``` - -## Security - -### Authentication - -VictoriaMetrics doesn't have built-in authentication. Use a reverse proxy: - -```nginx -server { - listen 80; - server_name victoriametrics.example.com; - - auth_basic "VictoriaMetrics"; - auth_basic_user_file /etc/nginx/.htpasswd; - - location / { - proxy_pass http://localhost:8428; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - } -} -``` - -### TLS/SSL - -```bash -# Use nginx or traefik for TLS termination -# Or use VictoriaMetrics with TLS: -docker run ... \ - -v /path/to/cert.pem:/cert.pem \ - -v /path/to/key.pem:/key.pem \ - victoriametrics/victoria-metrics:latest \ - --tls \ - --tlsCertFile=/cert.pem \ - --tlsKeyFile=/key.pem -``` - -## Scaling - -### Cluster Setup - -For high availability and horizontal scaling: - -```bash -# Start multiple VictoriaMetrics instances -docker run -d --name vm1 -p 8428:8428 victoriametrics/victoria-metrics:latest -docker run -d --name vm2 -p 8429:8428 victoriametrics/victoria-metrics:latest - -# Use load balancer to distribute queries -# Use vminsert/vmselect/vmstorage for true clustering -``` - -### Resource Requirements - -| Data Points/Hour | RAM | CPU | Storage/Day | -|------------------|-----|-----|-------------| -| 1,000 | 100MB | 0.1 CPU | 10MB | -| 10,000 | 500MB | 0.5 CPU | 100MB | -| 100,000 | 2GB | 1 CPU | 1GB | -| 1,000,000 | 8GB | 2 CPU | 10GB | - -## Migration - -### From InfluxDB - -```bash -# Export from InfluxDB -influx -database water_monitoring -execute "SELECT * FROM water_data" -format csv > data.csv - -# Import to VictoriaMetrics (convert to Prometheus format first) -# Use vmctl tool for migration -``` - -### From Prometheus - -```bash -# Use vmctl for direct migration -vmctl prometheus --prom-snapshot=/path/to/prometheus/data --vm-addr=http://localhost:8428 -``` - -This comprehensive setup guide should help you configure VictoriaMetrics for optimal performance with the Thailand Water Monitor system. diff --git a/grafana/dashboards/water-monitoring-dashboard.json b/grafana/dashboards/water-monitoring-dashboard.json deleted file mode 100644 index 0802170..0000000 --- a/grafana/dashboards/water-monitoring-dashboard.json +++ /dev/null @@ -1,543 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": null, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "victoriametrics-uid" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "vis": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "m" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "victoriametrics-uid" - }, - "expr": "water_level", - "interval": "", - "legendFormat": "{{station_code}} - {{station_name_en}}", - "refId": "A" - } - ], - "title": "Water Levels by Station", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "victoriametrics-uid" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "vis": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "cms" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "victoriametrics-uid" - }, - "expr": "water_discharge", - "interval": "", - "legendFormat": "{{station_code}} - {{station_name_en}}", - "refId": "A" - } - ], - "title": "Water Discharge by Station", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "victoriametrics-uid" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "displayMode": "auto", - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Water Level" - }, - "properties": [ - { - "id": "unit", - "value": "m" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Discharge" - }, - "properties": [ - { - "id": "unit", - "value": "cms" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Discharge %" - }, - "properties": [ - { - "id": "unit", - "value": "percent" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 3, - "options": { - "showHeader": true, - "sortBy": [ - { - "desc": false, - "displayName": "Station" - } - ] - }, - "pluginVersion": "8.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "victoriametrics-uid" - }, - "expr": "water_level", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "victoriametrics-uid" - }, - "expr": "water_discharge", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "victoriametrics-uid" - }, - "expr": "water_discharge_percent", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "C" - } - ], - "title": "Current Station Status", - "transformations": [ - { - "id": "merge", - "options": {} - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "__name__": true, - "job": true, - "instance": true - }, - "indexByName": { - "station_code": 0, - "station_name_en": 1, - "station_name_th": 2, - "Value #A": 3, - "Value #B": 4, - "Value #C": 5 - }, - "renameByName": { - "Value #A": "Water Level", - "Value #B": "Discharge", - "Value #C": "Discharge %", - "station_code": "Station", - "station_name_en": "English Name", - "station_name_th": "Thai Name" - } - } - } - ], - "type": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "victoriametrics-uid" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "vis": false - } - }, - "mappings": [], - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 4, - "options": { - "legend": { - "displayMode": "list", - "placement": "right" - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "victoriametrics-uid" - }, - "expr": "water_discharge_percent", - "interval": "", - "legendFormat": "{{station_code}}", - "refId": "A" - } - ], - "title": "Discharge Percentage Distribution", - "type": "piechart" - }, - { - "datasource": { - "type": "prometheus", - "uid": "victoriametrics-uid" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 70 - }, - { - "color": "red", - "value": 90 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 5, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "text": {} - }, - "pluginVersion": "8.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "victoriametrics-uid" - }, - "expr": "avg(water_discharge_percent)", - "interval": "", - "legendFormat": "Average Discharge %", - "refId": "A" - } - ], - "title": "Average Discharge Percentage", - "type": "gauge" - } - ], - "refresh": "30s", - "schemaVersion": 36, - "style": "dark", - "tags": [ - "water-monitoring", - "thailand", - "victoriametrics" - ], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "prometheus", - "uid": "victoriametrics-uid" - }, - "definition": "label_values(water_level, station_code)", - "hide": 0, - "includeAll": true, - "label": "Station", - "multi": true, - "name": "station", - "options": [], - "query": { - "query": "label_values(water_level, station_code)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - } - ] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "Asia/Bangkok", - "title": "Thailand Water Level Monitoring", - "uid": "water-monitoring-dashboard", - "version": 1, - "weekStart": "" -} diff --git a/grafana/provisioning/dashboards/dashboard.yml b/grafana/provisioning/dashboards/dashboard.yml deleted file mode 100644 index d23e98d..0000000 --- a/grafana/provisioning/dashboards/dashboard.yml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: 1 - -providers: - - name: 'water-monitoring' - orgId: 1 - folder: 'Water Monitoring' - type: file - disableDeletion: false - updateIntervalSeconds: 10 - allowUiUpdates: true - options: - path: /var/lib/grafana/dashboards diff --git a/grafana/provisioning/datasources/victoriametrics.yml b/grafana/provisioning/datasources/victoriametrics.yml deleted file mode 100644 index 68a4b77..0000000 --- a/grafana/provisioning/datasources/victoriametrics.yml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: 1 - -datasources: - - name: VictoriaMetrics - type: prometheus - access: proxy - url: http://victoriametrics:8428 - isDefault: true - editable: true - jsonData: - httpMethod: POST - queryTimeout: 60s - timeInterval: 30s - customQueryParameters: '' - secureJsonData: {} - version: 1 - uid: victoriametrics-uid diff --git a/pyproject.toml b/pyproject.toml index 0c87822..0438f77 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,122 +5,56 @@ build-backend = "setuptools.build_meta" [project] name = "northern-thailand-ping-river-monitor" version = "3.1.3" -description = "Real-time water level monitoring system for the Ping River Basin in Northern Thailand" +description = "Ping River Monitor: flood-model training, evaluation and archive backfills (the service, ping-collector, is in rust/)" readme = "README.md" license = {text = "MIT"} authors = [ - {name = "Ping River Monitor Team", email = "contact@example.com"} + {name = "buildfor.life"} ] keywords = [ "water monitoring", "hydrology", + "flood forecasting", "thailand", "ping river", - "environmental monitoring", "time series", - "fastapi", - "real-time data" ] classifiers = [ - "Development Status :: 4 - Beta", + "Development Status :: 5 - Production/Stable", "Intended Audience :: Science/Research", - "Intended Audience :: System Administrators", "Topic :: Scientific/Engineering :: Hydrology", - "Topic :: System :: Monitoring", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", "Operating System :: OS Independent", - "Environment :: Web Environment", - "Framework :: FastAPI" ] +# 3.11 only: pandas 2.0.3 ships no 3.12 wheels; psycopg2-binary 2.9.9 breaks on 3.13 requires-python = ">=3.11,<3.12" dependencies = [ - # Core dependencies "requests==2.34.2", - "schedule==1.2.0", "pandas==2.0.3", - "numpy>=1.24,<2", - # Flood forecasting (ML) + "numpy>=1.24,<2", # pandas 2.0.3 wheels are ABI-incompatible with numpy 2.x "scikit-learn==1.9.0", - # Web API framework - "fastapi==0.141.1", - "uvicorn[standard]==0.52.4", - "pydantic==2.13.5", - # Database adapters "sqlalchemy==2.0.23", - "influxdb==5.3.1", - "pymysql==1.2.0", "psycopg2-binary==2.9.9", - # Monitoring and metrics - "psutil==5.9.6" + # .env loading; rust/collector/src/envfile.rs ports exactly this version + "python-dotenv==1.1.1", ] -[project.optional-dependencies] -dev = [ - # Testing - "pytest==9.1.1", - "pytest-cov==4.1.0", - "pytest-asyncio==0.21.1", - # Code formatting and linting - "black==26.5.1", - "flake8==6.1.0", - "isort==5.12.0", - "mypy==1.7.1", - # Pre-commit hooks - "pre-commit==3.5.0", - # Development tools - "ipython==8.17.2", - "jupyter==1.0.0", - # Type stubs - "types-requests==2.33.0.20260906", - "types-python-dateutil==2.8.19.14" -] -docs = [ - "sphinx==7.2.6", - "sphinx-rtd-theme==1.3.0", - "sphinx-autodoc-typehints==1.25.2" -] -all = [ - "influxdb==5.3.1", - "pymysql==1.2.0", - "psycopg2-binary==2.9.9" -] - -[project.scripts] -ping-river-monitor = "src.main:main" -ping-river-api = "src.web_api:main" - [project.urls] -Homepage = "https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor" +Homepage = "https://water.buildfor.life/" Repository = "https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor" Issues = "https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/issues" -Documentation = "https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/wiki" [dependency-groups] dev = [ - # Testing "pytest==9.1.1", "pytest-cov==4.1.0", - "pytest-asyncio==0.21.1", - # Code formatting and linting "black==26.5.1", "flake8==6.1.0", "isort==5.12.0", - "mypy==1.7.1", - # Pre-commit hooks "pre-commit==3.5.0", - # Development tools "ipython==8.17.2", "jupyter==1.0.0", - # Type stubs - "types-requests==2.33.0.20260906", - "types-python-dateutil==2.8.19.14", - # Documentation - "sphinx==7.2.6", - "sphinx-rtd-theme==1.3.0", - "sphinx-autodoc-typehints==1.25.2", - "pyinstaller>=6.16.0", ] [tool.setuptools.packages.find] diff --git a/requirements-dev.txt b/requirements-dev.txt index 97fa80f..54f1a23 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,29 +1,8 @@ -# Development dependencies -r requirements.txt -# Testing pytest==9.1.1 pytest-cov==4.1.0 -pytest-asyncio==0.21.1 - -# Code formatting and linting black==26.5.1 flake8==6.1.0 isort==5.12.0 -mypy==1.7.1 - -# Pre-commit hooks pre-commit==3.5.0 - -# Documentation -sphinx==7.2.6 -sphinx-rtd-theme==1.3.0 -sphinx-autodoc-typehints==1.25.2 - -# Development tools -ipython==8.17.2 -jupyter==1.0.0 - -# Type stubs -types-requests==2.33.0.20260906 -types-python-dateutil==2.8.19.14 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 8a87a03..df8ee91 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,34 +1,9 @@ -# Core dependencies +# Python side: flood-model training, evaluation, backfills (pyproject.toml is +# the source; uv.lock pins the rest). The service is ping-collector (rust/). requests==2.34.2 -schedule==1.2.0 pandas==2.0.3 numpy>=1.24,<2 # pandas 2.0.3 wheels are ABI-incompatible with numpy 2.x - -# Flood forecasting (ML) scikit-learn==1.9.0 - -# Web API framework -fastapi==0.141.1 -uvicorn[standard]==0.52.4 -pydantic==2.13.5 - -# Database adapters sqlalchemy==2.0.23 -influxdb==5.3.1 -pymysql==1.2.0 psycopg2-binary==2.9.9 - -# Monitoring and metrics -psutil==5.9.6 - -# Development dependencies (optional) -pytest==9.1.1 -pytest-cov==4.1.0 -black==26.5.1 -flake8==6.1.0 -mypy==1.7.1 -pre-commit==3.5.0 - -# Documentation -sphinx==7.2.6 -sphinx-rtd-theme==1.3.0 +python-dotenv==1.1.1 diff --git a/run.py b/run.py deleted file mode 100644 index 71dc819..0000000 --- a/run.py +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env python3 -""" -Simple startup script for Thailand Water Monitor -""" - -import os -import sys - -# Add src directory to Python path -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "src")) - -if __name__ == "__main__": - from src.main import main - - main() diff --git a/rust/README.md b/rust/README.md index 4efbcb0..916892f 100644 --- a/rust/README.md +++ b/rust/README.md @@ -1,29 +1,24 @@ -# Rust services +# Rust: the service -Rewrite of the Ping River Monitor in Rust, one service at a time. The plan, -costs and the parity rules are in [docs/RUST_REWRITE.md](../docs/RUST_REWRITE.md). +Since October 2026 the whole Ping River Monitor service is one Rust binary, +`ping-collector`; Python is left with model training and evaluation +(`src/ml`) and the archive backfills. How it got here, and the parity rules +the rewrite followed: [docs/RUST_REWRITE.md](../docs/RUST_REWRITE.md). -| Crate | Replaces | Status | -| --- | --- | --- | -| `collector` (`ping-collector`) | RID, HII/ThaiWater, RID dams and Open-Meteo collection in the Python web app | phase 1: in production since 2026-10-17 | -| `forecast` (`ping-forecast`, library) | the flood forecast (`src/ml` serving path); run by `ping-collector` with ntfy | phase 2: in production since 2026-10-18 | -| `collector`: `src/web` | the FastAPI app (`src/web_api.py`): dashboard, read API, `/docs` | phase 3: in production since 2026-10-18 | +| Crate | What | +| --- | --- | +| `collector` (`ping-collector`) | collection (RID, HII/ThaiWater, RID dams, Open-Meteo), the forecast run, ntfy + Matrix notifications, the website and API (`collector/src/web`) | +| `forecast` (`ping-forecast`, library) | flood-forecast inference from the models trained in Python: features, HII gap fill, gradient-boosted trees, heuristic fallback | ## ping-collector -Collects into the same PostgreSQL tables as the Python app, so the two can be -swapped (and swapped back) with no data migration. The Python web app keeps -serving the API, forecasting and sending ntfy alerts with -`COLLECTION_ENABLED=false`, and re-forecasts whenever the newest RID reading in -the database advances. - ```text -ping-collector run # the service: startup gap fill, RID at :05 (+10 min retries - # while RID is behind), HII/dams/rain once per clock hour; - # with FORECAST_IN_COLLECTOR=true also forecast + ntfy/Matrix; - # with WEB_LISTEN the website and API +ping-collector run # the service (systemd): startup gap fill, RID at :05 (+10 min + # retries while RID is behind), HII/dams/rain once per clock + # hour; FORECAST_IN_COLLECTOR=true: forecast + ntfy/Matrix + # after each cycle with newer readings; WEB_LISTEN: website/API ping-collector serve [--listen A] [--no-analytics] - # the website and API only (default 127.0.0.1:8001) + # the website and API alone (default 127.0.0.1:8001) ping-collector once [--dry-run] # one cycle of every source; --dry-run writes nothing # and compares the RID fetch with the database ping-collector fill-gaps # re-fetch RID reports with holes @@ -33,131 +28,98 @@ ping-collector forecast [--json] [--save] [--now T] [--rain-file F] ping-collector notify --dry-run # the readings the notifier judges, and what it would send ``` -Configuration is the Python app's `.env` (read from the working directory, or -`--env-file`) with python-dotenv's rules, verified on 614 generated inputs -(`collector/testdata/`): an unparsable line is skipped with a warning giving -only its line number, never its value. Variables already set (e.g. by systemd's -`EnvironmentFile=`) win. Used keys: -`POSTGRES_CONNECTION_STRING` (or `POSTGRES_HOST/PORT/DB/USER/PASSWORD`), -`RID_POLL_OFFSET_MINUTES` (5), `RID_RETRY_MINUTES` (10), -`GAP_FILL_DAYS_ON_START` (14), `ENABLE_HII_COLLECTION`, -`ENABLE_RESERVOIR_COLLECTION`, `ENABLE_RAIN_COLLECTION`, `HII_BASIN_CODE` (6), -`STATION_CONFIG_PATH` (falls back to the bundled `src/data/stations.json`), -`SCRAPING_STATUS_FILE` (status JSON for `/scraping/status` when the website runs elsewhere), -`LOG_LEVEL` / `RUST_LOG`; for phase 2 `FORECAST_IN_COLLECTOR` (false), -`MODELS_DIR` (`models`), `NTFY_SERVER` / `NTFY_PUBLISH_URL` / `NTFY_TOPIC_PREFIX` -/ `NTFY_TOKEN` / `PUBLIC_URL` (as the Python app), `RID_STALE_ALERT_HOURS` (18, -0 = off). All timestamps are Asia/Bangkok wall-clock, whatever the server's -time zone. +Configuration: the environment, or `.env` in the working directory +(`--env-file` for another), every key with its default in +[.env.example](../.env.example). `.env` is parsed with python-dotenv's rules +(`collector/src/envfile.rs`, verified on 614 generated inputs) because the +Python tools read the same file: an unparsable line is skipped with a warning +giving only its line number, never its value. Variables already set (e.g. by +systemd's `EnvironmentFile=`) win. All timestamps are Asia/Bangkok wall-clock, +whatever the server's time zone. -Not ported: the one-off archive backfills (`scripts/backfill_*.py`), which -stay in Python. +### Collection -### Verified +The tables, upserts and parsers are the ones the Python collectors had (the +DDL is in `collector/src/db.rs`; the Python backfill stores create the same). +RID's evening report often lags 1-2 h: every poll stores every reading the +report has, yesterday's report is re-fetched while it has holes, and the +startup gap fill repairs the last `GAP_FILL_DAYS_ON_START` days. -- 20 tests: parsers, gap/schedule logic, and database tests against a real - PostgreSQL (late reports, midnight recovery, upsert revisions, NUMERIC - rounding, dam metadata preservation). -- Live parity: the Python collectors and `ping-collector` run back-to-back - against the real upstream APIs into two empty databases produced identical - rows in all 9 tables (16 stations, 1,104 RID readings, 103 + 76 HII rows, - 35 dams x 2 days, 168 rain hours). Rerun with - `scripts/collector_parity.py` after changing a parser, table or upsert on - either side. -- The Python API with `COLLECTION_ENABLED=false` served `/health`, - `/measurements/latest`, `/api/stats`, `/scraping/status` and model - forecasts from a database filled only by `ping-collector`. +### Forecast and notifications -## Forecast and notifications (phase 2) +After every cycle that brought newer RID or HII readings (and at least +hourly) the forecast runs over the last 336 h, is stored in `forecast_history` +(for skill tracking) and `forecast_current`, and the notifier compares the +readings and forecasts with `notification_state`. -With `FORECAST_IN_COLLECTOR=true` in the shared `.env`, `ping-collector` runs -the flood forecast after every cycle that brought newer RID or HII readings -(and at least hourly), stores it in `forecast_history` and `forecast_current` -(the Python API serves the latter as `/forecast`, falling back to computing -itself when it is older than `FORECAST_MAX_AGE_SECONDS`), and sends the ntfy -notifications. The Python app then neither forecasts nor notifies. - -- **Models** stay trained in Python. `src/ml/export.py` writes +- **Models** are trained in Python. `src/ml/export.py` writes `models/flood_.json.gz` next to each joblib bundle (`train_all()` - does it, `retrain.sh` promotes it, `python -m src.ml.export` exports - existing bundles); the collector reloads changed files on its next run. -- **Shared constants** (thresholds, flood stages, upstream leads, notify + does it; `python -m src.ml.export` for existing bundles); the service + reloads changed files on its next run, so a retrain needs no restart. +- **Shared constants** (thresholds, flood stages, upstream leads, notifier settings) are compiled in from `forecast/flood_config.json`, generated by `uv run python -m src.ml.export --config`. `tests/test_model_export.py` - fails while it differs from Python, and a change needs a collector rebuild. -- **Notifications**: same topics, texts, hysteresis and `notification_state` - keys as `src/notify.py`, so switching sends nothing twice. One change: for - the stations whose ThaiWater twin is the same sensor, a reading newer than - RID's (RID's report lags 2-10 h at night) drives the level alerts, marked - "ThaiWater telemetry; the RID report is not in yet". "Gauge feed stale" now - means no source has a reading for 3 h; RID alone lagging is reported once - after `RID_STALE_ALERT_HOURS`. -- Memory: 70 MB peak with all 15 models loaded (24-byte tree nodes). + fails while it differs from Python; a change needs a rebuild. +- **ntfy** topics, texts and hysteresis: [docs/NOTIFICATIONS.md](../docs/NOTIFICATIONS.md). + For the stations whose ThaiWater twin is the same sensor, a reading newer + than RID's (RID lags 2-10 h at night) drives the level alerts, marked as + ThaiWater telemetry. "Gauge feed stale" means no source has a reading for + 3 h; RID alone lagging is reported once after `RID_STALE_ALERT_HOURS`. +- **Matrix** (`MATRIX_*`): the room gets the same transitions (per station, + outlook, feed; not ntfy's per-topic summaries), each once, tracked under + its own `matrix:` keys in `notification_state`, so a Matrix outage is + retried on Matrix only. [docs/MATRIX_QUICK_START.md](../docs/MATRIX_QUICK_START.md). +- Memory: under 100 MB resident in production, all 15 models loaded + (24-byte tree nodes); the unit allows 512 MB. -Verified: +### Website and API -- `forecast/tests/parity.rs` replays fixtures from the Python production path - (`forecast/testdata/gen_forecast_fixtures.py`: the 2024 flood rising and at - its peak with RID lagging behind HII, a quiet window with dam data, a - stale-feed window, moved thresholds). HII fill, every feature and every - level are bit-identical; with all 15 local models 377 of 384 probabilities - are bit-identical and the rest within 2 ulp (`exp`). CI checks the P.1 model. -- `scripts/forecast_parity.py` does the same on the live database: Python and - `ping-collector forecast --json` at the same moment with the same rain - series. The install script runs it before `--cutover-forecast`. -- `collector/testdata/gen_notify_cases.py`: 652 hourly steps through - `src/notify.py` (thresholds, capacity guard, outlook, feed gaps, ntfy - outages); the Rust notifier sends the same 751 messages word for word. - -## Website and API (phase 3) - -`src/web` serves what the FastAPI app served: the dashboard -(`STATIC_DIR`/dashboard.html, default `src/static`, loaded at start), the -crawler files, `/static`, `/docs` + `/redoc` + `/openapi.json`, and every API -route with the same parameters, JSON (field order included), status codes, -`{"detail": ...}` errors, Cache-Control, gzip, CORS (`CORS_ALLOW_ORIGINS`), -Umami API tracking (`UMAMI_*`) and response caches (`*_CACHE_TTL_SECONDS`). -Admin routes need `ADMIN_API_KEY`. With `WEB_LISTEN=0.0.0.0:8000` it runs -inside `ping-collector run` (one process collects, forecasts, notifies and -serves; `/scraping/status` and `POST /scrape/trigger` then talk to the -collection loop directly); `serve` runs it alone. +`collector/src/web` (axum) serves the dashboard (`STATIC_DIR`/dashboard.html, +default `src/static`, loaded at start), `/static`, the crawler files, +`/docs` + `/redoc` + `/openapi.json`, and the read API, with gzip, +Cache-Control per path, CORS (`CORS_ALLOW_ORIGINS`), Umami API tracking +(`UMAMI_*`) and stale-while-revalidate response caches with byte budgets. +Station editing, `/config` and `POST /scrape/trigger` need `ADMIN_API_KEY` +(`X-API-Key` header). - Parameters are declared once per route (`web/params.rs`): the same list validates the request (pydantic-style 422s) and builds the OpenAPI document (`web/openapi.rs`), so `/docs` cannot drift from what is accepted. - The component schemas are FastAPI's (`web/openapi_components.json`). -- Python's output conventions are kept where they show: `isoformat()` - timestamps, `round()`, numpy's pairwise sums for the catchment means - (`web/pyfmt.rs`), the skill metrics' summation order. -- `scripts/api_parity.py --python URL --rust URL` compares the two live on - one database; `install_collector.sh --check-api` / `--cutover-api` run it - on the server against `ping-collector serve` on port 8001. +- The response formats are the retired FastAPI app's, kept so clients did + not notice the switch: field order, `isoformat()` timestamps, Python's + `round()`, numpy's pairwise sums for the catchment means (`web/pyfmt.rs`), + `{"detail": ...}` errors. +- An old stored forecast is served marked `X-Data-Stale: true`; `/health` + counts current ThaiWater telemetry of the same gauges while RID lags. -Matrix (`MATRIX_HOMESERVER`, `MATRIX_ACCESS_TOKEN`, `MATRIX_ROOM_ID`): the -notifier sends the room the same transitions as ntfy (per station, outlook, -feed; not ntfy's per-topic summaries), each once, with its own `matrix:` keys -in `notification_state` (seeded from the ntfy ones on first use, so nothing -already announced is repeated). It replaces the `run.py --alert-check` cron -job, which the install script comments out. - -### Develop +## Develop ```bash cd rust cargo fmt --all && cargo clippy --all-targets -- -D warnings docker run -d --name ping-pg -e POSTGRES_PASSWORD=ping -e POSTGRES_DB=ping -p 55432:5432 postgres:16 TEST_DATABASE_URL=postgres://postgres:ping@localhost:55432/ping cargo test +CARGO_TARGET_DIR=target/msrv cargo +1.88 check --all-targets --locked # the minimum version ``` Without `TEST_DATABASE_URL` the database tests skip. CI (`.gitea/workflows/rust.yml`) runs them against a PostgreSQL service and uploads the Linux binary as the `ping-collector-linux-x86_64` artifact. +- The forecast's parity with the Python reference: `forecast/tests/parity.rs` + replays fixtures from the Python production path + (`forecast/testdata/gen_forecast_fixtures.py`; regenerate after a change to + `src/ml/features.py`, `predict.py`, the HII fill or the dam frame). Features + and levels must stay bit-identical, probabilities within a few ulp. + `scripts/forecast_parity.py` compares the two on a live database. +- The notifier's texts are pinned by `collector/testdata/notify_python_cases.json.gz` + (751 messages recorded from the Python notifier before it was retired). + HTTP uses native-tls (OpenSSL on Linux), not rustls: app.rid.go.th (dams) only offers TLS 1.2 CBC cipher suites, which rustls refuses. Building on Linux needs `pkg-config` and `libssl-dev`. -### Deploy +## Deploy Needs Rust >= 1.88. Distro `cargo` packages are older (`resolver 3 is not valid` / `edition 2024` errors), so install the toolchain once with rustup: @@ -169,21 +131,15 @@ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profil cargo --version # >= 1.88 ``` -Then on the server, after `git pull` (the Python side of the switch ships in -the same commit): +Then on the server, in the checkout: ```bash +git pull cargo build --release --locked --manifest-path rust/Cargo.toml # or BIN= -sudo bash scripts/install_collector.sh # install + checks, nothing switched -sudo bash scripts/install_collector.sh --cutover # Rust collects from now on -sudo bash scripts/install_collector.sh --cutover-forecast # Rust also forecasts and notifies -sudo bash scripts/install_collector.sh --check-api # compare the Rust API with FastAPI -sudo bash scripts/install_collector.sh --cutover-api # Rust serves the website on :8000 -sudo bash scripts/install_collector.sh --rollback-api # FastAPI serves the website again -sudo bash scripts/install_collector.sh --rollback-forecast # forecasts + ntfy back to Python -sudo bash scripts/install_collector.sh --rollback # Python does everything again +sudo bash scripts/install.sh journalctl -u ping-collector -f ``` -Every install exports the models and runs `scripts/forecast_parity.py`; -`--cutover-forecast` refuses to switch if it fails (`FORCE=1` overrides). +`scripts/install.sh` installs the binary and the units, syncs the Python +training environment, runs a dry run of every source and the forecast check, +restarts the service and waits for the website (details in its header). diff --git a/rust/collector/src/config.rs b/rust/collector/src/config.rs index a288db9..f28df8e 100644 --- a/rust/collector/src/config.rs +++ b/rust/collector/src/config.rs @@ -1,5 +1,7 @@ -//! Configuration from the same environment / `.env` file as the Python app, -//! so both services can share `/opt/thailand-water-monitor/.env`. +//! Configuration from the environment / `.env` file +//! (`/opt/thailand-water-monitor/.env`), which the Python training tools +//! (src/config.py) read too. Names and defaults are the ones the Python app +//! used. use anyhow::{Context, Result, bail}; use std::env; @@ -33,10 +35,10 @@ pub struct Config { /// Days of RID holes re-fetched at startup (0 = off). pub gap_fill_days_on_start: u32, pub station_config_path: PathBuf, - /// Status JSON read by the Python API's /scraping/status. + /// Status JSON for /scraping/status when the website runs in a separate + /// process (`ping-collector serve`). pub status_file: Option, - /// FORECAST_IN_COLLECTOR: forecast and notify here; the Python API then - /// serves the stored forecast and does neither. + /// FORECAST_IN_COLLECTOR: forecast and notify (off: collect only). pub forecast_enabled: bool, /// Exported bundles (`flood_.json.gz`, src/ml/export.py). pub models_dir: PathBuf, @@ -62,7 +64,7 @@ pub struct MatrixConfig { pub room_id: String, } -/// The website and read API (src/web_api.py's settings, same names). +/// The website and read API (setting names as the retired Python app had them). #[derive(Clone, Debug)] pub struct WebConfig { /// WEB_LISTEN: `run` also serves the website on this address diff --git a/rust/collector/src/db.rs b/rust/collector/src/db.rs index 40cdab9..aa28744 100644 --- a/rust/collector/src/db.rs +++ b/rust/collector/src/db.rs @@ -30,8 +30,9 @@ pub async fn connect(url: &str, max_connections: u32) -> Result { .context("connecting to PostgreSQL") } -/// Idempotent DDL, identical to what the Python app creates (so either side -/// can start first against an empty database). +/// Idempotent DDL, identical to what the Python app created and the Python +/// backfill stores (src/hii_collector.py, src/rid_reservoir.py) still create, +/// so either can start first against an empty database. const SCHEMA: &[&str] = &[ "CREATE TABLE IF NOT EXISTS stations ( id SERIAL PRIMARY KEY, @@ -145,12 +146,12 @@ const SCHEMA: &[&str] = &[ created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (as_of, station_code, horizon_hours))", "CREATE INDEX IF NOT EXISTS idx_forecast_history_station ON forecast_history(station_code, as_of)", - // src/notify.py + // notify.rs "CREATE TABLE IF NOT EXISTS notification_state ( key VARCHAR(64) PRIMARY KEY, state VARCHAR(16) NOT NULL, value NUMERIC(8,3), updated_at TIMESTAMP NOT NULL)", - // The current /forecast payload, read by the Python API when - // FORECAST_IN_COLLECTOR=true (one row, id 'all'). + // The current /forecast payload (one row, id 'all'), served by + // `ping-collector serve` when the website runs as its own process. "CREATE TABLE IF NOT EXISTS forecast_current ( id VARCHAR(16) PRIMARY KEY, computed_at TIMESTAMP NOT NULL, @@ -727,7 +728,7 @@ pub async fn save_forecast_history( Ok(rows.len()) } -/// The payload the Python API serves as /forecast. +/// The payload /forecast serves (see `forecast_current` above). pub async fn save_forecast_current( pool: &PgPool, computed_at: NaiveDateTime, diff --git a/rust/collector/src/envfile.rs b/rust/collector/src/envfile.rs index 1a5248c..82b3dbe 100644 --- a/rust/collector/src/envfile.rs +++ b/rust/collector/src/envfile.rs @@ -2,7 +2,8 @@ //! interpolate `${VAR}` / `${VAR:-default}`, never override variables that //! are already set). //! -//! The Python app reads the same file, so the two must agree on every value. +//! The Python tools (src/config.py) read the same file, so the two must agree +//! on every value. //! dotenvy (the usual crate) is stricter: a stray `'` or `"` inside an //! unquoted value makes it swallow the rest of the file and abort, and its //! error message echoes the line, password included. Here an unparsable diff --git a/rust/collector/src/forecasting.rs b/rust/collector/src/forecasting.rs index e77d75b..609fd7a 100644 --- a/rust/collector/src/forecasting.rs +++ b/rust/collector/src/forecasting.rs @@ -1,5 +1,5 @@ -//! Forecasting and notifications in the collector (FORECAST_IN_COLLECTOR=true), -//! replacing the Python API's forecast_notify_task. +//! Forecasting and notifications in the collector (FORECAST_IN_COLLECTOR=true); +//! the port of the retired Python app's forecast_notify_task. //! //! After each collection cycle that brought newer RID or HII readings, and at //! least once per clock hour: load the last 336 h (RID plus the HII mirror diff --git a/rust/collector/src/main.rs b/rust/collector/src/main.rs index 5994930..e0783b1 100644 --- a/rust/collector/src/main.rs +++ b/rust/collector/src/main.rs @@ -1,11 +1,11 @@ -//! ping-collector: the Ping River Monitor's data collection service. +//! ping-collector: the Ping River Monitor service. //! -//! Replaces the collection half of the Python web app (RID hourly gauges, -//! HII/ThaiWater rain + water level, RID dams, Open-Meteo rain) and, with -//! FORECAST_IN_COLLECTOR=true, the flood forecast (rust/forecast) and the -//! ntfy/Matrix notifications, and with WEB_LISTEN the website and read API -//! (the port of the FastAPI app). Writes the same PostgreSQL tables as the -//! Python app did. See docs/RUST_REWRITE.md. +//! Collects RID hourly gauges, HII/ThaiWater rain + water level, RID dams +//! and Open-Meteo rain into PostgreSQL; with FORECAST_IN_COLLECTOR=true runs +//! the flood forecast (rust/forecast, models trained in Python) and the +//! ntfy/Matrix notifications; with WEB_LISTEN serves the website and API. +//! It replaced the Python app in 2026-10 (same tables, routes and messages); +//! see docs/RUST_REWRITE.md. mod config; mod dams; diff --git a/rust/collector/src/notify.rs b/rust/collector/src/notify.rs index fe5e366..1b80476 100644 --- a/rust/collector/src/notify.rs +++ b/rust/collector/src/notify.rs @@ -1,12 +1,13 @@ //! Public flood notifications over ntfy, and the same transitions to a -//! Matrix room: the port of src/notify.py, run by the collector when -//! FORECAST_IN_COLLECTOR=true. (Matrix replaces the old `run.py -//! --alert-check` cron job, which repeated an alert on every run while a -//! level stayed high.) +//! Matrix room, run by the collector when FORECAST_IN_COLLECTOR=true. A port +//! of the retired Python notifier (src/notify.py, removed after the 2026-10 +//! cutover); its settings still come from src/ml/export.py via +//! flood_config.json. (Matrix replaced the old `run.py --alert-check` cron +//! job, which repeated an alert on every run while a level stayed high.) //! //! Same topics, texts, hysteresis, capacity guard, outlook and persisted -//! state keys (`notification_state`) as Python, so a cutover neither repeats -//! nor drops a message. One change: for stations whose HII/ThaiWater twin is +//! state keys (`notification_state`) as Python had, so the cutover neither +//! repeated nor dropped a message. One change: for stations whose HII/ThaiWater twin is //! the same sensor, a reading newer than RID's (RID's report often lags 2-10 h //! at night) is used for the level alerts and the feed check. The feed is //! "stale" only when neither source has a reading for 3 h; RID lagging alone @@ -1011,8 +1012,10 @@ mod tests { ); } - /// 652 hourly steps from src/notify.py itself (testdata/gen_notify_cases.py): - /// the same notifications, word for word, in the same order. + /// 652 hourly steps recorded from the retired Python notifier (generator: + /// testdata/gen_notify_cases.py, in git history before the cleanup): the + /// same notifications, word for word, in the same order. A deliberate + /// text change updates the fixture with the new expected messages. #[test] fn matches_python_notify_on_generated_sequences() { use std::io::Read; diff --git a/rust/collector/src/service.rs b/rust/collector/src/service.rs index ae9114f..e870ad9 100644 --- a/rust/collector/src/service.rs +++ b/rust/collector/src/service.rs @@ -403,7 +403,7 @@ impl Collector { pub async fn notify(&self, dry_run: bool) -> Result<()> { if !dry_run && !self.cfg.forecast_enabled { bail!( - "the Python app notifies while FORECAST_IN_COLLECTOR is not true; \ + "notifications are off while FORECAST_IN_COLLECTOR is not true; \ use --dry-run" ); } diff --git a/rust/collector/src/web/cache.rs b/rust/collector/src/web/cache.rs index 58a32ba..970eb0a 100644 --- a/rust/collector/src/web/cache.rs +++ b/rust/collector/src/web/cache.rs @@ -1,4 +1,5 @@ -//! Stale-while-revalidate response cache, as `_cached_swr` in src/web_api.py: +//! Stale-while-revalidate response cache, as the retired FastAPI app's +//! `_cached_swr`: //! //! - fresh entry: served; //! - expired entry (no older than `max_stale`): served at once, marked stale, diff --git a/rust/collector/src/web/mod.rs b/rust/collector/src/web/mod.rs index bbcd7d4..7387d9f 100644 --- a/rust/collector/src/web/mod.rs +++ b/rust/collector/src/web/mod.rs @@ -1,8 +1,9 @@ -//! The website and read API: the port of the FastAPI app (src/web_api.py). +//! The website and read API: the port of the retired FastAPI app. //! //! Same routes, parameters, JSON fields, status codes, cache headers and -//! response caches as the Python app, checked route by route against it by -//! scripts/api_parity.py. It runs inside `ping-collector run` (WEB_LISTEN) +//! response caches as the Python app had; before the cutover (2026-10-18) a +//! route-by-route comparison of the two on production data found no +//! difference. It runs inside `ping-collector run` (WEB_LISTEN) //! or alone (`ping-collector serve`). /docs is the same Swagger UI, fed by an //! OpenAPI document built from the route table that also validates the //! parameters (openapi.rs, params.rs). diff --git a/rust/collector/src/web/tests.rs b/rust/collector/src/web/tests.rs index d635ac8..c155ad4 100644 --- a/rust/collector/src/web/tests.rs +++ b/rust/collector/src/web/tests.rs @@ -1,7 +1,7 @@ //! Route tests: the router as the server runs it, against a throwaway //! schema (TEST_DATABASE_URL; see db_tests.rs) or, for the routes that do //! not touch the database, a pool that never connects. Output formats are -//! the FastAPI app's (scripts/api_parity.py compares the two live). +//! the retired FastAPI app's (field order, isoformat, errors). use super::{AppState, router}; use crate::config::Config; diff --git a/rust/collector/testdata/gen_notify_cases.py b/rust/collector/testdata/gen_notify_cases.py deleted file mode 100644 index 7f8f3ba..0000000 --- a/rust/collector/testdata/gen_notify_cases.py +++ /dev/null @@ -1,127 +0,0 @@ -"""Notification cases for rust/collector/src/notify.rs, from src/notify.py itself. - - uv run python rust/collector/testdata/gen_notify_cases.py - -Random hourly sequences around the thresholds (capacity guard, hysteresis, -danger escalation, the P.1 outlook, feed gaps, ntfy outages for some topics) -run through notify.evaluate with an in-memory state; every delivered -notification is recorded. The Rust test replays the same steps (RID readings -only, so the ThaiWater additions stay out of play) and must deliver exactly -the same messages. -""" - -import datetime -import gzip -import json -import random -import sys -from pathlib import Path - -HERE = Path(__file__).resolve().parent -sys.path.insert(0, str(HERE.parents[2])) - -from src import notify # noqa: E402 - - -class Publisher(notify.NtfyPublisher): - def __init__(self): - super().__init__("http://ntfy.invalid", prefix="ping") - self.down = set() - - def publish(self, n): - return n.topic not in self.down - - -STATIONS = { # around (warning, danger) - "P.1": (3.7, 4.2), - "P.77": (4.3, 4.9), - "P.21": (3.2, 3.6), - "P.20": (2.35, 2.8), -} - - -def main(): - rng = random.Random(20261018) - sequences = [] - for _ in range(40): - state = notify.InMemoryState() - pub = Publisher() - t = datetime.datetime(2026, 9, 1) + datetime.timedelta( - hours=rng.randrange(2000) - ) - codes = rng.sample(sorted(STATIONS), rng.randint(1, 4)) - levels = {c: STATIONS[c][0] + rng.uniform(-0.6, 0.6) for c in codes} - p_out = rng.random() - steps = [] - for _ in range(rng.randint(8, 25)): - gap = rng.choice([1, 1, 1, 1, 2, 4, 6]) - t += datetime.timedelta(hours=gap) - # the clock sometimes runs ahead of the data (a stale feed) - now = t + datetime.timedelta(minutes=rng.choice([5, 5, 30, 200, 300])) - readings = [] - for c in codes: - levels[c] += rng.uniform(-0.25, 0.25) - pct = rng.choice([None, rng.uniform(5, 120)]) - level = round(levels[c], 2) if rng.random() > 0.05 else None - readings.append( - { - "station_code": c, - "timestamp": t, - "water_level": level, - "discharge_percent": None if pct is None else round(pct, 2), - } - ) - p_out = min(1.0, max(0.0, p_out + rng.uniform(-0.3, 0.3))) - forecasts = [ - { - "station_code": "P.1", - "horizon_hours": rng.choice([24, 24, 24, 12]), - "p_warning": p_out, - "predicted_max_level": round(rng.uniform(2.5, 5.0), 3), - "source": rng.choice(["model", "model", "model", "heuristic"]), - } - ] - pub.down = set( - rng.sample( - [ - "ping-p1-warning", - "ping-warning", - "ping-status", - "ping-p1-outlook", - ], - rng.choice([0, 0, 0, 1]), - ) - ) - sent = notify.evaluate(readings, forecasts, state, pub, now=now) - steps.append( - { - "now": now.isoformat(), - "readings": [ - {**r, "timestamp": r["timestamp"].isoformat()} for r in readings - ], - "forecasts": forecasts, - "down": sorted(pub.down), - "sent": [ - { - "topic": n.topic, - "title": n.title, - "message": n.message, - "priority": n.priority, - "tags": n.tags or [], - } - for n in sent - ], - } - ) - sequences.append(steps) - out = HERE / "notify_python_cases.json.gz" - with open(out, "wb") as raw, gzip.GzipFile(fileobj=raw, mode="wb", mtime=0) as gz: - gz.write(json.dumps({"sequences": sequences}).encode()) - total = sum(len(s["sent"]) for seq in sequences for s in seq) - print( - f"{len(sequences)} sequences, {sum(map(len, sequences))} steps, {total} notifications" - ) - - -if __name__ == "__main__": - main() diff --git a/scripts/api_parity.py b/scripts/api_parity.py deleted file mode 100644 index 85692b6..0000000 --- a/scripts/api_parity.py +++ /dev/null @@ -1,389 +0,0 @@ -#!/usr/bin/env python3 -"""Compare the Rust website/API with the FastAPI app, route by route. - -Phase 3 of docs/RUST_REWRITE.md: before the Rust service takes over port -8000, both answer the same requests against the same database and every -response is compared: status, content type, Cache-Control and the JSON body -(floats exactly; lists of database rows without ORDER BY as sets). Static -files and the dashboard byte for byte. - - # the FastAPI app on :8000, `ping-collector serve --listen 127.0.0.1:8001` - uv run python scripts/api_parity.py \\ - --python http://127.0.0.1:8000 --rust http://127.0.0.1:8001 - -Query parameters vary per run (limits, hours) so the FastAPI app's response -caches (up to 15 min) answer with fresh data, not an older copy. Data the -collector writes during the run can still make a route differ once: a -mismatch is retried after a pause before it counts. - -Intended differences (the Rust side may answer differently): - /health a lagging RID report with current ThaiWater telemetry for - the same gauges is healthy (as the notifier, phase 2) - /scraping/status extra fields (collector version, forecast run) - /metrics process counters: compared by shape only - /sensors/thaiwater live upstream: compared, a difference only warns - /api/hii/rainfall/catchment correlation within 1e-9 (numpy uses BLAS) - /api/stats newer counts on the Rust side are accepted (the FastAPI app - caches this one for 5 minutes) - *.geojson content type application/geo+json (FastAPI: octet-stream) - -Exit status 0 when every hard check passed. -""" - -import argparse -import random -import sys -import time -from typing import Any, Callable, List, Optional, Tuple - -import requests - -SOFT = "soft" # a difference warns, does not fail - - -def get(base: str, path: str, method: str = "GET", **kw) -> requests.Response: - last = None - for _ in range(3): - try: - return requests.request(method, base + path, timeout=120, **kw) - except requests.RequestException as e: # connection reset etc. - last = e - time.sleep(1) - raise last - - -def first_diff(a: Any, b: Any, path: str = "$", rel: float = 0.0) -> Optional[str]: - """Where two decoded JSON values differ (None if equal).""" - if isinstance(a, bool) or isinstance(b, bool): - return None if a is b else f"{path}: {a!r} != {b!r}" - if isinstance(a, (int, float)) and isinstance(b, (int, float)): - if a == b: - return None - if rel and abs(a - b) <= rel * max(abs(a), abs(b)): - return None - return f"{path}: {a!r} != {b!r}" - if type(a) is not type(b): - return f"{path}: {type(a).__name__} {a!r} != {type(b).__name__} {b!r}" - if isinstance(a, dict): - if list(a.keys()) != list(b.keys()): - if set(a.keys()) != set(b.keys()): - return f"{path}: keys {sorted(set(a) ^ set(b))} only on one side" - for k in a: - d = first_diff(a[k], b[k], f"{path}.{k}", rel) - if d: - return d - return None - if isinstance(a, list): - if len(a) != len(b): - return f"{path}: {len(a)} items != {len(b)} items" - for i, (x, y) in enumerate(zip(a, b)): - d = first_diff(x, y, f"{path}[{i}]", rel) - if d: - return d - return None - return None if a == b else f"{path}: {a!r} != {b!r}" - - -def media_type(r: requests.Response) -> str: - return r.headers.get("content-type", "").split(";")[0].strip() - - -class Case: - def __init__( - self, - path: str, - method: str = "GET", - compare: Optional[Callable] = None, - sort_key: Optional[str] = None, - soft: bool = False, - body: Any = None, - headers: Optional[dict] = None, - ): - self.path, self.method, self.compare = path, method, compare - self.sort_key, self.soft, self.body = sort_key, soft, body - self.headers = headers or {} - - def run(self, py: str, rs: str) -> Tuple[Optional[str], Optional[str]]: - """(hard difference, warning)""" - kw = {"headers": self.headers} - if self.body is not None: - kw["json"] = self.body - a = get(py, self.path, self.method, **kw) - b = get(rs, self.path, self.method, **kw) - if a.status_code != b.status_code: - return self._fail( - f"status {a.status_code} != {b.status_code} " - f"({a.text[:120]!r} / {b.text[:120]!r})" - ) - if media_type(a) != media_type(b): - warn = f"content-type {media_type(a)!r} != {media_type(b)!r}" - if media_type(a).startswith("application/json") or media_type(b).startswith( - "application/json" - ): - return self._fail(warn) - return None, warn - if self.method == "GET" and a.status_code == 200: - ca, cb = a.headers.get("cache-control"), b.headers.get("cache-control") - if ca != cb: - return self._fail(f"Cache-Control {ca!r} != {cb!r}") - if self.compare is not None: - d = self.compare(a, b) - elif media_type(a) == "application/json": - ja, jb = a.json(), b.json() - if a.status_code == 422: - # pydantic's messages are approximated; type and location must agree - ja = [(e["type"], e["loc"]) for e in ja["detail"]] - jb = [(e["type"], e["loc"]) for e in jb["detail"]] - if any(t.startswith("date") for t, _ in ja): - ja, jb = [loc for _, loc in ja], [loc for _, loc in jb] - elif self.sort_key and isinstance(ja, list): - ja = sorted(ja, key=lambda r: str(r.get(self.sort_key))) - jb = sorted(jb, key=lambda r: str(r.get(self.sort_key))) - d = first_diff(ja, jb) - else: - d = ( - None - if a.content == b.content - else f"bodies differ ({len(a.content)} / {len(b.content)} bytes)" - ) - return self._fail(d) if d else (None, None) - - def _fail(self, d: str) -> Tuple[Optional[str], Optional[str]]: - return (None, d) if self.soft else (d, None) - - -def compare_health(a, b) -> Optional[str]: - ja, jb = a.json(), b.json() - if set(ja["checks"]) != set(jb["checks"]): - return f"checks {sorted(ja['checks'])} != {sorted(jb['checks'])}" - for name, ca in ja["checks"].items(): - cb = jb["checks"][name] - if ca["status"] == cb["status"]: - continue - # the intended change: RID lagging, ThaiWater current - if ( - name == "database" - and ca["status"] == "degraded" - and cb["status"] == "healthy" - and "ThaiWater telemetry current" in cb["message"] - ): - continue - if name in ("api", "memory"): - continue # probes of the RID API / own process, compared loosely - return f"{name}: {ca} != {cb}" - return None - - -def compare_metrics(a, b) -> Optional[str]: - ja, jb = a.json(), b.json() - return None if set(ja) == set(jb) else f"keys {sorted(ja)} != {sorted(jb)}" - - -def compare_status(a, b) -> Optional[str]: - ja, jb = a.json(), b.json() - for k, v in ja.items(): - if jb.get(k) != v: - return f"{k}: {v!r} != {jb.get(k)!r}" - return None - - -def compare_skill(a, b) -> Optional[str]: - ja, jb = a.json(), b.json() - va, vb = ja.pop("verified_until"), jb.pop("verified_until") - gap = abs( - time.mktime(time.strptime(va[:19], "%Y-%m-%dT%H:%M:%S")) - - time.mktime(time.strptime(vb[:19], "%Y-%m-%dT%H:%M:%S")) - ) - if gap > 300: - return f"verified_until {va} != {vb}" - return first_diff(ja, jb) - - -def compare_stats(a, b) -> Optional[str]: - """Exact, except that the FastAPI app may answer from its 5-minute cache - while the collector has written newer rows since.""" - ja, jb = a.json(), b.json() - if ja == jb: - return None - for k, v in ja.items(): - w = jb.get(k) - if k.endswith("_measurements") or k == "days_spanned": - if not isinstance(w, int) or w < v: - return f"{k}: {v!r} > {w!r}" - elif k == "last_timestamp": - if not isinstance(w, str) or w < v: - return f"{k}: {v!r} > {w!r}" - elif k == "coverage_percent": - if not isinstance(w, (int, float)) or abs(w - v) > 0.2: - return f"{k}: {v!r} != {w!r}" - elif v != w: - return f"{k}: {v!r} != {w!r}" - return None - - -def compare_catchment(a, b) -> Optional[str]: - return first_diff(a.json(), b.json(), rel=1e-9) - - -def compare_openapi(a, b) -> Optional[str]: - ja, jb = a.json(), b.json() - if ja["info"]["title"] != jb["info"]["title"]: - return "title differs" - ops = lambda spec: { # noqa: E731 - (m, p): op for p, item in spec["paths"].items() for m, op in item.items() - } - oa, ob = ops(ja), ops(jb) - if set(oa) != set(ob): - return f"operations only on one side: {sorted(set(oa) ^ set(ob))}" - for k, op in oa.items(): - other = ob[k] - for field in ("operationId", "summary", "parameters", "requestBody"): - d = first_diff(op.get(field), other.get(field), f"{k}.{field}") - if d: - return d - if set(op["responses"]) != set(other["responses"]): - return f"{k}: responses {sorted(op['responses'])} != {sorted(other['responses'])}" - for name in ja["components"]["schemas"]: - if name not in jb["components"]["schemas"]: - return f"schema {name} missing" - return None - - -def compare_html_docs(a, b) -> Optional[str]: - return ( - None - if "swagger-ui" in b.text and "/openapi.json" in b.text - else "no Swagger UI" - ) - - -def cases(admin_key: Optional[str]) -> List[Case]: - r = random.Random(time.time()) - hours = r.randint(150, 190) # fresh keys for the Python app's caches - limit = r.randint(40000, 49999) - feed_hours = r.randint(20, 30) - skill_station = r.choice( - ["P.67", "P.103", "P.20", "P.75", "P.92", "P.21", "P.5", "P.82"] - ) - day = time.strftime("%Y-%m-%d", time.localtime(time.time() - 86400 * 3)) - day2 = time.strftime("%Y-%m-%d", time.localtime(time.time() - 86400)) - out = [ - Case("/"), - Case("/robots.txt"), - Case("/llms.txt"), - Case("/sitemap.xml"), - Case("/static/ping-river-network.geojson", soft=True), - Case("/static/custom-markers.json"), - Case("/static/flood-zones-p1.geojson", soft=True), - Case("/static/flood-2024-all.json"), - Case("/static/nope.json"), - Case("/openapi.json", compare=compare_openapi), - Case("/docs", compare=compare_html_docs), - Case("/health", compare=compare_health), - Case("/metrics", compare=compare_metrics), - Case("/stations"), - Case("/stations/8"), - Case("/stations/1"), - Case("/stations/999"), - Case("/stations/abc"), - Case(f"/measurements/latest?limit={r.randint(200, 499)}"), - Case(f"/measurements/latest?limit={r.randint(1, 15)}"), - Case(f"/measurements/history/P.1?hours={hours}&limit={limit}"), - Case(f"/measurements/history/P.67?hours={hours + 1}"), - Case(f"/measurements/history/P.1?start={day}&end={day2}&limit={limit - 1}"), - Case(f"/measurements/history/P.1?start={day}&limit={r.randint(5, 50)}"), - Case(f"/measurements/history/P.1?start={day2}&end={day}"), - Case(f"/measurements/history/NOPE?hours={hours}"), - Case("/measurements/history/P.1?hours=0"), - Case("/measurements/history/P.1?limit=100001"), - Case("/measurements/history/P.1?start=yesterday"), - Case(f"/measurements/station/P.1?hours={r.randint(20, 48)}"), - Case("/measurements/station/P.103?hours=6&limit=2"), - Case(f"/api/hii/rainfall/latest?hours={feed_hours}", sort_key="station_id"), - Case( - f"/api/hii/waterlevel/latest?hours={feed_hours + 1}", sort_key="station_id" - ), - Case("/api/hii/waterlevel/latest?hours=500"), - Case( - f"/api/hii/rainfall/catchment?days={r.randint(15, 30)}", - compare=compare_catchment, - ), - Case("/sensors/thaiwater", sort_key="id", soft=True), - Case("/forecast"), - Case(f"/api/forecast/history/P.1?hours={hours}&horizon=24"), - Case(f"/api/forecast/history/P.103?hours={feed_hours}"), - Case(f"/api/forecast/history/P.1?start={day}&end={day2}&horizon=6"), - Case("/api/forecast/history/P.1?horizon=99"), - Case("/api/notifications"), - # a station/horizon pair the dashboard does not ask for: not in the - # FastAPI app's 15-minute cache; the dashboard's own one may be - Case( - f"/api/forecast/skill?station_code={skill_station}" - f"&horizon={r.randint(1, 48)}", - compare=compare_skill, - ), - Case( - "/api/forecast/skill?station_code=P.1&horizon=24", - compare=compare_skill, - soft=True, - ), - Case("/api/forecast/skill?horizon=0"), - Case("/api/stats", compare=compare_stats), - Case("/scraping/status", compare=compare_status), - Case("/nope"), - Case("/forecast", method="DELETE"), - Case("/config"), - Case("/scrape/trigger", method="POST"), - Case("/stations", method="POST", body={"station_code": "X"}), - Case("/stations/8", method="DELETE"), - ] - if admin_key: - key = {"X-API-Key": admin_key} - out += [ - Case("/config", headers={"X-API-Key": "wrong"}), - Case("/stations", method="POST", body={"thai_name": 1}, headers=key), - Case("/stations/999", method="PUT", body={}, headers=key), - ] - return out - - -def main() -> int: - ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) - ap.add_argument("--python", default="http://127.0.0.1:8000") - ap.add_argument("--rust", default="http://127.0.0.1:8001") - ap.add_argument( - "--admin-key", help="ADMIN_API_KEY, to compare the admin errors too" - ) - ap.add_argument("--retries", type=int, default=2) - ap.add_argument("--pause", type=float, default=30.0, help="seconds before a retry") - args = ap.parse_args() - - failed, warned, passed = [], [], 0 - for case in cases(args.admin_key): - label = f"{case.method} {case.path}" - for attempt in range(args.retries + 1): - try: - hard, warn = case.run(args.python, args.rust) - except Exception as e: # noqa: BLE001 - report and go on - hard, warn = f"{type(e).__name__}: {e}", None - if not hard or attempt == args.retries: - break - print(f" retry {label}: {hard}", flush=True) - time.sleep(args.pause) - if hard: - failed.append((label, hard)) - print(f"FAIL {label}: {hard}", flush=True) - elif warn: - warned.append((label, warn)) - print(f"warn {label}: {warn}", flush=True) - else: - passed += 1 - print( - f"{passed} identical, {len(warned)} warnings, {len(failed)} different " - f"({args.python} vs {args.rust})" - ) - return 1 if failed else 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/backfill_rain_db.py b/scripts/backfill_rain_db.py index e1f2c7a..fc5abe7 100644 --- a/scripts/backfill_rain_db.py +++ b/scripts/backfill_rain_db.py @@ -2,7 +2,7 @@ """Backfill the openmeteo_rain table with the full 2021+ catchment history. Usage: - uv run scripts/backfill_rain_db.py # DB from Config/.env + uv run scripts/backfill_rain_db.py # DB from POSTGRES_* / .env uv run scripts/backfill_rain_db.py --db-url postgresql://... """ @@ -13,7 +13,7 @@ import sys sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) -from src.config import Config +from src.config import database_url from src.ml.rain import backfill_db @@ -25,21 +25,17 @@ def main(argv=None) -> int: logging.basicConfig( level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s" ) - if args.db_url: - # postgresql+psycopg2://... -> postgresql (driver suffix is not a dialect) - connection_string = args.db_url - db_type = args.db_url.split(":", 1)[0].split("+", 1)[0] - else: - cfg = Config.get_database_config() - if cfg["type"] not in ("sqlite", "postgresql", "mysql"): - print(f"requires a SQL database, got {cfg['type']}", file=sys.stderr) - return 1 - connection_string, db_type = cfg["connection_string"], cfg["type"] + connection_string = args.db_url or database_url() + if not connection_string: + print( + "no database: set POSTGRES_CONNECTION_STRING or POSTGRES_*", file=sys.stderr + ) + return 1 from sqlalchemy import create_engine engine = create_engine(connection_string, pool_pre_ping=True) - saved = backfill_db(engine, db_type) + saved = backfill_db(engine) print(f"backfilled {saved} hourly rows into openmeteo_rain") return 0 if saved else 1 diff --git a/scripts/backfill_rid_reservoir.py b/scripts/backfill_rid_reservoir.py index 17f3f30..cbe4bc5 100644 --- a/scripts/backfill_rid_reservoir.py +++ b/scripts/backfill_rid_reservoir.py @@ -3,7 +3,7 @@ Two paths, both idempotent and both skipping what is already stored, so a rerun repairs holes left by transient failures and is safe alongside the -hourly live collector: +live collector (ping-collector, today + yesterday): --dam-id (default: Mae Ngat) one dam, whole range, via api/dam — a handful of requests for the entire 2009-today archive @@ -26,13 +26,8 @@ import sys sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) -from src.config import Config -from src.rid_reservoir import ( - MAE_NGAT_DAM_ID, - RidReservoirStore, - backfill, - backfill_dam, -) +from src.config import database_url +from src.rid_reservoir import MAE_NGAT_DAM_ID, RidReservoirStore, backfill, backfill_dam DEFAULT_START = datetime.date(2018, 8, 1) # start of the water_measurements grid @@ -71,18 +66,14 @@ def main(argv=None) -> int: logging.basicConfig( level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s" ) - if args.db_url: - # postgresql+psycopg2://... -> postgresql (driver suffix is not a dialect) - connection_string = args.db_url - db_type = args.db_url.split(":", 1)[0].split("+", 1)[0] - else: - cfg = Config.get_database_config() - if cfg["type"] not in ("sqlite", "postgresql", "mysql"): - print(f"requires a SQL database, got {cfg['type']}", file=sys.stderr) - return 1 - connection_string, db_type = cfg["connection_string"], cfg["type"] + connection_string = args.db_url or database_url() + if not connection_string: + print( + "no database: set POSTGRES_CONNECTION_STRING or POSTGRES_*", file=sys.stderr + ) + return 1 - store = RidReservoirStore(connection_string, db_type) + store = RidReservoirStore(connection_string) if not store.connect(): print( "database connection failed — check the connection string", diff --git a/scripts/backtest_render.py b/scripts/backtest_render.py index 3c23d35..70c09fd 100644 --- a/scripts/backtest_render.py +++ b/scripts/backtest_render.py @@ -14,7 +14,7 @@ non-zero exit if the model gives less than 12 h of warning before the first 3.70 m crossing of the 2024 event. Usage: - python scripts/backtest_render.py # uses FLOOD_ML_DB_URL/Config + python scripts/backtest_render.py # uses FLOOD_ML_DB_URL / POSTGRES_* python scripts/backtest_render.py --db-url postgresql://... """ @@ -60,7 +60,11 @@ def fit_backtest_model(df_long: pd.DataFrame, train_end: str, use_dam: bool = Fa rain_series = rain_mod.catchment_mean(rain_mod.load_history()) dam_frame = dam_mod.load_history() if use_dam else None X, Y, _meta = features.build_matrix( - df_long, STATION, (HORIZON,), stats_end=train_end, rain=rain_series, + df_long, + STATION, + (HORIZON,), + stats_end=train_end, + rain=rain_series, dam=dam_frame, ) train_mask = X.index <= pd.Timestamp(train_end) @@ -92,6 +96,7 @@ def event_series(df_long, X, reg, clf, window_start: str, window_end: str): # whichever is more alarmed. The classifier alone proved unreliable on # out-of-distribution extremes (silent on the 2024 record flood). import numpy as np + p_clf = clf.predict_proba(Xw)[:, 1] p_sig = 1.0 / (1.0 + np.exp(-(forecasts["pred_max"] - STAGE1) / 0.15)) forecasts["p_flood"] = np.maximum(p_clf, p_sig) @@ -109,55 +114,107 @@ def _style_axes(ax): ax.set_axisbelow(True) -def render(observed, forecasts, flood_start, first_alert, out_path, *, - title, subtitle, detail=False, show_stage7=False, peak_note=None): +def render( + observed, + forecasts, + flood_start, + first_alert, + out_path, + *, + title, + subtitle, + detail=False, + show_stage7=False, + peak_note=None, +): fig, (ax, axp) = plt.subplots( - 2, 1, figsize=(12.6, 7.6), sharex=True, + 2, + 1, + figsize=(12.6, 7.6), + sharex=True, gridspec_kw={"height_ratios": [2.2, 1], "hspace": 0.12}, ) fig.patch.set_facecolor("white") marker = dict(marker="o", markersize=3) if detail else {} - ax.plot(observed.index, observed.values, color=BLUE, linewidth=2.2, - label="Observed level" + (" (hourly)" if detail else ""), **marker) + ax.plot( + observed.index, + observed.values, + color=BLUE, + linewidth=2.2, + label="Observed level" + (" (hourly)" if detail else ""), + **marker, + ) marker = dict(marker="s", markersize=3) if detail else {} - ax.plot(forecasts.index, forecasts["pred_max"], color=AMBER, linewidth=2, - linestyle="--", label="Predicted 24 h peak (issued at that hour)", **marker) + ax.plot( + forecasts.index, + forecasts["pred_max"], + color=AMBER, + linewidth=2, + linestyle="--", + label="Predicted 24 h peak (issued at that hour)", + **marker, + ) ax.axhline(STAGE1, color=RED, linewidth=1, alpha=0.65) - ax.annotate(f"{STAGE1:.2f} m · stage 1 · flooding begins", xy=(0.06, STAGE1), - xycoords=("axes fraction", "data"), xytext=(0, 5), - textcoords="offset points", color=RED, fontsize=10.5) + ax.annotate( + f"{STAGE1:.2f} m · stage 1 · flooding begins", + xy=(0.06, STAGE1), + xycoords=("axes fraction", "data"), + xytext=(0, 5), + textcoords="offset points", + color=RED, + fontsize=10.5, + ) if show_stage7: ax.axhline(STAGE7, color=RED, linewidth=1, alpha=0.65) - ax.annotate(f"{STAGE7:.2f} m · stage 7 · widespread", xy=(0.06, STAGE7), - xycoords=("axes fraction", "data"), xytext=(0, 5), - textcoords="offset points", color=RED, fontsize=10.5) + ax.annotate( + f"{STAGE7:.2f} m · stage 7 · widespread", + xy=(0.06, STAGE7), + xycoords=("axes fraction", "data"), + xytext=(0, 5), + textcoords="offset points", + color=RED, + fontsize=10.5, + ) if peak_note: peak_ts = observed.idxmax() - ax.annotate(peak_note, xy=(peak_ts, observed.max()), - xytext=(12, 10), textcoords="offset points", - color=BLUE, fontsize=11.5, fontweight="bold") + ax.annotate( + peak_note, + xy=(peak_ts, observed.max()), + xytext=(12, 10), + textcoords="offset points", + color=BLUE, + fontsize=11.5, + fontweight="bold", + ) ax.set_ylabel("P.1 water level (m)", color=INK, fontsize=11.5) ax.legend(loc="upper left", frameon=False, fontsize=10.5) _style_axes(ax) axp.plot(forecasts.index, forecasts["p_flood"], color=AMBER, linewidth=1.8) - axp.fill_between(forecasts.index, 0, forecasts["p_flood"], - color=AMBER, alpha=0.28) + axp.fill_between(forecasts.index, 0, forecasts["p_flood"], color=AMBER, alpha=0.28) axp.axhline(0.5, color=INK, linewidth=0.9, linestyle=":", alpha=0.6) axp.set_ylim(-0.02, 1.1) axp.set_ylabel(f"P(flooding within {HORIZON} h)", color=INK, fontsize=11.5) _style_axes(axp) if first_alert is not None: - lead_h = None if flood_start is None else \ - int((flood_start - first_alert).total_seconds() // 3600) - lead_txt = "" if lead_h is None else ( - f"\n({lead_h} h before flooding began)" if lead_h >= 0 - else f"\n({-lead_h} h after flooding began)" + lead_h = ( + None + if flood_start is None + else int((flood_start - first_alert).total_seconds() // 3600) + ) + lead_txt = ( + "" + if lead_h is None + else ( + f"\n({lead_h} h before flooding began)" + if lead_h >= 0 + else f"\n({-lead_h} h after flooding began)" + ) ) if detail and flood_start is not None: for a in (ax, axp): @@ -166,34 +223,61 @@ def render(observed, forecasts, flood_start, first_alert, out_path, *, # Anchor labels away from each other in chronological order so a # late alert (alert AFTER crossing) cannot overprint the labels. events = sorted( - [(first_alert, "model alert", AMBER), (flood_start, "flooding begins", BLUE)] + [ + (first_alert, "model alert", AMBER), + (flood_start, "flooding begins", BLUE), + ] ) - for (ts, label, color), (offset, align) in zip(events, ((-8, "right"), (8, "left"))): - ax.annotate(f"{label}\n{ts:%d %b %H:%M}", - xy=(ts, observed.min()), xytext=(offset, 18), - textcoords="offset points", ha=align, - color=color, fontsize=11, fontweight="bold") + for (ts, label, color), (offset, align) in zip( + events, ((-8, "right"), (8, "left")) + ): + ax.annotate( + f"{label}\n{ts:%d %b %H:%M}", + xy=(ts, observed.min()), + xytext=(offset, 18), + textcoords="offset points", + ha=align, + color=color, + fontsize=11, + fontweight="bold", + ) mid_y = observed.min() + (observed.max() - observed.min()) * 0.28 - ax.annotate("", xy=(flood_start, mid_y), xytext=(first_alert, mid_y), - arrowprops=dict(arrowstyle="<->", color=INK, lw=1.3)) + ax.annotate( + "", + xy=(flood_start, mid_y), + xytext=(first_alert, mid_y), + arrowprops=dict(arrowstyle="<->", color=INK, lw=1.3), + ) arrow_label = ( f"{lead_h} h warning" if lead_h >= 0 else f"alert {-lead_h} h late" ) - ax.annotate(arrow_label, - xy=(first_alert + (flood_start - first_alert) / 2, mid_y), - xytext=(0, 8), textcoords="offset points", ha="center", - color=INK, fontsize=11.5, fontweight="bold") + ax.annotate( + arrow_label, + xy=(first_alert + (flood_start - first_alert) / 2, mid_y), + xytext=(0, 8), + textcoords="offset points", + ha="center", + color=INK, + fontsize=11.5, + fontweight="bold", + ) else: - axp.annotate(f"first alert · {first_alert:%d %b %H:%M}{lead_txt}", - xy=(first_alert, 0.62), xytext=(10, 0), - textcoords="offset points", color=RED, fontsize=10.5, - bbox=dict(facecolor="white", alpha=0.75, edgecolor="none")) + axp.annotate( + f"first alert · {first_alert:%d %b %H:%M}{lead_txt}", + xy=(first_alert, 0.62), + xytext=(10, 0), + textcoords="offset points", + color=RED, + fontsize=10.5, + bbox=dict(facecolor="white", alpha=0.75, edgecolor="none"), + ) locator = mdates.DayLocator(interval=1 if detail else 3) axp.xaxis.set_major_locator(locator) axp.xaxis.set_major_formatter(mdates.DateFormatter("%d %b")) - fig.suptitle(f"{title}\n{subtitle}", x=0.07, y=0.985, ha="left", - fontsize=15, color=INK) + fig.suptitle( + f"{title}\n{subtitle}", x=0.07, y=0.985, ha="left", fontsize=15, color=INK + ) fig.subplots_adjust(top=0.885, left=0.07, right=0.97, bottom=0.07) fig.savefig(out_path, dpi=110) plt.close(fig) @@ -204,16 +288,20 @@ def main(argv=None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--db-url", default=None) parser.add_argument("--out-dir", default=os.path.join("docs", "img")) - parser.add_argument("--dam", action="store_true", - help="ablation: include Mae Ngat reservoir features " - "(2026-08 result: costs 1-3 h of alert lead)") - parser.add_argument("--no-hii-fill", action="store_true", - help="ablation: load without the HII gap-fill merge") + parser.add_argument( + "--dam", + action="store_true", + help="ablation: include Mae Ngat reservoir features " + "(2026-08 result: costs 1-3 h of alert lead)", + ) + parser.add_argument( + "--no-hii-fill", + action="store_true", + help="ablation: load without the HII gap-fill merge", + ) args = parser.parse_args(argv) - df = data.load_measurements( - db_url=args.db_url, hii_fill=not args.no_hii_fill - ) + df = data.load_measurements(db_url=args.db_url, hii_fill=not args.no_hii_fill) if df.empty: print("no measurement data available", file=sys.stderr) return 1 @@ -222,41 +310,66 @@ def main(argv=None) -> int: # --- October 2024 record flood: trained only on data before 1 Sep 2024 --- X, reg, clf = fit_backtest_model(df, "2024-08-31", use_dam=args.dam) obs, fc, flood_start, first_alert = event_series( - df, X, reg, clf, "2024-09-10", "2024-10-14 23:00") + df, X, reg, clf, "2024-09-10", "2024-10-14 23:00" + ) peak = float(obs.max()) - render(obs, fc, flood_start, first_alert, - os.path.join(args.out_dir, "backtest-2024-p1.png"), - title="October 2024 flood: what the model saw coming", - subtitle="P.1 Nawarat Bridge — model trained only on data before 1 Sep 2024", - show_stage7=True, peak_note=f"record peak {peak:.2f} m") + render( + obs, + fc, + flood_start, + first_alert, + os.path.join(args.out_dir, "backtest-2024-p1.png"), + title="October 2024 flood: what the model saw coming", + subtitle="P.1 Nawarat Bridge — model trained only on data before 1 Sep 2024", + show_stage7=True, + peak_note=f"record peak {peak:.2f} m", + ) obs_d, fc_d, flood_d, alert_d = event_series( - df, X, reg, clf, "2024-09-21 18:00", "2024-09-28 06:00") + df, X, reg, clf, "2024-09-21 18:00", "2024-09-28 06:00" + ) lead_h = None if alert_d is not None and flood_d is not None: lead_h = int((flood_d - alert_d).total_seconds() // 3600) - render(obs_d, fc_d, flood_d, alert_d, - os.path.join(args.out_dir, "backtest-2024-p1-detail.png"), - title="Detection in detail: 22–28 September 2024, hour by hour", - subtitle=( - f"the model alerts {lead_h} h before the river crosses the flooding line" - if lead_h is not None and lead_h > 0 - else "model alert vs the river crossing the flooding line" - ), - detail=True) + render( + obs_d, + fc_d, + flood_d, + alert_d, + os.path.join(args.out_dir, "backtest-2024-p1-detail.png"), + title="Detection in detail: 22–28 September 2024, hour by hour", + subtitle=( + f"the model alerts {lead_h} h before the river crosses the flooding line" + if lead_h is not None and lead_h > 0 + else "model alert vs the river crossing the flooding line" + ), + detail=True, + ) # --- September 2025 flood: the deployed configuration (trained <= 2024) --- X25, reg25, clf25 = fit_backtest_model(df, "2024-12-31", use_dam=args.dam) obs25, fc25, flood25, alert25 = event_series( - df, X25, reg25, clf25, "2025-09-22", "2025-10-02 12:00") - pred_at_alert = float(fc25.loc[alert25:, "pred_max"].iloc[:24].max()) if alert25 is not None else None + df, X25, reg25, clf25, "2025-09-22", "2025-10-02 12:00" + ) + pred_at_alert = ( + float(fc25.loc[alert25:, "pred_max"].iloc[:24].max()) + if alert25 is not None + else None + ) note = f"peak {float(obs25.max()):.2f} m" + ( - f" (predicted {pred_at_alert:.2f} m)" if pred_at_alert is not None else "") - render(obs25, fc25, flood25, alert25, - os.path.join(args.out_dir, "backtest-2025-p1.png"), - title="The September 2025 flood — as forecast by the deployed configuration", - subtitle="model trained only on data through 2024; this event was never seen in training", - detail=True, peak_note=note) + f" (predicted {pred_at_alert:.2f} m)" if pred_at_alert is not None else "" + ) + render( + obs25, + fc25, + flood25, + alert25, + os.path.join(args.out_dir, "backtest-2025-p1.png"), + title="The September 2025 flood — as forecast by the deployed configuration", + subtitle="model trained only on data through 2024; this event was never seen in training", + detail=True, + peak_note=note, + ) print(f"2024: flooding began {flood_start}, first alert {first_alert}") print(f"2025: flooding began {flood25}, first alert {alert25}") diff --git a/scripts/collector_parity.py b/scripts/collector_parity.py deleted file mode 100644 index 771ec4b..0000000 --- a/scripts/collector_parity.py +++ /dev/null @@ -1,150 +0,0 @@ -"""Live parity check: Python collectors vs the Rust ping-collector. - -Runs one collection of every source with the Python code into PY_DB and with -`ping-collector` into RS_DB (both should start empty), back to back against the -real upstream APIs, then diffs all nine collector tables row by row (ids and -created/updated timestamps ignored). Exit status 1 on any difference. - - uv run python scripts/collector_parity.py \ - --py-db postgresql://postgres:ping@localhost:55432/py \ - --rs-db postgresql://postgres:ping@localhost:55432/rs \ - --collector rust/target/debug/ping-collector - -Upstream snapshots can move between the two runs (a new HII hour, a revised -RID value); rerun before treating a small difference as a bug. -""" - -import argparse -import os -import subprocess -import sys -from pathlib import Path - -REPO = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO)) - -TABLES = { - "stations": ( - "id", - "id, station_code, thai_name, english_name, latitude, longitude, geohash", - ), - "water_measurements": ( - "timestamp, station_id", - "timestamp, station_id, water_level, discharge, discharge_percent, status", - ), - "hii_rain_stations": ( - "id", - "id, oldcode, name_th, name_en, latitude, longitude, sub_basin_id, agency", - ), - "hii_rainfall": ( - "station_id, timestamp", - "station_id, timestamp, rain_1h, rain_24h", - ), - "hii_wl_stations": ( - "id", - "id, oldcode, rid_code, name_th, name_en, latitude, longitude, agency, " - "river_name, offset_msl, ground_level_msl, min_bank_msl, " - "critical_level_msl, critical_level_m, qmax, is_key_station", - ), - "hii_waterlevel": ( - "station_id, timestamp", - "station_id, timestamp, wl_msl, wl_m, discharge, flow_rate, " - "storage_percent, situation_level, diff_wl_bank", - ), - "rid_dams": ( - "dam_id", - "dam_id, region, name_th, latitude, longitude, capacity_max_mcm, " - "capacity_normal_mcm", - ), - "rid_reservoir_daily": ( - "dam_id, date", - "dam_id, date, storage_mcm, storage_pct, inflow_mcm, outflow_mcm, level_msl", - ), - "openmeteo_rain": ( - "timestamp", - "timestamp, chiang_dao, mae_taeng, mae_ngat, mae_rim, chiang_mai, " - "catchment_mean", - ), -} - - -def run_python(db_url: str, gap_days: int): - os.environ.update( - DB_TYPE="postgresql", POSTGRES_CONNECTION_STRING=db_url, LOG_LEVEL="WARNING" - ) - from sqlalchemy import create_engine - - from src.config import Config - from src.hii_collector import HiiCollector - from src.ml import rain - from src.rid_reservoir import RidReservoirCollector - from src.water_scraper_v3 import EnhancedWaterMonitorScraper - - cfg = Config.get_database_config() - scraper = EnhancedWaterMonitorScraper(cfg) - scraper.fill_data_gaps(gap_days) - scraper.run_scraping_cycle() - HiiCollector(cfg).run_cycle() - RidReservoirCollector(cfg).run_cycle() - rain.save_to_db(rain.fetch_forecast(), create_engine(db_url), "postgresql") - - -def run_rust(db_url: str, collector: str, gap_days: int): - env = {**os.environ, "POSTGRES_CONNECTION_STRING": db_url, "DB_TYPE": "postgresql"} - for args in (["fill-gaps", str(gap_days)], ["once"]): - subprocess.run([collector, *args], env=env, check=True) - - -def table_rows(db_url: str, table: str, key: str, cols: str) -> dict: - import psycopg2 - - with psycopg2.connect(db_url) as conn: - cur = conn.cursor() - cur.execute(f"SELECT {cols} FROM {table} ORDER BY {key}") - nkey = len(key.split(",")) - return {row[:nkey]: row for row in cur.fetchall()} - - -def diff(py_db: str, rs_db: str) -> bool: - identical = True - for table, (key, cols) in TABLES.items(): - a = table_rows(py_db, table, key, cols) - b = table_rows(rs_db, table, key, cols) - only_py, only_rs = sorted(set(a) - set(b)), sorted(set(b) - set(a)) - differ = sorted(k for k in set(a) & set(b) if a[k] != b[k]) - same = not (only_py or only_rs or differ) - identical &= same - print( - f"{table:<20} py {len(a):>6} rs {len(b):>6} only-py {len(only_py):>4} " - f"only-rs {len(only_rs):>4} differing {len(differ):>4} " - f"{'identical' if same else 'DIFFERENT'}" - ) - for k in only_py[:3]: - print(" only py:", a[k]) - for k in only_rs[:3]: - print(" only rs:", b[k]) - for k in differ[:3]: - print(" py:", a[k], "\n rs:", b[k]) - return identical - - -def main(): - parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - parser.add_argument("--py-db", required=True) - parser.add_argument("--rs-db", required=True) - parser.add_argument( - "--collector", default=str(REPO / "rust/target/debug/ping-collector") - ) - parser.add_argument("--gap-days", type=int, default=2) - parser.add_argument("--diff-only", action="store_true") - args = parser.parse_args() - if not args.diff_only: - run_python(args.py_db, args.gap_days) - run_rust(args.rs_db, args.collector, args.gap_days) - ok = diff(args.py_db, args.rs_db) - print("ALL IDENTICAL" if ok else "differences found") - sys.exit(0 if ok else 1) - - -if __name__ == "__main__": - main() diff --git a/scripts/dev_proxy.py b/scripts/dev_proxy.py index fb6f18f..793bf11 100644 --- a/scripts/dev_proxy.py +++ b/scripts/dev_proxy.py @@ -25,18 +25,29 @@ class Handler(http.server.BaseHTTPRequestHandler): self._send(200, "application/json", Path(file).read_bytes()) return if self.path.startswith("/static/"): - f = STATIC / self.path[len("/static/"):].split("?")[0] + f = STATIC / self.path[len("/static/") :].split("?")[0] if f.is_file(): - ctype = "application/json" if f.suffix in (".json", ".geojson") else "application/octet-stream" + ctype = ( + "application/json" + if f.suffix in (".json", ".geojson") + else "application/octet-stream" + ) self._send(200, ctype, f.read_bytes()) return try: req = urllib.request.Request( UPSTREAM + self.path, - headers={"User-Agent": "Mozilla/5.0 (dev_proxy; +https://buildfor.life)", "Accept": "application/json"}, + headers={ + "User-Agent": "Mozilla/5.0 (dev_proxy; +https://buildfor.life)", + "Accept": "application/json", + }, ) with urllib.request.urlopen(req, timeout=60) as r: - self._send(r.status, r.headers.get("Content-Type", "application/json"), r.read()) + self._send( + r.status, + r.headers.get("Content-Type", "application/json"), + r.read(), + ) except urllib.error.HTTPError as e: self._send(e.code, "application/json", e.read()) diff --git a/scripts/e2e_notify.py b/scripts/e2e_notify.py deleted file mode 100644 index adb59f4..0000000 --- a/scripts/e2e_notify.py +++ /dev/null @@ -1,132 +0,0 @@ -"""Drive the production notify path in-process: startup init -> seeded readings --> forecast cache -> _notify_transitions -> sqlite state -> real ntfy.""" - -import asyncio -import datetime -import json -import os -import sys - -import requests - -os.environ.update( - DB_TYPE="sqlite", - WATER_DB_PATH=os.path.join(os.environ["LOCALAPPDATA"], "Temp", "smoke3.db"), - NTFY_SERVER="http://127.0.0.1:2586", - NTFY_TOKEN=os.environ.get("NTFY_TOKEN", ""), - NTFY_TOPIC_PREFIX="ping", -) -for f in ("smoke3.db",): - p = os.path.join(os.environ["LOCALAPPDATA"], "Temp", f) - if os.path.exists(p): - os.remove(p) - -from src import web_api # noqa: E402 -from src.config import Config # noqa: E402 - -assert Config.NTFY_SERVER - - -async def main(): - # what the lifespan does at startup, minus the scheduler - from src import notify as notify_mod - from src.forecast_history import ForecastHistoryStore - from src.water_scraper_v3 import EnhancedWaterMonitorScraper - - db_config = Config.get_database_config() - web_api.app_state["scraper"] = EnhancedWaterMonitorScraper(db_config) - store = ForecastHistoryStore(db_config["connection_string"], db_config["type"]) - store.connect() - web_api.app_state["forecast_store"] = store - state = notify_mod.NotificationState(store.engine, store.db_type) - pub = notify_mod.NtfyPublisher( - Config.NTFY_SERVER, prefix=Config.NTFY_TOPIC_PREFIX, token=Config.NTFY_TOKEN - ) - web_api.app_state["notify"] = (pub, state) - - scraper = web_api.app_state["scraper"] - now = datetime.datetime.now().replace(minute=0, second=0, microsecond=0) - - def seed(level_p1, level_p103, ts): - rows = [ - { - "station_code": "P.1", - "station_id": 1, - "timestamp": ts, - "water_level": level_p1, - "discharge": 400.0, - "station_name_en": "Nawarat Bridge", - "station_name_th": "สะพานนวรัฐ", - "discharge_percent": 30.0, - "status": "active", - }, - { - "station_code": "P.103", - "station_id": 2, - "timestamp": ts, - "water_level": level_p103, - "discharge": 300.0, - "station_name_en": "Ring Road 3", - "station_name_th": "วงแหวน 3", - "discharge_percent": 20.0, - "status": "active", - }, - ] - scraper.db_adapter.save_measurements(rows) - - def forecast(p): - with web_api.FORECAST_CACHE_LOCK: - web_api.FORECAST_CACHE["all"] = ( - 0, - [ - { - "station_code": "P.1", - "horizon_hours": 24, - "p_warning": p, - "predicted_max_level": 3.9, - "source": "model", - } - ], - ) - - def poll(topic): - out = [] - for line in ( - requests.get(f"{Config.NTFY_SERVER}/{topic}/json?poll=1", timeout=5) - .text.strip() - .splitlines() - ): - m = json.loads(line) - if m.get("event") == "message": - out.append(m.get("title") or m.get("message", "")[:40]) - return out - - # cycle 1: quiet - seed(1.6, 3.2, now - datetime.timedelta(hours=2)) - forecast(0.02) - await web_api._notify_transitions() - # cycle 2: P.1 crosses warning, model outlook on - seed(3.75, 3.3, now - datetime.timedelta(hours=1)) - forecast(0.7) - await web_api._notify_transitions() - # cycle 3: same state -> silence - seed(3.80, 3.3, now) - forecast(0.65) - await web_api._notify_transitions() - - print("ping-p1-warning:", poll("ping-p1-warning")) - print("ping-warning: ", poll("ping-warning")) - print("ping-p1-outlook:", poll("ping-p1-outlook")) - print("ping-p103-warning:", poll("ping-p103-warning")) - from sqlalchemy import text - - with store.engine.connect() as c: - print( - "state table:", - c.execute( - text("SELECT key, state, value FROM notification_state ORDER BY key") - ).fetchall(), - ) - - -asyncio.run(main()) diff --git a/scripts/forecast_parity.py b/scripts/forecast_parity.py index 115b725..786bd3e 100644 --- a/scripts/forecast_parity.py +++ b/scripts/forecast_parity.py @@ -1,8 +1,10 @@ #!/usr/bin/env python3 -"""Compare the Python forecast with ping-collector's on the live database. +"""Compare ping-collector's forecast with the Python reference on the live database. -Run before setting FORECAST_IN_COLLECTOR=true, and after any change to src/ml -or rust/forecast. On the server, as the service user from the app dir: +src/ml/predict.py is the reference implementation the Rust forecaster +(rust/forecast) was ported from. Run after any change to src/ml or +rust/forecast; scripts/install.sh runs it on every install. On the server, as +the service user from the app dir: sudo -u water-monitor .venv/bin/python scripts/forecast_parity.py diff --git a/scripts/install.sh b/scripts/install.sh index 25878cc..3a503a3 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -1,127 +1,202 @@ #!/usr/bin/env bash +# Install or update the Ping River Monitor on a systemd host. # -# Install the Thailand Water Level Monitor as a hardened systemd service. +# sudo bash scripts/install.sh # -# Creates a dedicated system user, deploys the code to /opt, builds a uv-managed -# virtualenv, installs the systemd unit, and enables the service. Idempotent: -# safe to re-run to update an existing install. -# -# Usage (as root, from a checkout of the repo): -# sudo bash scripts/install.sh -# -# Override defaults via environment variables: -# APP_DIR=/opt/thailand-water-monitor SERVICE_USER=water-monitor sudo -E bash scripts/install.sh +# What runs: +# ping-collector.service the service (rust/collector): RID/HII/dam/rain +# collection, forecast, ntfy + Matrix +# notifications, website + API on WEB_LISTEN +# water-monitor-retrain.timer monthly model retrain (Python, scripts/retrain.sh) # +# Re-run after every update (git pull + build). Steps, all idempotent: +# 1. system user/group (water-monitor) +# 2. code: run from a checkout other than APP_DIR, it is rsynced there (the +# trained models, .env, stations.json and bin/ are kept) and handed to +# the service user. Production runs in place (APP_DIR is the checkout): +# then nothing is copied and no ownership changes. +# 3. the binary: BIN=/path/to/ping-collector (e.g. the CI artifact +# "ping-collector-linux-x86_64") or rust/target/release/ping-collector, +# built beforehand as a normal user: +# cargo build --release --locked --manifest-path rust/Cargo.toml +# (Rust >= 1.88 from rustup, plus pkg-config and libssl-dev) +# 4. the Python training environment (.venv, `uv sync --frozen`): in place +# as the user running sudo (who pulled and built), otherwise as root; +# SKIP_UV=1 skips it +# 5. .env: created from .env.example on a fresh install (edit, then re-run); +# WEB_LISTEN and FORECAST_IN_COLLECTOR default to the production values +# when absent +# 6. systemd units; the retired Python web app (water-monitor.service) and +# the old `run.py --alert-check` Matrix cron job are removed +# 7. checks (SKIP_CHECKS=1 skips them): a dry run of every source against +# the database, and the forecast compared with the Python reference on +# the live data (scripts/forecast_parity.py) +# 8. (re)start, then wait for the website to answer set -euo pipefail APP_DIR="${APP_DIR:-/opt/thailand-water-monitor}" -SERVICE_USER="${SERVICE_USER:-water-monitor}" +SERVICE_USER="${SERVICE_USER:-water-monitor}" # as in the units SERVICE_GROUP="${SERVICE_GROUP:-${SERVICE_USER}}" -SERVICE_NAME="water-monitor.service" -RETRAIN_NAME="water-monitor-retrain" - -# Resolve the repo root (parent of this scripts/ directory). -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +ENV_FILE="${APP_DIR}/.env" +PYTHON="${APP_DIR}/.venv/bin/python" +REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +RETRAIN="water-monitor-retrain" log() { printf '\033[1;32m==>\033[0m %s\n' "$*"; } -warn() { printf '\033[1;33m[warn]\033[0m %s\n' "$*"; } +warn() { printf '\033[1;33m[warn]\033[0m %s\n' "$*" >&2; } die() { printf '\033[1;31m[error]\033[0m %s\n' "$*" >&2; exit 1; } -[ "$(id -u)" -eq 0 ] || die "This script must be run as root (use sudo)." +[ "$(id -u)" -eq 0 ] || die "run as root (sudo)" -# 1. Dedicated system user/group (no login, no home) -------------------------- -if ! getent group "${SERVICE_GROUP}" >/dev/null; then - log "Creating group ${SERVICE_GROUP}" - groupadd --system "${SERVICE_GROUP}" -fi -if ! id "${SERVICE_USER}" >/dev/null 2>&1; then - log "Creating system user ${SERVICE_USER}" +set_default() { # key value: append when the key is not set at all + grep -q "^$1=" "$ENV_FILE" || printf '%s=%s\n' "$1" "$2" >>"$ENV_FILE" +} + +# A command the way the services run it: same user, working directory and +# EnvironmentFile parsing (systemd's). +as_service() { + if systemd-run --version >/dev/null 2>&1; then + systemd-run --quiet --pipe --wait --collect \ + -p User="$SERVICE_USER" -p Group="$SERVICE_GROUP" \ + -p WorkingDirectory="$APP_DIR" -p EnvironmentFile="$ENV_FILE" "$@" + else + (cd "$APP_DIR" && runuser -u "$SERVICE_USER" -- "$@") + fi +} + +wait_http() { # url seconds + local i + for i in $(seq "$2"); do + curl -fs -o /dev/null --max-time 5 "$1" 2>/dev/null && return 0 + sleep 1 + done + return 1 +} + +# 1. user --------------------------------------------------------------------- +getent group "$SERVICE_GROUP" >/dev/null || groupadd --system "$SERVICE_GROUP" +if ! id "$SERVICE_USER" >/dev/null 2>&1; then + log "Creating system user $SERVICE_USER" useradd --system --no-create-home --shell /usr/sbin/nologin \ - --gid "${SERVICE_GROUP}" "${SERVICE_USER}" + --gid "$SERVICE_GROUP" "$SERVICE_USER" fi -# 2. Deploy code to APP_DIR ---------------------------------------------------- -log "Deploying code to ${APP_DIR}" -mkdir -p "${APP_DIR}" -if command -v rsync >/dev/null 2>&1; then +# 2. code --------------------------------------------------------------------- +in_place=0 +[ -d "$APP_DIR" ] && [ "$(cd "$APP_DIR" && pwd -P)" = "$(cd "$REPO_DIR" && pwd -P)" ] && in_place=1 +if [ "$in_place" = 0 ]; then + command -v rsync >/dev/null || die "rsync is needed to copy $REPO_DIR to $APP_DIR" + log "Copying $REPO_DIR to $APP_DIR" + mkdir -p "$APP_DIR" rsync -a --delete \ - --exclude '.git' --exclude '.venv' --exclude 'venv' \ - --exclude '__pycache__' --exclude '*.pyc' \ - --exclude '*.db' --exclude '.env' --exclude 'stations.json' \ - "${REPO_DIR}/" "${APP_DIR}/" -else - warn "rsync not found; falling back to cp (will not prune deleted files)" - cp -r "${REPO_DIR}/." "${APP_DIR}/" + --exclude '.git' --exclude '.venv' --exclude '__pycache__' --exclude '/rust/target' \ + --exclude '/.env' --exclude '/stations.json' --exclude '/bin' \ + --exclude '/collector-status.json' --exclude '/crontab-*.bak' \ + --exclude '/models/flood_*' --exclude '/models/metrics.json' \ + --exclude '/models/cache' --exclude '/models/.staging' --exclude '/models/.previous' \ + "$REPO_DIR/" "$APP_DIR/" fi -# 3. Build the uv-managed virtualenv ------------------------------------------ -# Prefer an already-installed uv. For stricter supply-chain control install uv -# ahead of time via your distro / package manager; this script only fetches the -# upstream installer (piped to a root shell) when AUTO_INSTALL_UV=1 is set, and -# pins the version so the fetched script is reproducible. -UV_VERSION="${UV_VERSION:-0.5.11}" -if ! command -v uv >/dev/null 2>&1; then - if [ "${AUTO_INSTALL_UV:-0}" = "1" ]; then - warn "uv not found; installing pinned uv ${UV_VERSION} from astral.sh (runs as root)" - curl -LsSf "https://astral.sh/uv/${UV_VERSION}/install.sh" \ - | env UV_INSTALL_DIR=/usr/local/bin sh +# 3. binary ------------------------------------------------------------------- +BIN="${BIN:-${REPO_DIR}/rust/target/release/ping-collector}" +[ -x "$BIN" ] || die "no binary at $BIN: build it (see the header) or pass BIN=/path/to/ping-collector" +log "Installing $BIN -> ${APP_DIR}/bin/ping-collector ($("$BIN" --version))" +install -D -m 0755 -o root -g root "$BIN" "${APP_DIR}/bin/ping-collector" + +# 4. Python training environment ------------------------------------------------ +if [ "${SKIP_UV:-0}" != 1 ]; then + uv_user=root # copy mode: handed to the service user below + [ "$in_place" = 1 ] && uv_user="${SUDO_USER:-root}" + log "Syncing the Python training environment (${APP_DIR}/.venv) as $uv_user" + if [ "$uv_user" = root ]; then + (cd "$APP_DIR" && uv sync --python 3.11 --frozen) || + warn "uv sync failed: the monthly retrain needs ${APP_DIR}/.venv (uv sync --python 3.11 --frozen)" else - die "uv not found. Install it (e.g. your package manager, or 'pipx install uv'), - or re-run with AUTO_INSTALL_UV=1 to fetch the pinned upstream installer." - fi -fi -UV="$(command -v uv)" - -log "Syncing uv-managed virtualenv at ${APP_DIR}/.venv" -cd "${APP_DIR}" -# ONE environment: uv sync owns .venv/ (from pyproject.toml + uv.lock, so the -# ML extras such as scikit-learn/joblib are present) and both systemd units -# run its interpreter directly. Never create a second env by another name -- -# a stale 'venv/' once coexisted here and broke manual retrains with -# ModuleNotFoundError while the service itself ran fine. -"${UV}" sync --python 3.11 --frozen -if [ -d "${APP_DIR}/venv" ]; then - warn "Removing stale ${APP_DIR}/venv (superseded by .venv)" - rm -rf "${APP_DIR}/venv" -fi - -# 4. Environment file ---------------------------------------------------------- -if [ ! -f "${APP_DIR}/.env" ]; then - if [ -f "${REPO_DIR}/.env" ]; then - log "Copying .env from checkout" - cp "${REPO_DIR}/.env" "${APP_DIR}/.env" - else - warn "No .env found. Copy .env.example to ${APP_DIR}/.env and fill in" - warn "MATRIX_ACCESS_TOKEN / MATRIX_ROOM_ID and DB settings before starting." + runuser -u "$uv_user" -- bash -lc "cd '$APP_DIR' && uv sync --python 3.11 --frozen" || + warn "uv sync as $uv_user failed: run 'uv sync --python 3.11 --frozen' in $APP_DIR as the owner of .venv" fi fi -# 5. Ownership and permissions ------------------------------------------------- -# Service user needs write access for logs / stations.json. -log "Setting ownership to ${SERVICE_USER}:${SERVICE_GROUP}" -chown -R "${SERVICE_USER}:${SERVICE_GROUP}" "${APP_DIR}" -# Restrict traversal to root + the service user, and lock down the secrets file -# (contains the Matrix token and DB credentials). -chmod 0750 "${APP_DIR}" -if [ -f "${APP_DIR}/.env" ]; then - chmod 0600 "${APP_DIR}/.env" +# 5. .env --------------------------------------------------------------------- +fresh_env=0 +if [ ! -f "$ENV_FILE" ]; then + cp "${APP_DIR}/.env.example" "$ENV_FILE" + chmod 0600 "$ENV_FILE" + fresh_env=1 +fi +set_default WEB_LISTEN 0.0.0.0:8000 +set_default FORECAST_IN_COLLECTOR true + +if [ "$in_place" = 0 ]; then + # a copy belongs to the service (it writes models/, stations.json and the + # status file); an in-place checkout keeps its owner (git pull, builds) + chown -R "${SERVICE_USER}:${SERVICE_GROUP}" "$APP_DIR" + chown root:root "${APP_DIR}/bin/ping-collector" + chmod 0750 "$APP_DIR" fi -# 6. Install and enable the systemd units ------------------------------------- +# 6. units -------------------------------------------------------------------- log "Installing systemd units" -install -m 0644 "${SCRIPT_DIR}/${SERVICE_NAME}" "/etc/systemd/system/${SERVICE_NAME}" -install -m 0644 "${SCRIPT_DIR}/${RETRAIN_NAME}.service" "/etc/systemd/system/${RETRAIN_NAME}.service" -install -m 0644 "${SCRIPT_DIR}/${RETRAIN_NAME}.timer" "/etc/systemd/system/${RETRAIN_NAME}.timer" +install -m 0644 "${APP_DIR}/scripts/ping-collector.service" /etc/systemd/system/ping-collector.service +install -m 0644 "${APP_DIR}/scripts/${RETRAIN}.service" "/etc/systemd/system/${RETRAIN}.service" +install -m 0644 "${APP_DIR}/scripts/${RETRAIN}.timer" "/etc/systemd/system/${RETRAIN}.timer" +if [ -f /etc/systemd/system/water-monitor.service ]; then + log "Removing the retired Python web app (water-monitor.service)" + systemctl disable --now water-monitor.service 2>/dev/null || true + rm -f /etc/systemd/system/water-monitor.service +fi systemctl daemon-reload -systemctl enable "${SERVICE_NAME}" -systemctl enable --now "${RETRAIN_NAME}.timer" +systemctl enable ping-collector.service +systemctl enable --now "${RETRAIN}.timer" -log "Done." +# The old Matrix cron job (`run.py --alert-check`, repeating an alert every +# 15 minutes while a level stayed high): ping-collector sends Matrix now. +for user in root "$SERVICE_USER"; do + crontab -l -u "$user" 2>/dev/null | grep -v '^\s*#' | grep -q -- '--alert-check' || continue + backup="${APP_DIR}/crontab-${user}.$(date +%Y%m%d%H%M%S).bak" + crontab -l -u "$user" >"$backup" + sed '/--alert-check/ s|^\([^#]\)|# retired: Matrix is sent by ping-collector # \1|' "$backup" | + crontab -u "$user" - + log "Commented out the old --alert-check cron job of $user (backup: $backup)" +done +if grep -rlqs -- '--alert-check' /etc/cron.d /etc/crontab 2>/dev/null; then + warn "an --alert-check job is still in $(grep -rls -- '--alert-check' /etc/cron.d /etc/crontab | tr '\n' ' '): remove it" +fi + +if [ "$fresh_env" = 1 ]; then + warn "created $ENV_FILE from .env.example: set the database (POSTGRES_*), NTFY_* and MATRIX_*, then re-run" + exit 0 +fi + +# 7. checks ------------------------------------------------------------------- +if [ "${SKIP_CHECKS:-0}" != 1 ]; then + log "Dry run of every source against the configured database (writes nothing)" + as_service "${APP_DIR}/bin/ping-collector" once --dry-run + if compgen -G "${APP_DIR}/models/flood_*.joblib" >/dev/null && [ -x "$PYTHON" ]; then + log "Exporting trained models for the forecaster (models/flood_*.json.gz)" + as_service "$PYTHON" -m src.ml.export + log "Forecast check: ping-collector against the Python reference, same data, moment and rain" + as_service "$PYTHON" scripts/forecast_parity.py || + warn "the forecast check found differences (see above); the service runs regardless" + else + warn "no trained models or no ${PYTHON}: forecast check skipped (the forecast falls back to the heuristic)" + fi +fi + +# 8. (re)start ---------------------------------------------------------------- +systemctl restart ping-collector.service +listen="$(grep '^WEB_LISTEN=' "$ENV_FILE" | tail -1 | cut -d= -f2- | tr -d '"'"'"' ')" +if [ -n "$listen" ]; then + port="${listen##*:}" + # readiness: a route that answers 200 whatever the data (/health may be 503) + if wait_http "http://127.0.0.1:${port}/api/notifications" 60; then + log "Website and API answer on :${port} ($(curl -s --max-time 5 "http://127.0.0.1:${port}/scraping/status" | grep -o '"collector":"[^"]*"'))" + else + warn "no answer on port ${port} yet: journalctl -u ping-collector" + fi +fi +sleep 3 +systemctl --no-pager --lines=0 status ping-collector.service || true +journalctl -u ping-collector -n 25 --no-pager || true echo -echo "Next steps:" -echo " sudo systemctl start ${SERVICE_NAME}" -echo " systemctl status ${SERVICE_NAME}" -echo " sudo journalctl -u ${SERVICE_NAME} -f" -echo " systemctl list-timers ${RETRAIN_NAME}.timer # monthly flood-model retrain" -echo " sudo systemctl start ${RETRAIN_NAME}.service # retrain now" +log "Done. Logs: journalctl -u ping-collector -f; retrain: systemctl list-timers ${RETRAIN}.timer" diff --git a/scripts/install_collector.sh b/scripts/install_collector.sh deleted file mode 100755 index 05fbce7..0000000 --- a/scripts/install_collector.sh +++ /dev/null @@ -1,286 +0,0 @@ -#!/usr/bin/env bash -# Install the Rust service (rust/collector, `ping-collector`) and move work -# from the Python app to it, or back. -# -# sudo bash scripts/install_collector.sh # install binary + unit; checks only -# sudo bash scripts/install_collector.sh --cutover # Rust collects (COLLECTION_ENABLED=false) -# sudo bash scripts/install_collector.sh --cutover-forecast # Rust also forecasts and notifies: -# # ntfy + Matrix (FORECAST_IN_COLLECTOR=true) -# sudo bash scripts/install_collector.sh --check-api # compare the Rust website/API with the -# # Python app's, change nothing -# sudo bash scripts/install_collector.sh --cutover-api # Rust serves the website and API on -# # port 8000 (WEB_LISTEN); the Python app stops -# sudo bash scripts/install_collector.sh --rollback-api # the Python app serves the website again -# sudo bash scripts/install_collector.sh --rollback-forecast # forecasts + notifications back to Python -# sudo bash scripts/install_collector.sh --rollback # Python does everything again -# -# Every install also exports the trained models for the Rust forecaster and -# compares its forecast with Python's on the live data -# (scripts/forecast_parity.py); --cutover-forecast refuses to switch when they -# differ. --cutover-api first runs `ping-collector serve` next to the Python -# app and compares every route (scripts/api_parity.py); it refuses to switch -# when they differ. FORCE=1 overrides either check. -# -# Once ping-collector sends the Matrix messages (FORECAST_IN_COLLECTOR=true -# and MATRIX_* set), the old `run.py --alert-check` cron job is commented out -# (backup next to the app); it repeated an alert on every run. -# -# The binary comes from BIN=/path/to/ping-collector (e.g. the "ping-collector- -# linux-x86_64" CI artifact) or from rust/target/release/ping-collector, built -# beforehand as a normal user: -# cargo build --release --locked --manifest-path rust/Cargo.toml -# (needs Rust >= 1.88 from rustup -- distro cargo is too old -- plus pkg-config -# and libssl-dev; see rust/README.md). -set -euo pipefail - -APP_DIR="${APP_DIR:-/opt/thailand-water-monitor}" -SERVICE_USER="${SERVICE_USER:-water-monitor}" # as in scripts/ping-collector.service -ENV_FILE="${APP_DIR}/.env" -STATUS_FILE="${APP_DIR}/collector-status.json" -PYTHON="${APP_DIR}/.venv/bin/python" -REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -WEB_ADDR="${WEB_ADDR:-0.0.0.0:8000}" # where the Python app listens today -CHECK_PORT="${CHECK_PORT:-8001}" # the Rust side during --check-api -MODE="${1:-install}" - -log() { printf '\033[1;32m==>\033[0m %s\n' "$*"; } -warn() { printf '\033[1;33m[warn]\033[0m %s\n' "$*" >&2; } -die() { printf '\033[1;31m[error]\033[0m %s\n' "$*" >&2; exit 1; } - -[ "$(id -u)" -eq 0 ] || die "run as root (sudo)" -[ -f "$ENV_FILE" ] || die "$ENV_FILE not found" - -set_env() { - local key="$1" value="$2" - if grep -q "^${key}=" "$ENV_FILE"; then - sed -i "s|^${key}=.*|${key}=${value}|" "$ENV_FILE" - else - printf '%s=%s\n' "$key" "$value" >>"$ENV_FILE" - fi -} - -# Run a command the way the services run: same user, working directory and -# EnvironmentFile parsing (systemd's, not a .env library's). -as_service() { - if systemd-run --version >/dev/null 2>&1; then - systemd-run --quiet --pipe --wait --collect \ - -p User="$SERVICE_USER" -p Group="$(id -gn "$SERVICE_USER")" \ - -p WorkingDirectory="$APP_DIR" -p EnvironmentFile="$ENV_FILE" "$@" - else - (cd "$APP_DIR" && runuser -u "$SERVICE_USER" -- "$@") - fi -} - -show() { - sleep "$1" - systemctl --no-pager --lines=0 status ping-collector.service || true - journalctl -u ping-collector -n 25 --no-pager || true -} - -env_is() { grep -q "^$1=$2\s*$" "$ENV_FILE"; } - -wait_http() { # url seconds - local i - for i in $(seq "$2"); do - curl -fs -o /dev/null --max-time 5 "$1" 2>/dev/null && return 0 - sleep 1 - done - return 1 -} - -# The Matrix messages come from ping-collector once it notifies; the old -# cron job (`run.py --alert-check`, every 15 min, repeating while a level -# stays high) is commented out, with a backup of the crontab. -retire_matrix_cron() { - env_is FORECAST_IN_COLLECTOR true || return 0 - grep -q '^MATRIX_ACCESS_TOKEN=..*' "$ENV_FILE" || return 0 - local user backup - for user in root "$SERVICE_USER"; do - crontab -l -u "$user" 2>/dev/null | grep -v '^\s*#' | grep -q -- '--alert-check' || continue - backup="${APP_DIR}/crontab-${user}.$(date +%Y%m%d%H%M%S).bak" - crontab -l -u "$user" >"$backup" - sed '/--alert-check/ s|^\([^#]\)|# retired: Matrix is sent by ping-collector # \1|' "$backup" | - crontab -u "$user" - - log "Matrix: commented out the old --alert-check cron job of $user (backup: $backup)" - done - if grep -rlqs -- '--alert-check' /etc/cron.d /etc/crontab 2>/dev/null; then - warn "an --alert-check job is still in $(grep -rls -- '--alert-check' /etc/cron.d /etc/crontab | tr '\n' ' '): remove it, ping-collector sends Matrix now" - fi - if systemctl list-timers --all --no-pager 2>/dev/null | grep -qi alert; then - warn "a systemd timer mentions 'alert' (systemctl list-timers --all): if it runs --alert-check, disable it" - fi -} - -# ping-collector serve next to the Python app; every route compared. -api_check() { - local unit=ping-api-check ok=0 - systemctl stop "$unit" 2>/dev/null || true - systemd-run --quiet --unit="$unit" --collect \ - -p User="$SERVICE_USER" -p Group="$(id -gn "$SERVICE_USER")" \ - -p WorkingDirectory="$APP_DIR" -p EnvironmentFile="$ENV_FILE" \ - "${APP_DIR}/bin/ping-collector" serve --listen "127.0.0.1:${CHECK_PORT}" --no-analytics - if wait_http "http://127.0.0.1:${CHECK_PORT}/api/notifications" 30; then - as_service "$PYTHON" scripts/api_parity.py \ - --python "http://127.0.0.1:${WEB_ADDR##*:}" --rust "http://127.0.0.1:${CHECK_PORT}" && ok=1 - else - warn "ping-collector serve did not answer on port ${CHECK_PORT}:" - journalctl -u "$unit" -n 20 --no-pager || true - fi - systemctl stop "$unit" 2>/dev/null || true - [ "$ok" = 1 ] -} - -rollback_api() { - if env_is WEB_LISTEN '..*'; then - log "The Python app serves the website and API again" - set_env WEB_LISTEN "" - systemctl restart ping-collector.service - fi - systemctl enable water-monitor.service - systemctl restart water-monitor.service -} - -matrix_backups() { - ls "${APP_DIR}"/crontab-*.bak 2>/dev/null | sed 's/^/ /' || true -} - -case "$MODE" in - --rollback) - log "Stopping ping-collector; the Python app collects, serves, forecasts and notifies again" - systemctl disable --now ping-collector.service 2>/dev/null || true - set_env WEB_LISTEN "" - set_env FORECAST_IN_COLLECTOR false - set_env COLLECTION_ENABLED true - systemctl enable water-monitor.service - systemctl restart water-monitor.service - if [ -n "$(matrix_backups)" ]; then - warn "Matrix alerts came from ping-collector; to restore the old cron job: crontab -u USER BACKUP" - matrix_backups - fi - log "Done. Check: journalctl -u water-monitor -n 50" - exit 0 - ;; - --rollback-api) - rollback_api - log "Done. Check: journalctl -u water-monitor -n 50; curl -s http://127.0.0.1:${WEB_ADDR##*:}/health" - exit 0 - ;; - --rollback-forecast) - # the Python app forecasts in its own process: it must serve again - rollback_api - log "Forecasts and notifications back to the Python app; ping-collector keeps collecting" - set_env FORECAST_IN_COLLECTOR false - systemctl restart ping-collector.service water-monitor.service - if [ -n "$(matrix_backups)" ]; then - warn "Matrix alerts stop with this (the Python app sends ntfy only); old cron job: crontab -u USER BACKUP" - matrix_backups - fi - log "Done. Check: journalctl -u water-monitor -n 50" - exit 0 - ;; - --check-api) - [ -x "${APP_DIR}/bin/ping-collector" ] || die "install first" - if env_is WEB_LISTEN '..*'; then - die "ping-collector already serves the website (WEB_LISTEN is set)" - fi - api_check && log "The Rust API answers as the Python app" || die "the API check found differences" - exit 0 - ;; - install | --cutover | --cutover-forecast | --cutover-api) ;; - *) die "unknown option: $MODE" ;; -esac - -BIN="${BIN:-${REPO_DIR}/rust/target/release/ping-collector}" -[ -x "$BIN" ] || die "no binary at $BIN: build it (see header) or pass BIN=/path/to/ping-collector" - -log "Installing $BIN -> ${APP_DIR}/bin/ping-collector ($("$BIN" --version))" -install -D -m 0755 -o root -g root "$BIN" "${APP_DIR}/bin/ping-collector" -install -m 0644 "${REPO_DIR}/scripts/ping-collector.service" /etc/systemd/system/ping-collector.service -systemctl daemon-reload - -# Both services read and write /scraping/status through this file (inside -# the app dir: the units use PrivateTmp). -set_env SCRAPING_STATUS_FILE "$STATUS_FILE" -touch "$STATUS_FILE" -chown "${SERVICE_USER}:$(id -gn "$SERVICE_USER")" "$STATUS_FILE" - -log "Dry run against the configured database (writes nothing)" -as_service "${APP_DIR}/bin/ping-collector" once --dry-run - -forecast_ok=1 -if compgen -G "${APP_DIR}/models/flood_*.joblib" >/dev/null; then - log "Exporting the trained models for the Rust forecaster (models/flood_*.json.gz)" - as_service "$PYTHON" -m src.ml.export - log "Forecast check: Python and ping-collector on the same data, moment and rain" - as_service "$PYTHON" scripts/forecast_parity.py || forecast_ok=0 -else - warn "no trained models in ${APP_DIR}/models: forecast check skipped" - forecast_ok=0 -fi - -case "$MODE" in - --cutover) - log "Cutover: ping-collector collects; the Python app forecasts and notifies" - set_env COLLECTION_ENABLED false - systemctl enable ping-collector.service - systemctl restart ping-collector.service water-monitor.service - show 20 - log "Next step: --cutover-forecast. Rollback: sudo bash scripts/install_collector.sh --rollback" - ;; - --cutover-forecast) - grep -q '^COLLECTION_ENABLED=false' "$ENV_FILE" || - die "ping-collector must collect first: run --cutover" - if [ "$forecast_ok" != 1 ] && [ "${FORCE:-0}" != 1 ]; then - die "the forecast check did not pass; not switching (FORCE=1 overrides)" - fi - log "Cutover: ping-collector also forecasts and notifies; the Python app serves its forecast" - set_env FORECAST_IN_COLLECTOR true - retire_matrix_cron - systemctl enable ping-collector.service - systemctl restart ping-collector.service water-monitor.service - show 30 - log "Next step: --cutover-api. Rollback: sudo bash scripts/install_collector.sh --rollback-forecast" - ;; - --cutover-api) - env_is FORECAST_IN_COLLECTOR true || die "ping-collector must forecast first: run --cutover-forecast" - if env_is WEB_LISTEN '..*'; then - log "ping-collector already serves the website; restarting it on the new binary" - retire_matrix_cron - systemctl restart ping-collector.service - show 10 - exit 0 - fi - api_ok=1 - if [ "$(systemctl is-active water-monitor.service 2>/dev/null)" = active ]; then - log "API check: ping-collector serve on :${CHECK_PORT} next to the Python app, every route compared" - api_check || api_ok=0 - else - warn "the Python app is not running: nothing to compare the API with" - api_ok=0 - fi - if [ "$api_ok" != 1 ] && [ "${FORCE:-0}" != 1 ]; then - die "the API check did not pass; not switching (FORCE=1 overrides)" - fi - log "Cutover: ping-collector serves the website and API on ${WEB_ADDR}; the Python app stops" - set_env WEB_LISTEN "$WEB_ADDR" - retire_matrix_cron - systemctl disable --now water-monitor.service - systemctl restart ping-collector.service - if wait_http "http://127.0.0.1:${WEB_ADDR##*:}/health" 60; then - log "The website answers from ping-collector: $(curl -s --max-time 5 "http://127.0.0.1:${WEB_ADDR##*:}/scraping/status" | grep -o '"collector":"[^"]*"')" - else - warn "no answer on port ${WEB_ADDR##*:} yet; check journalctl -u ping-collector" - fi - show 5 - log "Rollback if needed: sudo bash scripts/install_collector.sh --rollback-api" - ;; - *) - retire_matrix_cron - if [ "$(systemctl is-active ping-collector.service 2>/dev/null)" = active ]; then - systemctl restart ping-collector.service - log "Installed; ping-collector restarted on the new binary" - else - log "Installed, not started. Switch over with: sudo bash scripts/install_collector.sh --cutover" - fi - ;; -esac diff --git a/scripts/install_ntfy.sh b/scripts/install_ntfy.sh index 1f7f8b0..786c7d4 100644 --- a/scripts/install_ntfy.sh +++ b/scripts/install_ntfy.sh @@ -92,7 +92,7 @@ if [[ -f "$env_file" ]] && ! grep -q '^NTFY_SERVER=' "$env_file"; then echo "NTFY_TOPIC_PREFIX=ping" echo "NTFY_TOKEN=${token}" } >> "$env_file" - echo "wrote NTFY_* to ${env_file}; restart water-monitor to enable" + echo "wrote NTFY_* to ${env_file}; restart ping-collector to enable" else echo "NTFY_TOKEN=${token}" fi diff --git a/scripts/load_test.py b/scripts/load_test.py index e528af7..07ccd9e 100644 --- a/scripts/load_test.py +++ b/scripts/load_test.py @@ -9,7 +9,7 @@ and errors per stage plus the slowest endpoints. Run against a LOCAL instance for full stress (never full-stress production — it hosts live flood monitoring): - python -m uvicorn src.web_api:app --port 8125 # separate shell + rust/target/release/ping-collector serve --listen 127.0.0.1:8125 --no-analytics # separate shell python scripts/load_test.py http://localhost:8125 A gentle production baseline (low, fixed concurrency): @@ -98,9 +98,7 @@ def run_stage(base, clients, seconds): by_endpoint = {} for path, elapsed in results: by_endpoint.setdefault(path, []).append(elapsed) - slowest = sorted( - by_endpoint.items(), key=lambda kv: -statistics.median(kv[1]) - )[:4] + slowest = sorted(by_endpoint.items(), key=lambda kv: -statistics.median(kv[1]))[:4] for path, values in slowest: print( f" slow: {path:45} n={len(values):5} " diff --git a/scripts/locustfile.py b/scripts/locustfile.py index 5bb8230..84570e1 100644 --- a/scripts/locustfile.py +++ b/scripts/locustfile.py @@ -7,7 +7,8 @@ the page makes, polling like the auto-refresh does) and API consumers Full stress against a LOCAL instance (never full-stress production — it hosts live flood monitoring): - # separate shell: python -m uvicorn src.web_api:app --port 8125 + # separate shell: + # rust/target/release/ping-collector serve --listen 127.0.0.1:8125 --no-analytics .venv/Scripts/python.exe -m locust -f scripts/locustfile.py \ --host http://localhost:8125 --headless \ --users 200 --spawn-rate 20 --run-time 2m \ @@ -77,9 +78,11 @@ class ApiConsumer(FastHttpUser): @task(3) def hii_feeds(self): - self.client.get(random.choice( - ["/api/hii/waterlevel/latest", "/api/hii/rainfall/latest"] - ), headers=GZIP, name="/api/hii/[feed]/latest") + self.client.get( + random.choice(["/api/hii/waterlevel/latest", "/api/hii/rainfall/latest"]), + headers=GZIP, + name="/api/hii/[feed]/latest", + ) @task(2) def forecast(self): diff --git a/scripts/migrate_sqlite_to_postgres.py b/scripts/migrate_sqlite_to_postgres.py deleted file mode 100644 index 958a6cf..0000000 --- a/scripts/migrate_sqlite_to_postgres.py +++ /dev/null @@ -1,619 +0,0 @@ -#!/usr/bin/env python3 -""" -SQLite to PostgreSQL Migration Tool -Migrates all data from SQLite database to PostgreSQL -""" - -import os -import sys -import logging -import sqlite3 -from datetime import datetime, timezone -from typing import Dict, List, Optional, Tuple, Any -from dataclasses import dataclass - -# Add src to path for imports -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - -@dataclass -class MigrationStats: - stations_migrated: int = 0 - measurements_migrated: int = 0 - errors: List[str] = None - start_time: Optional[datetime] = None - end_time: Optional[datetime] = None - - def __post_init__(self): - if self.errors is None: - self.errors = [] - -class SQLiteToPostgresMigrator: - def __init__(self, sqlite_path: str, postgres_config: Dict[str, Any]): - self.sqlite_path = sqlite_path - self.postgres_config = postgres_config - self.sqlite_conn = None - self.postgres_adapter = None - self.stats = MigrationStats() - - # Setup logging with UTF-8 encoding - logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(levelname)s - %(message)s', - handlers=[ - logging.StreamHandler(), - logging.FileHandler('migration.log', encoding='utf-8') - ] - ) - self.logger = logging.getLogger(__name__) - - def connect_databases(self) -> bool: - """Connect to both SQLite and PostgreSQL databases""" - try: - # Connect to SQLite - if not os.path.exists(self.sqlite_path): - self.logger.error(f"SQLite database not found: {self.sqlite_path}") - return False - - self.sqlite_conn = sqlite3.connect(self.sqlite_path) - self.sqlite_conn.row_factory = sqlite3.Row # For dict-like access - self.logger.info(f"Connected to SQLite database: {self.sqlite_path}") - - # Connect to PostgreSQL - from database_adapters import create_database_adapter - self.postgres_adapter = create_database_adapter( - self.postgres_config['type'], - connection_string=self.postgres_config['connection_string'] - ) - - if not self.postgres_adapter.connect(): - self.logger.error("Failed to connect to PostgreSQL") - return False - - self.logger.info("Connected to PostgreSQL database") - return True - - except Exception as e: - self.logger.error(f"Database connection error: {e}") - return False - - def analyze_sqlite_schema(self) -> Dict[str, List[str]]: - """Analyze SQLite database structure""" - try: - cursor = self.sqlite_conn.cursor() - - # Get all tables - cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'") - tables = [row[0] for row in cursor.fetchall()] - - schema_info = {} - for table in tables: - cursor.execute(f"PRAGMA table_info({table})") - columns = [row[1] for row in cursor.fetchall()] - schema_info[table] = columns - - # Get row count - cursor.execute(f"SELECT COUNT(*) FROM {table}") - count = cursor.fetchone()[0] - self.logger.info(f"Table '{table}': {len(columns)} columns, {count} rows") - - return schema_info - - except Exception as e: - self.logger.error(f"Schema analysis error: {e}") - return {} - - def migrate_stations(self) -> bool: - """Migrate station data""" - try: - cursor = self.sqlite_conn.cursor() - - # Try different possible table names and structures - station_queries = [ - # Modern structure - """SELECT id, station_code, station_name_th as thai_name, station_name_en as english_name, - latitude, longitude, geohash, created_at, updated_at - FROM stations""", - - # Alternative structure 1 - """SELECT id, station_code, thai_name, english_name, - latitude, longitude, geohash, created_at, updated_at - FROM stations""", - - # Legacy structure - """SELECT station_id as id, station_code, station_name as thai_name, - station_name as english_name, lat as latitude, lon as longitude, - NULL as geohash, datetime('now') as created_at, datetime('now') as updated_at - FROM water_stations""", - - # Simple structure - """SELECT rowid as id, station_code, name as thai_name, name as english_name, - NULL as latitude, NULL as longitude, NULL as geohash, - datetime('now') as created_at, datetime('now') as updated_at - FROM stations""", - ] - - stations_data = [] - - for query in station_queries: - try: - cursor.execute(query) - rows = cursor.fetchall() - if rows: - self.logger.info(f"Found {len(rows)} stations using query variant") - - for row in rows: - station = { - 'station_id': row[0], - 'station_code': row[1] or f"STATION_{row[0]}", - 'station_name_th': row[2] or f"Station {row[0]}", - 'station_name_en': row[3] or f"Station {row[0]}", - 'latitude': row[4], - 'longitude': row[5], - 'geohash': row[6], - 'status': 'active' - } - stations_data.append(station) - break - - except sqlite3.OperationalError as e: - if "no such table" in str(e).lower() or "no such column" in str(e).lower(): - continue - else: - raise - - if not stations_data: - self.logger.warning("No stations found in SQLite database") - return True - - # Insert stations into PostgreSQL using raw SQL - # Since the adapter is designed for measurements, we'll use direct SQL - try: - from sqlalchemy import create_engine, text - engine = create_engine(self.postgres_config['connection_string']) - - # Process stations individually to avoid transaction rollback issues - for station in stations_data: - try: - with engine.begin() as conn: - # Use PostgreSQL UPSERT syntax with correct column names - station_sql = """ - INSERT INTO stations (id, station_code, thai_name, english_name, latitude, longitude, geohash) - VALUES (:station_id, :station_code, :thai_name, :english_name, :latitude, :longitude, :geohash) - ON CONFLICT (id) DO UPDATE SET - thai_name = EXCLUDED.thai_name, - english_name = EXCLUDED.english_name, - latitude = EXCLUDED.latitude, - longitude = EXCLUDED.longitude, - geohash = EXCLUDED.geohash, - updated_at = CURRENT_TIMESTAMP - """ - - conn.execute(text(station_sql), { - 'station_id': station['station_id'], - 'station_code': station['station_code'], - 'thai_name': station['station_name_th'], - 'english_name': station['station_name_en'], - 'latitude': station.get('latitude'), - 'longitude': station.get('longitude'), - 'geohash': station.get('geohash') - }) - - self.stats.stations_migrated += 1 - - except Exception as e: - error_msg = f"Error migrating station {station.get('station_code', 'unknown')}: {str(e)[:100]}..." - self.logger.warning(error_msg) - self.stats.errors.append(error_msg) - - self.logger.info(f"Migrated {self.stats.stations_migrated} stations") - - except Exception as e: - self.logger.error(f"Station migration failed: {e}") - return False - - self.logger.info(f"Migrated {self.stats.stations_migrated} stations") - return True - - except Exception as e: - self.logger.error(f"Station migration error: {e}") - return False - - def migrate_measurements(self, batch_size: int = 5000) -> bool: - """Migrate measurement data in batches""" - try: - cursor = self.sqlite_conn.cursor() - - # Try different possible measurement table structures - measurement_queries = [ - # Modern structure - """SELECT w.timestamp, w.station_id, s.station_code, s.station_name_th, s.station_name_en, - w.water_level, w.discharge, w.discharge_percent, w.status - FROM water_measurements w - JOIN stations s ON w.station_id = s.id - ORDER BY w.timestamp""", - - # Alternative with different join - """SELECT w.timestamp, w.station_id, s.station_code, s.thai_name, s.english_name, - w.water_level, w.discharge, w.discharge_percent, 'active' as status - FROM water_measurements w - JOIN stations s ON w.station_id = s.id - ORDER BY w.timestamp""", - - # Legacy structure - """SELECT timestamp, station_id, station_code, station_name, station_name, - water_level, discharge, discharge_percent, 'active' as status - FROM measurements - ORDER BY timestamp""", - - # Simple structure without joins - """SELECT timestamp, station_id, 'UNKNOWN' as station_code, 'Unknown' as station_name_th, 'Unknown' as station_name_en, - water_level, discharge, discharge_percent, 'active' as status - FROM water_measurements - ORDER BY timestamp""", - ] - - measurements_processed = 0 - - for query in measurement_queries: - try: - # Get total count first - count_query = query.replace("SELECT", "SELECT COUNT(*) FROM (SELECT").replace("ORDER BY w.timestamp", "") + ")" - cursor.execute(count_query) - total_measurements = cursor.fetchone()[0] - - if total_measurements == 0: - continue - - self.logger.info(f"Found {total_measurements} measurements to migrate") - - # Process in batches - offset = 0 - while True: - batch_query = f"{query} LIMIT {batch_size} OFFSET {offset}" - cursor.execute(batch_query) - rows = cursor.fetchall() - - if not rows: - break - - # Convert to measurement format - measurements = [] - for row in rows: - try: - # Parse timestamp - timestamp_str = row[0] - if isinstance(timestamp_str, str): - try: - timestamp = datetime.fromisoformat(timestamp_str.replace('Z', '+00:00')) - except: - # Try other common formats - for fmt in ['%Y-%m-%d %H:%M:%S', '%Y-%m-%d %H:%M:%S.%f', '%Y-%m-%dT%H:%M:%S']: - try: - timestamp = datetime.strptime(timestamp_str, fmt) - break - except: - continue - else: - timestamp = datetime.now() - else: - timestamp = timestamp_str - - measurement = { - 'timestamp': timestamp, - 'station_id': row[1] or 999, - 'station_code': row[2] or 'UNKNOWN', - 'station_name_th': row[3] or 'Unknown', - 'station_name_en': row[4] or 'Unknown', - 'water_level': float(row[5]) if row[5] is not None else None, - 'discharge': float(row[6]) if row[6] is not None else None, - 'discharge_percent': float(row[7]) if row[7] is not None else None, - 'status': row[8] or 'active' - } - measurements.append(measurement) - - except Exception as e: - error_msg = f"Error processing measurement row: {e}" - self.logger.warning(error_msg) - continue - - # Save batch to PostgreSQL using fast bulk insert - if measurements: - try: - self._fast_bulk_insert(measurements) - measurements_processed += len(measurements) - self.stats.measurements_migrated += len(measurements) - self.logger.info(f"Migrated {measurements_processed}/{total_measurements} measurements") - except Exception as e: - error_msg = f"Error saving measurement batch: {e}" - self.logger.error(error_msg) - self.stats.errors.append(error_msg) - - offset += batch_size - - # If we processed measurements, we're done - if measurements_processed > 0: - break - - except sqlite3.OperationalError as e: - if "no such table" in str(e).lower() or "no such column" in str(e).lower(): - continue - else: - raise - - if measurements_processed == 0: - self.logger.warning("No measurements found in SQLite database") - else: - self.logger.info(f"Successfully migrated {measurements_processed} measurements") - - return True - - except Exception as e: - self.logger.error(f"Measurement migration error: {e}") - return False - - def _fast_bulk_insert(self, measurements: List[Dict]) -> bool: - """Super fast bulk insert using PostgreSQL COPY or VALUES clause""" - try: - import psycopg2 - from urllib.parse import urlparse - import io - - # Parse connection string for direct psycopg2 connection - parsed = urlparse(self.postgres_config['connection_string']) - - # Try super fast COPY method first - try: - conn = psycopg2.connect( - host=parsed.hostname, - port=parsed.port or 5432, - database=parsed.path[1:], - user=parsed.username, - password=parsed.password - ) - - with conn: - with conn.cursor() as cur: - # Prepare data for COPY - data_buffer = io.StringIO() - null_val = '\\N' - for m in measurements: - data_buffer.write(f"{m['timestamp']}\t{m['station_id']}\t{m['water_level'] or null_val}\t{m['discharge'] or null_val}\t{m['discharge_percent'] or null_val}\t{m['status']}\n") - - data_buffer.seek(0) - - # Use COPY for maximum speed - cur.copy_from( - data_buffer, - 'water_measurements', - columns=('timestamp', 'station_id', 'water_level', 'discharge', 'discharge_percent', 'status'), - sep='\t' - ) - - conn.close() - return True - - except Exception as copy_error: - # Fallback to SQLAlchemy bulk insert - self.logger.debug(f"COPY failed, using bulk VALUES: {copy_error}") - - from sqlalchemy import create_engine, text - engine = create_engine(self.postgres_config['connection_string']) - - with engine.begin() as conn: - # Use PostgreSQL's fast bulk insert with ON CONFLICT - values_list = [] - for m in measurements: - timestamp = m['timestamp'].isoformat() if hasattr(m['timestamp'], 'isoformat') else str(m['timestamp']) - values_list.append( - f"('{timestamp}', {m['station_id']}, {m['water_level'] or 'NULL'}, " - f"{m['discharge'] or 'NULL'}, {m['discharge_percent'] or 'NULL'}, '{m['status']}')" - ) - - # Build bulk insert query with ON CONFLICT handling - bulk_sql = f""" - INSERT INTO water_measurements (timestamp, station_id, water_level, discharge, discharge_percent, status) - VALUES {','.join(values_list)} - ON CONFLICT (timestamp, station_id) DO UPDATE SET - water_level = EXCLUDED.water_level, - discharge = EXCLUDED.discharge, - discharge_percent = EXCLUDED.discharge_percent, - status = EXCLUDED.status - """ - - conn.execute(text(bulk_sql)) - return True - - except Exception as e: - self.logger.warning(f"Fast bulk insert failed: {e}") - # Final fallback to original method - try: - success = self.postgres_adapter.save_measurements(measurements) - return success - except Exception as fallback_e: - self.logger.error(f"All insert methods failed: {fallback_e}") - return False - - def verify_migration(self) -> bool: - """Verify the migration by comparing counts""" - try: - # Get SQLite counts - cursor = self.sqlite_conn.cursor() - - sqlite_stations = 0 - sqlite_measurements = 0 - - # Try to get station count - for table in ['stations', 'water_stations']: - try: - cursor.execute(f"SELECT COUNT(*) FROM {table}") - sqlite_stations = cursor.fetchone()[0] - break - except: - continue - - # Try to get measurement count - for table in ['water_measurements', 'measurements']: - try: - cursor.execute(f"SELECT COUNT(*) FROM {table}") - sqlite_measurements = cursor.fetchone()[0] - break - except: - continue - - # Get PostgreSQL counts - postgres_measurements = self.postgres_adapter.get_latest_measurements(limit=999999) - postgres_count = len(postgres_measurements) - - self.logger.info("Migration Verification:") - self.logger.info(f"SQLite stations: {sqlite_stations}") - self.logger.info(f"SQLite measurements: {sqlite_measurements}") - self.logger.info(f"PostgreSQL measurements retrieved: {postgres_count}") - self.logger.info(f"Migrated stations: {self.stats.stations_migrated}") - self.logger.info(f"Migrated measurements: {self.stats.measurements_migrated}") - - return True - - except Exception as e: - self.logger.error(f"Verification error: {e}") - return False - - def run_migration(self, sqlite_path: str = None) -> bool: - """Run the complete migration process""" - self.stats.start_time = datetime.now() - - if sqlite_path: - self.sqlite_path = sqlite_path - - self.logger.info("=" * 60) - self.logger.info("SQLite to PostgreSQL Migration Tool") - self.logger.info("=" * 60) - self.logger.info(f"SQLite database: {self.sqlite_path}") - self.logger.info(f"PostgreSQL: {self.postgres_config['type']}") - - try: - # Step 1: Connect to databases - self.logger.info("Step 1: Connecting to databases...") - if not self.connect_databases(): - return False - - # Step 2: Analyze SQLite schema - self.logger.info("Step 2: Analyzing SQLite database structure...") - schema_info = self.analyze_sqlite_schema() - if not schema_info: - self.logger.error("Could not analyze SQLite database structure") - return False - - # Step 3: Migrate stations - self.logger.info("Step 3: Migrating station data...") - if not self.migrate_stations(): - self.logger.error("Station migration failed") - return False - - # Step 4: Migrate measurements - self.logger.info("Step 4: Migrating measurement data...") - if not self.migrate_measurements(): - self.logger.error("Measurement migration failed") - return False - - # Step 5: Verify migration - self.logger.info("Step 5: Verifying migration...") - self.verify_migration() - - self.stats.end_time = datetime.now() - duration = self.stats.end_time - self.stats.start_time - - # Final report - self.logger.info("=" * 60) - self.logger.info("MIGRATION COMPLETED") - self.logger.info("=" * 60) - self.logger.info(f"Duration: {duration}") - self.logger.info(f"Stations migrated: {self.stats.stations_migrated}") - self.logger.info(f"Measurements migrated: {self.stats.measurements_migrated}") - - if self.stats.errors: - self.logger.warning(f"Errors encountered: {len(self.stats.errors)}") - for error in self.stats.errors[:10]: # Show first 10 errors - self.logger.warning(f" - {error}") - if len(self.stats.errors) > 10: - self.logger.warning(f" ... and {len(self.stats.errors) - 10} more errors") - else: - self.logger.info("No errors encountered") - - return True - - except Exception as e: - self.logger.error(f"Migration failed: {e}") - return False - - finally: - # Cleanup - if self.sqlite_conn: - self.sqlite_conn.close() - -def main(): - """Main entry point""" - import argparse - - parser = argparse.ArgumentParser(description="Migrate SQLite data to PostgreSQL") - parser.add_argument("sqlite_path", nargs="?", help="Path to SQLite database file") - parser.add_argument("--batch-size", type=int, default=5000, help="Batch size for processing measurements") - parser.add_argument("--fast", action="store_true", help="Use maximum speed mode (batch-size 10000)") - parser.add_argument("--dry-run", action="store_true", help="Analyze only, don't migrate") - - args = parser.parse_args() - - # Set fast mode - if args.fast: - args.batch_size = 10000 - - # Get SQLite path - sqlite_path = args.sqlite_path - if not sqlite_path: - # Try to find common SQLite database files - possible_paths = [ - "water_levels.db", - "water_monitoring.db", - "database.db", - "../water_levels.db" - ] - - for path in possible_paths: - if os.path.exists(path): - sqlite_path = path - break - - if not sqlite_path: - print("SQLite database file not found. Please specify the path:") - print(" python migrate_sqlite_to_postgres.py /path/to/database.db") - return False - - # Get PostgreSQL configuration - try: - from config import Config - postgres_config = Config.get_database_config() - - if postgres_config['type'] != 'postgresql': - print("Error: PostgreSQL not configured. Set DB_TYPE=postgresql in your .env file") - return False - - except Exception as e: - print(f"Error loading PostgreSQL configuration: {e}") - return False - - # Run migration - migrator = SQLiteToPostgresMigrator(sqlite_path, postgres_config) - - if args.dry_run: - print("DRY RUN MODE - Analyzing SQLite database structure only") - if migrator.connect_databases(): - schema_info = migrator.analyze_sqlite_schema() - print("\nSQLite database structure analysis complete.") - print("Run without --dry-run to perform the actual migration.") - return True - - success = migrator.run_migration() - return success - -if __name__ == "__main__": - success = main() - sys.exit(0 if success else 1) \ No newline at end of file diff --git a/scripts/ping-collector.service b/scripts/ping-collector.service index 97948eb..d632787 100644 --- a/scripts/ping-collector.service +++ b/scripts/ping-collector.service @@ -14,10 +14,9 @@ Restart=always RestartSec=30 TimeoutStopSec=20 -# Same .env as water-monitor.service (POSTGRES_*, RID_*, GAP_FILL_*, -# SCRAPING_STATUS_FILE, FORECAST_IN_COLLECTOR, NTFY_*). The Python app must run -# with COLLECTION_ENABLED=false. Models are read from models/ (relative to -# WorkingDirectory). +# Every key with its default: .env.example (POSTGRES_*, WEB_LISTEN, +# FORECAST_IN_COLLECTOR, NTFY_*, MATRIX_*, ...). Models are read from models/ +# (relative to WorkingDirectory); installed by scripts/install.sh. EnvironmentFile=/opt/thailand-water-monitor/.env # Security settings diff --git a/scripts/retrain.sh b/scripts/retrain.sh index 7cd5ff1..90dd2d4 100644 --- a/scripts/retrain.sh +++ b/scripts/retrain.sh @@ -5,15 +5,15 @@ # # Why a script rather than ExecStart=train_flood_model.py: # * train.py writes each station's bundle straight into models/ over ~12 min, -# and the API's hourly precompute reloads bundles by mtime. Training into -# a staging dir and mv-ing (atomic on one filesystem) means the API never -# sees a half-written joblib file or a mixed old/new set. +# and ping-collector reloads bundles by mtime on every forecast run. +# Training into a staging dir and mv-ing (atomic on one filesystem) means +# it never sees a half-written bundle or a mixed old/new set. # * A run that produced gauge-only (v2) bundles, or trained too few stations, # must NOT replace the deployed models. train.py already aborts on a # missing rain series; this script re-checks the written metrics anyway. -# * No restart is needed: predict.py (Python) and ping-collector (Rust, -# FORECAST_IN_COLLECTOR=true) reload changed bundles on their next -# forecast run, so the new models are live within an hour. +# * No restart is needed: ping-collector reloads changed bundles +# (models/flood_*.json.gz, written by train.py next to each .joblib) on +# its next forecast run, so the new models are live within an hour. # # Exit codes: 0 ok, 2 training refused (see log), 3 verification failed. set -euo pipefail @@ -88,4 +88,4 @@ for f in "${STAGE_DIR}"/flood_*.joblib "${STAGE_DIR}"/flood_*.json.gz "${STAGE_D mv -f "${f}" "${MODELS_DIR}/${name}" done rm -rf "${STAGE_DIR}" -log "deployed ${VERSION} (${TRAINED} stations); previous generation in models/.previous. The API picks it up on its next hourly precompute." +log "deployed ${VERSION} (${TRAINED} stations); previous generation in models/.previous. ping-collector loads it on its next forecast run." diff --git a/scripts/setup_postgres.py b/scripts/setup_postgres.py deleted file mode 100644 index 21c2aa1..0000000 --- a/scripts/setup_postgres.py +++ /dev/null @@ -1,175 +0,0 @@ -#!/usr/bin/env python3 -""" -PostgreSQL setup script for Northern Thailand Ping River Monitor -This script helps you configure and test your PostgreSQL connection -""" - -import os -import sys -import logging -from typing import Optional -from urllib.parse import urlparse - -def setup_logging(): - logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') - -def test_postgres_connection(connection_string: str) -> bool: - """Test connection to PostgreSQL database""" - try: - from sqlalchemy import create_engine, text - - # Test connection - engine = create_engine(connection_string, pool_pre_ping=True) - with engine.connect() as conn: - result = conn.execute(text("SELECT version()")) - version = result.fetchone()[0] - logging.info(f"✅ Connected to PostgreSQL successfully!") - logging.info(f"Database version: {version}") - return True - - except ImportError: - logging.error("❌ psycopg2-binary not installed. Run: uv add psycopg2-binary") - return False - except Exception as e: - logging.error(f"❌ Connection failed: {e}") - return False - -def parse_connection_string(connection_string: str) -> dict: - """Parse PostgreSQL connection string into components""" - try: - parsed = urlparse(connection_string) - return { - 'host': parsed.hostname, - 'port': parsed.port or 5432, - 'database': parsed.path[1:] if parsed.path else None, - 'username': parsed.username, - 'password': parsed.password, - } - except Exception as e: - logging.error(f"Failed to parse connection string: {e}") - return {} - -def create_database_if_not_exists(connection_string: str, database_name: str) -> bool: - """Create database if it doesn't exist""" - try: - from sqlalchemy import create_engine, text - - # Connect to default postgres database to create our database - parsed = urlparse(connection_string) - admin_connection = connection_string.replace(f"/{parsed.path[1:]}", "/postgres") - - engine = create_engine(admin_connection, pool_pre_ping=True) - - with engine.connect() as conn: - # Check if database exists - result = conn.execute(text( - "SELECT 1 FROM pg_database WHERE datname = :db_name" - ), {"db_name": database_name}) - - if result.fetchone(): - logging.info(f"✅ Database '{database_name}' already exists") - return True - else: - # Create database - conn.execute(text("COMMIT")) # End transaction - conn.execute(text(f'CREATE DATABASE "{database_name}"')) - logging.info(f"✅ Created database '{database_name}'") - return True - - except Exception as e: - logging.error(f"❌ Failed to create database: {e}") - return False - -def initialize_tables(connection_string: str) -> bool: - """Initialize database tables""" - try: - # Import the database adapter to create tables - sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - from database_adapters import SQLAdapter - - adapter = SQLAdapter(connection_string=connection_string, db_type='postgresql') - if adapter.connect(): - logging.info("✅ Database tables initialized successfully") - return True - else: - logging.error("❌ Failed to initialize tables") - return False - - except Exception as e: - logging.error(f"❌ Failed to initialize tables: {e}") - return False - -def interactive_setup(): - """Interactive setup wizard""" - print("🐘 PostgreSQL Setup Wizard for Ping River Monitor") - print("=" * 50) - - # Get connection details - host = input("PostgreSQL host (e.g., 192.168.1.100): ").strip() - port = input("PostgreSQL port [5432]: ").strip() or "5432" - database = input("Database name [water_monitoring]: ").strip() or "water_monitoring" - username = input("Username: ").strip() - password = input("Password: ").strip() - - # Optional SSL - use_ssl = input("Use SSL connection? (y/N): ").strip().lower() == 'y' - ssl_params = "?sslmode=require" if use_ssl else "" - - connection_string = f"postgresql://{username}:{password}@{host}:{port}/{database}{ssl_params}" - - print(f"\nGenerated connection string:") - print(f"POSTGRES_CONNECTION_STRING={connection_string}") - - return connection_string - -def main(): - setup_logging() - - print("🚀 Northern Thailand Ping River Monitor - PostgreSQL Setup") - print("=" * 60) - - # Check if connection string is provided via environment - connection_string = os.getenv('POSTGRES_CONNECTION_STRING') - - if not connection_string: - print("No POSTGRES_CONNECTION_STRING found in environment.") - print("Starting interactive setup...\n") - connection_string = interactive_setup() - - # Suggest adding to .env file - print(f"\n💡 Add this to your .env file:") - print(f"DB_TYPE=postgresql") - print(f"POSTGRES_CONNECTION_STRING={connection_string}") - - # Parse connection details - config = parse_connection_string(connection_string) - if not config.get('host'): - logging.error("Invalid connection string format") - return False - - print(f"\n🔗 Connecting to PostgreSQL at {config['host']}:{config['port']}") - - # Test connection - if not test_postgres_connection(connection_string): - return False - - # Try to create database - database_name = config.get('database', 'water_monitoring') - if database_name: - create_database_if_not_exists(connection_string, database_name) - - # Initialize tables - if not initialize_tables(connection_string): - return False - - print("\n🎉 PostgreSQL setup completed successfully!") - print("\nNext steps:") - print("1. Update your .env file with the connection string") - print("2. Run: make run-test") - print("3. Run: make run-api") - - return True - -if __name__ == "__main__": - success = main() - sys.exit(0 if success else 1) \ No newline at end of file diff --git a/scripts/setup_uv.bat b/scripts/setup_uv.bat deleted file mode 100644 index 84ef633..0000000 --- a/scripts/setup_uv.bat +++ /dev/null @@ -1,48 +0,0 @@ -@echo off -REM Setup script for uv-based development environment on Windows - -echo 🚀 Setting up Northern Thailand Ping River Monitor with uv... - -REM Check if uv is installed -uv --version >nul 2>&1 -if %errorlevel% neq 0 ( - echo ❌ uv is not installed. Please install it first: - echo powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex" - exit /b 1 -) - -echo ✅ uv found -uv --version - -REM Initialize uv project if not already initialized -if not exist "uv.lock" ( - echo 🔧 Initializing uv project... - uv sync -) else ( - echo 📦 Syncing dependencies with uv... - uv sync -) - -REM Install pre-commit hooks -echo 🎣 Installing pre-commit hooks... -uv run pre-commit install - -REM Create .env file if it doesn't exist -if not exist ".env" ( - if exist ".env.example" ( - echo 📝 Creating .env file from template... - copy .env.example .env - echo ⚠️ Please edit .env file with your configuration - ) -) - -echo ✅ Setup complete! -echo. -echo 📚 Quick start commands: -echo make install-dev # Install all dependencies -echo make run-test # Run a test cycle -echo make run-api # Start the web API -echo make test # Run tests -echo make lint # Check code quality -echo. -echo 🎉 Happy monitoring! \ No newline at end of file diff --git a/scripts/setup_uv.sh b/scripts/setup_uv.sh deleted file mode 100644 index 2299736..0000000 --- a/scripts/setup_uv.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash -# Setup script for uv-based development environment - -set -e - -echo "🚀 Setting up Northern Thailand Ping River Monitor with uv..." - -# Check if uv is installed -if ! command -v uv &> /dev/null; then - echo "❌ uv is not installed. Please install it first:" - echo " curl -LsSf https://astral.sh/uv/install.sh | sh" - exit 1 -fi - -echo "✅ uv found: $(uv --version)" - -# Initialize uv project if not already initialized -if [ ! -f "uv.lock" ]; then - echo "🔧 Initializing uv project..." - uv sync -else - echo "📦 Syncing dependencies with uv..." - uv sync -fi - -# Install pre-commit hooks -echo "🎣 Installing pre-commit hooks..." -uv run pre-commit install - -# Create .env file if it doesn't exist -if [ ! -f ".env" ] && [ -f ".env.example" ]; then - echo "📝 Creating .env file from template..." - cp .env.example .env - echo "⚠️ Please edit .env file with your configuration" -fi - -echo "✅ Setup complete!" -echo "" -echo "📚 Quick start commands:" -echo " make install-dev # Install all dependencies" -echo " make run-test # Run a test cycle" -echo " make run-api # Start the web API" -echo " make test # Run tests" -echo " make lint # Check code quality" -echo "" -echo "🎉 Happy monitoring!" \ No newline at end of file diff --git a/scripts/summarize_eval.py b/scripts/summarize_eval.py index 995e398..24d7a4d 100644 --- a/scripts/summarize_eval.py +++ b/scripts/summarize_eval.py @@ -25,7 +25,8 @@ def summarize(paths): for fold in station["folds"]: for name, m in fold["variants"].items(): a = agg.setdefault( - name, {"mae": [], "mae_hi": [], "brier": [], "fa": 0, "events": []} + name, + {"mae": [], "mae_hi": [], "brier": [], "fa": 0, "events": []}, ) a["mae"].append(m["mae"]) if m.get("mae_above_2p5") is not None: @@ -39,8 +40,18 @@ def summarize(paths): if e["peak_pred_24h_before"] is None else e["peak_pred_24h_before"] - e["peak_level"] ) - a["events"].append((fold["year"], e["crossing"][:10], e["lead_h"], e["peak_level"], err)) - print(f"{'variant':22} {'MAE':>6} {'MAE_hi':>7} {'Brier':>7} {'FA':>3} events: year crossing lead_h peak(err24h)") + a["events"].append( + ( + fold["year"], + e["crossing"][:10], + e["lead_h"], + e["peak_level"], + err, + ) + ) + print( + f"{'variant':22} {'MAE':>6} {'MAE_hi':>7} {'Brier':>7} {'FA':>3} events: year crossing lead_h peak(err24h)" + ) for name, a in agg.items(): ev = " ".join( f"{y} {d} {'—' if l is None else format(l, '+.0f')}h {p:.2f}({'—' if err is None else format(err, '+.2f')})" @@ -54,8 +65,10 @@ def summarize(paths): f"{a['fa']:>3} {ev}" ) if leads: - print(f"{'':22} lead: mean {statistics.mean(leads):+.1f} h, min {min(leads):+.0f} h, " - f"missed {sum(1 for *_, l, _, _ in a['events'] if l is None)}/{len(a['events'])}") + print( + f"{'':22} lead: mean {statistics.mean(leads):+.1f} h, min {min(leads):+.0f} h, " + f"missed {sum(1 for *_, l, _, _ in a['events'] if l is None)}/{len(a['events'])}" + ) if __name__ == "__main__": diff --git a/scripts/water-monitor-retrain.service b/scripts/water-monitor-retrain.service index bcb55fe..295b3eb 100644 --- a/scripts/water-monitor-retrain.service +++ b/scripts/water-monitor-retrain.service @@ -10,11 +10,11 @@ User=water-monitor Group=water-monitor WorkingDirectory=/opt/thailand-water-monitor EnvironmentFile=/opt/thailand-water-monitor/.env -# Same interpreter as water-monitor.service -- the uv-managed .venv. +# The uv-managed .venv (Python training stack; scripts/install.sh syncs it). # scripts/retrain.sh trains into models/.staging, refuses to promote anything # that is not a rain-enabled (hgb-v3) set covering the expected stations, then -# renames the bundles into place. The API reloads them on its next hourly -# precompute; no restart, so a failed run leaves the old models serving. +# renames the bundles into place. ping-collector reloads them on its next +# forecast run; no restart, so a failed run leaves the old models serving. ExecStart=/bin/bash /opt/thailand-water-monitor/scripts/retrain.sh # HistGradientBoosting is CPU-bound; cap threads so training cannot starve # the API (docs/FLOOD_FORECASTING.md section 6 measured 4 as the sweet spot). diff --git a/scripts/water-monitor.service b/scripts/water-monitor.service deleted file mode 100644 index 949707b..0000000 --- a/scripts/water-monitor.service +++ /dev/null @@ -1,41 +0,0 @@ -[Unit] -Description=Thailand Water Level Monitor -Documentation=https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor -After=network.target -Wants=network-online.target - -[Service] -Type=simple -User=water-monitor -Group=water-monitor -WorkingDirectory=/opt/thailand-water-monitor -# The uv-managed env (uv sync -> .venv). Same interpreter for water-monitor-retrain.service. -ExecStart=/opt/thailand-water-monitor/.venv/bin/python run.py --web-api -ExecReload=/bin/kill -HUP $MAINPID -Restart=always -RestartSec=60 -TimeoutStopSec=30 - -# DB_TYPE / POSTGRES_CONNECTION_STRING / MATRIX_* come from the .env file. -EnvironmentFile=/opt/thailand-water-monitor/.env -Environment=PYTHONPATH=/opt/thailand-water-monitor -# Serving path is latency-bound; single-threaded BLAS is 2.6x faster per call -# (docs/FLOOD_FORECASTING.md section 6). Training sets its own value. -Environment=OMP_NUM_THREADS=1 -Environment=PYTHONUNBUFFERED=1 - -# Security settings -NoNewPrivileges=true -PrivateTmp=true -ProtectSystem=strict -ProtectHome=true -ReadWritePaths=/opt/thailand-water-monitor -CapabilityBoundingSet= - -# Logging -StandardOutput=journal -StandardError=journal -SyslogIdentifier=water-monitor - -[Install] -WantedBy=multi-user.target diff --git a/spikes/rust-forecast/.gitignore b/spikes/rust-forecast/.gitignore deleted file mode 100644 index 5d1f033..0000000 --- a/spikes/rust-forecast/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -target/ -data/ diff --git a/spikes/rust-forecast/Cargo.lock b/spikes/rust-forecast/Cargo.lock deleted file mode 100644 index cdec5bd..0000000 --- a/spikes/rust-forecast/Cargo.lock +++ /dev/null @@ -1,132 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "autocfg" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" - -[[package]] -name = "chrono" -version = "0.4.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" -dependencies = [ - "num-traits", -] - -[[package]] -name = "itoa" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" - -[[package]] -name = "memchr" -version = "2.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" - -[[package]] -name = "num-traits" -version = "0.2.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" -dependencies = [ - "autocfg", -] - -[[package]] -name = "ping-forecast-spike" -version = "0.1.0" -dependencies = [ - "chrono", - "serde", - "serde_json", -] - -[[package]] -name = "proc-macro2" -version = "1.0.107" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quote" -version = "1.0.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "serde" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" -dependencies = [ - "serde_core", - "serde_derive", -] - -[[package]] -name = "serde_core" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "serde_json" -version = "1.0.154" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" -dependencies = [ - "itoa", - "memchr", - "serde", - "serde_core", - "zmij", -] - -[[package]] -name = "syn" -version = "3.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "unicode-ident" -version = "1.0.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" - -[[package]] -name = "zmij" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/spikes/rust-forecast/Cargo.toml b/spikes/rust-forecast/Cargo.toml deleted file mode 100644 index 8b14f93..0000000 --- a/spikes/rust-forecast/Cargo.toml +++ /dev/null @@ -1,15 +0,0 @@ -[package] -name = "ping-forecast-spike" -version = "0.1.0" -edition = "2024" -publish = false - -[dependencies] -serde = { version = "1", features = ["derive"] } -# float_roundtrip: exact float parsing. Tree thresholds often equal real data -# values, so a 1-ulp parse error flips `x <= threshold` (default parser is not exact). -serde_json = { version = "1", features = ["float_roundtrip"] } -chrono = { version = "0.4", default-features = false, features = ["std"] } - -[profile.release] -debug = false diff --git a/spikes/rust-forecast/README.md b/spikes/rust-forecast/README.md deleted file mode 100644 index 8174670..0000000 --- a/spikes/rust-forecast/README.md +++ /dev/null @@ -1,43 +0,0 @@ -# Rust forecast spike - -Answers one question: can the flood forecast be served from Rust without -train/serve skew, while training stays in Python (scikit-learn)? -Assessment and migration plan: [docs/RUST_REWRITE.md](../../docs/RUST_REWRITE.md). - -- `export/export_fixture.py`: dumps `models/flood_P.1.joblib` (all 9 HGB heads) as - flat tree arrays, plus two fixtures from `models/cache` (Aug-Oct 2024 record - flood, Jun-Aug 2026): the observed hourly grid of P.1 and its 7 upstream - gauges, rain, the pandas feature matrix and every head's sklearn prediction. -- `src/features.rs`: port of `src/ml/features.py::build_features` (P.1 path: - lags, rises, rolling stats, upstream leads, season, rain). -- `src/hgb.rs`: HistGradientBoosting inference (~90 lines). -- `src/main.rs`: compares bit-for-bit and times serving-shaped work. - -```bash -cd spikes/rust-forecast/export -uv run --project ../../.. python export_fixture.py # writes ../data/*.json (gitignored) -cd .. && cargo run --release -uv run --project ../.. python export/time_python.py # Python timing for comparison -``` - -Result (model hgb-v3+cbb3bf7, 3,600 hourly rows): - -| | Result | -| --- | --- | -| Features, 68 columns x 3,600 rows | 244,800 / 244,800 cells bit-identical | -| Regression heads (max_6/12/24) | bit-identical on every row | -| Classifier heads (warn/danger) | equal to the last ulp (<= 2.2e-16, `exp` vs scipy `expit`) | -| One station: 336 h window -> features + 9 heads | Rust 0.25 ms, Python 158 ms | -| Model load | Rust 16 ms (JSON), Python 199 ms (joblib) | - -Two traps the spike had to get past, both of which flip tree branches because -HGB thresholds often equal real data values exactly: - -1. `serde_json` parses floats best-effort (can be 1 ulp off) unless the - `float_roundtrip` feature is on. -2. pandas `rolling().sum()/.mean()` are incremental with Kahan compensation, - a "run of identical values" shortcut and a sign clamp. A plain window sum - differs in the last bits; `rolling_kahan()` replicates pandas exactly. - -Not covered yet: dam features (Mae Ngat), HII gap fill, Open-Meteo catchment -mean, the heuristic fallback, stage probabilities, the other 14 stations. diff --git a/spikes/rust-forecast/export/export_fixture.py b/spikes/rust-forecast/export/export_fixture.py deleted file mode 100644 index 66f302a..0000000 --- a/spikes/rust-forecast/export/export_fixture.py +++ /dev/null @@ -1,132 +0,0 @@ -"""Export a trained flood bundle + a feature/prediction fixture for the Rust spike. - -Run from the monitor repo's environment: - uv run --project ../../.. python export_fixture.py (from this directory) - -Writes ../data/model_.json (all heads as flat tree arrays) and -../data/fixture_.json (observed hourly grid + rain, the Python feature -matrix, and every head's Python prediction for every row). -""" - -import json -import math -import sys -from pathlib import Path - -import joblib -import numpy as np -import pandas as pd - -REPO = Path(__file__).resolve().parents[3] # repo root -sys.path.insert(0, str(REPO)) - -from src.ml import features # noqa: E402 - -OUT = Path(__file__).resolve().parent.parent / "data" -OUT.mkdir(exist_ok=True) -STATION = "P.1" -WINDOWS = { - "flood2024": ("2024-08-15", "2024-10-31"), # record flood, high-level paths - "recent2026": ("2026-06-01", "2026-08-13"), # latest cached data -} - - -def clean(values): - return [None if (v is None or (isinstance(v, float) and math.isnan(v))) else float(v) for v in values] - - -def export_head(est): - preds = est._predictors - assert all(len(it) == 1 for it in preds), "multiclass not expected" - trees = [] - for (tp,) in preds: - n = tp.nodes - assert not n["is_categorical"].any() - assert np.isfinite(n["value"]).all() and not np.isneginf(n["num_threshold"]).any() - assert not np.isnan(n["num_threshold"]).any() - trees.append( - { - "feature": n["feature_idx"].astype(int).tolist(), - # +inf thresholds (a "missing vs everything" split) are not - # valid JSON: written as null, read back as +inf in Rust - "threshold": [ - None if np.isposinf(t) else float(t) for t in n["num_threshold"] - ], - "missing_left": n["missing_go_to_left"].astype(bool).tolist(), - "left": n["left"].astype(int).tolist(), - "right": n["right"].astype(int).tolist(), - "leaf": n["is_leaf"].astype(bool).tolist(), - "value": n["value"].astype(float).tolist(), - } - ) - kind = "classifier" if hasattr(est, "predict_proba") else "regressor" - return { - "kind": kind, - "baseline": float(np.ravel(est._baseline_prediction)[0]), - "trees": trees, - } - - -bundle = joblib.load(REPO / "models" / f"flood_{STATION}.joblib") -model = { - "station_code": STATION, - "model_version": bundle["model_version"], - "regression_target": bundle.get("regression_target"), - "feature_names": bundle["feature_names"], - "horizons": bundle["horizons"], - "sigma": {str(k): float(v) for k, v in bundle["sigma"].items()}, - "thresholds": bundle["thresholds"], - "heads": {name: export_head(est) for name, est in bundle["heads"].items()}, -} -(OUT / f"model_{STATION}.json").write_text(json.dumps(model)) -n_nodes = sum(len(t["value"]) for h in model["heads"].values() for t in h["trees"]) -print(f"model {model['model_version']}: {len(model['heads'])} heads, {n_nodes} nodes") - -stations = [STATION] + [u for u, _ in features.UPSTREAM_LEADS[STATION]] -long = pd.concat( - pd.read_csv(REPO / "models" / "cache" / f"{s}.csv.gz", parse_dates=["timestamp"]) - for s in stations -) -rain_all = pd.read_csv(REPO / "models" / "cache" / "rain_openmeteo.csv.gz") -print("rain columns:", list(rain_all.columns)) -ts_col = rain_all.columns[0] -rain_all[ts_col] = pd.to_datetime(rain_all[ts_col]) -rain_all = rain_all.set_index(ts_col)[rain_all.columns[1]].astype(float) - -for name, (start, end) in WINDOWS.items(): - df = long[(long.timestamp >= start) & (long.timestamp < end)] - grid = features.make_hourly_grid(df) - idx = grid.observed.index - rain = rain_all.reindex(idx) - # punch deterministic holes so NaN handling of the rolling sums is exercised - rain.iloc[::97] = np.nan - feats = features.build_features(grid, STATION, rain=rain)[model["feature_names"]] - - preds = {} - for head_name, est in bundle["heads"].items(): - if hasattr(est, "predict_proba"): - preds[head_name] = est.predict_proba(feats)[:, 1].tolist() - else: - preds[head_name] = est.predict(feats).tolist() - - fixture = { - "station": STATION, - "timestamps": [t.strftime("%Y-%m-%dT%H:%M:%S") for t in idx], - "observed": { - s: { - "water_level": clean(grid.observed[(s, "water_level")].tolist()), - "discharge": clean(grid.observed[(s, "discharge")].tolist()), - } - for s in stations - if (s, "water_level") in grid.observed.columns - }, - "rain": clean(rain.tolist()), - "feature_names": model["feature_names"], - "features": [clean(row) for row in feats.to_numpy().tolist()], - "predictions": preds, - } - (OUT / f"fixture_{name}.json").write_text(json.dumps(fixture)) - print( - f"{name}: {len(idx)} hours, {len(fixture['observed'])} stations, " - f"P.1 max {np.nanmax(grid.observed[(STATION, 'water_level')]):.2f} m" - ) diff --git a/spikes/rust-forecast/export/time_python.py b/spikes/rust-forecast/export/time_python.py deleted file mode 100644 index 8dcde03..0000000 --- a/spikes/rust-forecast/export/time_python.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Python timing for the same serving-shaped work as the Rust spike.""" - -import sys -import time -from pathlib import Path - -import joblib -import numpy as np -import pandas as pd - -REPO = Path(__file__).resolve().parents[3] # repo root -sys.path.insert(0, str(REPO)) -from src.ml import features # noqa: E402 - -bundle = joblib.load(REPO / "models" / "flood_P.1.joblib") -stations = ["P.1"] + [u for u, _ in features.UPSTREAM_LEADS["P.1"]] -long = pd.concat( - pd.read_csv(REPO / "models" / "cache" / f"{s}.csv.gz", parse_dates=["timestamp"]) - for s in stations -) -df = long[(long.timestamp >= "2026-07-30") & (long.timestamp < "2026-08-13")] -rain_all = pd.read_csv(REPO / "models" / "cache" / "rain_openmeteo.csv.gz", parse_dates=["timestamp"]) -rain_all = rain_all.set_index("timestamp")["chiang_dao"].astype(float) - -reps = 20 -t = time.perf_counter() -for _ in range(reps): - grid = features.make_hourly_grid(df) - rain = rain_all.reindex(grid.observed.index) - row = features.build_features(grid, "P.1", rain=rain)[bundle["feature_names"]].iloc[[-1]] - out = { - k: (h.predict_proba(row)[0, 1] if hasattr(h, "predict_proba") else h.predict(row)[0]) - for k, h in bundle["heads"].items() - } -print(f"python: grid + features + 9 heads for the last hour: {(time.perf_counter() - t) / reps * 1000:.1f} ms per station") -t = time.perf_counter() -joblib.load(REPO / "models" / "flood_P.1.joblib") -print(f"python: bundle load {(time.perf_counter() - t) * 1000:.0f} ms") diff --git a/spikes/rust-forecast/src/features.rs b/spikes/rust-forecast/src/features.rs deleted file mode 100644 index 47994d6..0000000 --- a/spikes/rust-forecast/src/features.rs +++ /dev/null @@ -1,318 +0,0 @@ -//! Port of `src/ml/features.py::build_features` (pandas) to plain Rust. -//! -//! Pandas semantics reproduced: `ffill(limit)`, `shift(k)` (NaN-padded), -//! `rolling(w, min_periods=1)` aggregations that skip NaN and yield NaN for an -//! all-NaN window, NaN propagation through subtraction. - -use chrono::{Datelike, NaiveDateTime}; -use std::collections::HashMap; -use std::f64::consts::PI; - -pub const FFILL_LIMIT_H: usize = 3; -pub const BASIN_ANCHOR: &str = "P.1"; -const MONSOON_MONTHS: [u32; 5] = [6, 7, 8, 9, 10]; - -pub fn upstream_leads(station: &str) -> &'static [(&'static str, i64)] { - match station { - "P.1" => &[ - ("P.103", 1), - ("P.67", 7), - ("P.21", 9), - ("P.75", 12), - ("P.4A", 12), - ("P.92", 15), - ("P.20", 17), - ], - "P.21" => &[ - ("P.67", 1), - ("P.75", 3), - ("P.4A", 3), - ("P.92", 6), - ("P.20", 8), - ], - "P.67" => &[("P.75", 5), ("P.4A", 5), ("P.92", 8), ("P.20", 10)], - "P.75" | "P.4A" => &[("P.92", 3), ("P.20", 5)], - "P.92" => &[("P.20", 2)], - "P.5" => &[("P.1", 12), ("P.103", 13)], - "P.81" => &[("P.1", 4), ("P.103", 5)], - _ => &[], // spike: P.103's table omitted - } -} - -pub type Series = Vec; - -pub struct Grid { - pub times: Vec, - /// station -> observed (unfilled) hourly series, NaN where missing - pub level: HashMap, - pub discharge: HashMap, -} - -fn nan_series(n: usize) -> Series { - vec![f64::NAN; n] -} - -pub fn ffill(v: &[f64], limit: usize) -> Series { - let mut out = v.to_vec(); - let mut last = f64::NAN; - let mut run = 0; - for x in out.iter_mut() { - if x.is_nan() { - run += 1; - if !last.is_nan() && run <= limit { - *x = last; - } - } else { - last = *x; - run = 0; - } - } - out -} - -/// pandas `shift(k)`: positive k looks back, negative k looks ahead. -pub fn shift(v: &[f64], k: i64) -> Series { - let n = v.len() as i64; - (0..n) - .map(|i| { - let j = i - k; - if (0..n).contains(&j) { - v[j as usize] - } else { - f64::NAN - } - }) - .collect() -} - -fn sub(a: &[f64], b: &[f64]) -> Series { - a.iter().zip(b).map(|(x, y)| x - y).collect() -} - -#[derive(Clone, Copy, PartialEq)] -enum Agg { - Mean, - Sum, - Max, - Min, -} - -/// pandas `rolling(w, min_periods=1).()` over the trailing window [i-w+1, i]. -/// -/// Sum/Mean replicate pandas' `roll_sum`/`roll_mean` (window/aggregations.pyx) -/// operation for operation: an incremental window with separate Kahan -/// compensations for adds and removes, the "run of identical values" shortcut -/// and the sign clamp of the mean. A plain per-window sum differs in the last -/// bits, and HGB thresholds often sit exactly on real data values, so last-bit -/// differences flip tree branches. Max/Min involve no arithmetic: exact as is. -fn rolling(v: &[f64], w: usize, agg: Agg) -> Series { - match agg { - Agg::Sum | Agg::Mean => rolling_kahan(v, w, agg == Agg::Mean), - Agg::Max | Agg::Min => (0..v.len()) - .map(|i| { - let lo = (i + 1).saturating_sub(w); - v[lo..=i] - .iter() - .copied() - .filter(|x| !x.is_nan()) - .fold(f64::NAN, |acc, x| { - if acc.is_nan() { - x - } else if agg == Agg::Max { - acc.max(x) - } else { - acc.min(x) - } - }) - }) - .collect(), - } -} - -fn rolling_kahan(v: &[f64], w: usize, mean: bool) -> Series { - let (mut sum, mut comp_add, mut comp_remove) = (0.0f64, 0.0f64, 0.0f64); - let (mut nobs, mut neg_ct, mut same_run) = (0i64, 0i64, 0i64); - let mut prev = if v.is_empty() { f64::NAN } else { v[0] }; - let mut out = Vec::with_capacity(v.len()); - for i in 0..v.len() { - // window [s, e) for row i; previous window was [i-w, i) clipped - if i >= w { - let val = v[i - w]; - if !val.is_nan() { - nobs -= 1; - let y = -val - comp_remove; - let t = sum + y; - comp_remove = t - sum - y; - sum = t; - if val.is_sign_negative() { - neg_ct -= 1; - } - } - } - let val = v[i]; - if !val.is_nan() { - nobs += 1; - let y = val - comp_add; - let t = sum + y; - comp_add = t - sum - y; - sum = t; - if val.is_sign_negative() { - neg_ct += 1; - } - same_run = if val == prev { same_run + 1 } else { 1 }; - prev = val; - } - out.push(if nobs < 1 { - f64::NAN - } else if mean { - if same_run >= nobs { - prev - } else { - let r = sum / nobs as f64; - if neg_ct == 0 && r < 0.0 || neg_ct == nobs && r > 0.0 { - 0.0 - } else { - r - } - } - } else if same_run >= nobs { - prev * nobs as f64 - } else { - sum - }); - } - out -} - -/// Hours since the last observed value (0 when observed; NaN before the first). -fn hours_since_observed(mask: &[bool]) -> Series { - let mut last: Option = None; - mask.iter() - .enumerate() - .map(|(i, &m)| { - if m { - last = Some(i); - } - last.map_or(f64::NAN, |l| (i - l) as f64) - }) - .collect() -} - -/// Feature columns by name, in pandas' construction order. -pub fn build_features(grid: &Grid, station: &str, rain: Option<&[f64]>) -> Vec<(String, Series)> { - let n = grid.times.len(); - let filled = |s: &str| -> Series { - grid.level - .get(s) - .map_or_else(|| nan_series(n), |v| ffill(v, FFILL_LIMIT_H)) - }; - let mut cols: Vec<(String, Series)> = Vec::new(); - let mut put = |name: String, s: Series| cols.push((name, s)); - - let level = filled(station); - let discharge = grid - .discharge - .get(station) - .cloned() - .unwrap_or_else(|| nan_series(n)); - let mask: Vec = grid.level.get(station).map_or_else( - || vec![false; n], - |v| v.iter().map(|x| !x.is_nan()).collect(), - ); - - put("level".into(), level.clone()); - for k in [1, 2, 3, 6, 12, 24, 48, 72] { - put(format!("level_lag_{k}"), shift(&level, k)); - } - for k in [1, 3, 6, 12, 24] { - put(format!("rise_{k}"), sub(&level, &shift(&level, k))); - } - put("roll_mean_6".into(), rolling(&level, 6, Agg::Mean)); - put("roll_mean_24".into(), rolling(&level, 24, Agg::Mean)); - put("roll_max_6".into(), rolling(&level, 6, Agg::Max)); - put("roll_max_24".into(), rolling(&level, 24, Agg::Max)); - put("roll_max_72".into(), rolling(&level, 72, Agg::Max)); - put("roll_min_24".into(), rolling(&level, 24, Agg::Min)); - - put("discharge".into(), discharge.clone()); - put("discharge_lag_6".into(), shift(&discharge, 6)); - put("discharge_lag_24".into(), shift(&discharge, 24)); - put( - "discharge_rise_6".into(), - sub(&discharge, &shift(&discharge, 6)), - ); - - let age = hours_since_observed(&mask); - put( - "obs_age_h".into(), - age.iter() - .map(|&a| { - if a <= FFILL_LIMIT_H as f64 { - a - } else { - f64::NAN - } - }) - .collect(), - ); - let mask_f: Series = mask.iter().map(|&m| if m { 1.0 } else { 0.0 }).collect(); - put("cov_24h".into(), rolling(&mask_f, 24, Agg::Mean)); - - for &(up, lead) in upstream_leads(station) { - let u = filled(up); - let u_rise_6 = sub(&u, &shift(&u, 6)); - let u_rollmax_24 = rolling(&u, 24, Agg::Max); - let near = (lead - 3).max(0); - put(format!("{up}_level_lag_{near}"), shift(&u, near)); - put(format!("{up}_level_lag_{lead}"), shift(&u, lead)); - put(format!("{up}_level_lag_{}", lead + 3), shift(&u, lead + 3)); - put(format!("{up}_rise_6_lag_{lead}"), shift(&u_rise_6, lead)); - put( - format!("{up}_rollmax_24_lag_{near}"), - shift(&u_rollmax_24, near), - ); - } - - if station != BASIN_ANCHOR { - let p1 = filled(BASIN_ANCHOR); - put("P1_level".into(), p1.clone()); - put("P1_rollmax_24".into(), rolling(&p1, 24, Agg::Max)); - put("P1_rise_24".into(), sub(&p1, &shift(&p1, 24))); - } - - let doy: Vec = grid.times.iter().map(|t| t.ordinal() as f64).collect(); - put( - "doy_sin".into(), - doy.iter().map(|d| (2.0 * PI * d / 365.25).sin()).collect(), - ); - put( - "doy_cos".into(), - doy.iter().map(|d| (2.0 * PI * d / 365.25).cos()).collect(), - ); - put( - "is_monsoon".into(), - grid.times - .iter() - .map(|t| { - if MONSOON_MONTHS.contains(&t.month()) { - 1.0 - } else { - 0.0 - } - }) - .collect(), - ); - - if let Some(r) = rain { - put("rain_6h".into(), rolling(r, 6, Agg::Sum)); - put("rain_24h".into(), rolling(r, 24, Agg::Sum)); - put("rain_72h".into(), rolling(r, 72, Agg::Sum)); - // forward sum over (t, t+24] - let mut ahead = shift(r, -1); - ahead.reverse(); - let mut fc = rolling(&ahead, 24, Agg::Sum); - fc.reverse(); - put("rain_fc24".into(), fc); - } - cols -} diff --git a/spikes/rust-forecast/src/hgb.rs b/spikes/rust-forecast/src/hgb.rs deleted file mode 100644 index f3fcf25..0000000 --- a/spikes/rust-forecast/src/hgb.rs +++ /dev/null @@ -1,85 +0,0 @@ -//! Inference for scikit-learn HistGradientBoosting models exported as flat -//! tree arrays (see export/export_fixture.py). Mirrors sklearn's -//! `_predictor.pyx`: NaN follows `missing_go_to_left`, otherwise -//! `x <= threshold` goes left; raw = baseline + sum of leaf values, summed in -//! tree order (same order as sklearn, so results are bit-identical). - -use serde::Deserialize; -use std::collections::BTreeMap; - -fn thresholds<'de, D: serde::Deserializer<'de>>(d: D) -> Result, D::Error> { - let raw: Vec> = Deserialize::deserialize(d)?; - Ok(raw - .into_iter() - .map(|t| t.unwrap_or(f64::INFINITY)) - .collect()) -} - -#[derive(Deserialize)] -pub struct Tree { - feature: Vec, - #[serde(deserialize_with = "thresholds")] - threshold: Vec, - missing_left: Vec, - left: Vec, - right: Vec, - leaf: Vec, - value: Vec, -} - -impl Tree { - #[inline] - fn predict(&self, x: &[f64]) -> f64 { - let mut n = 0; - while !self.leaf[n] { - let v = x[self.feature[n]]; - let go_left = if v.is_nan() { - self.missing_left[n] - } else { - v <= self.threshold[n] - }; - n = if go_left { self.left[n] } else { self.right[n] }; - } - self.value[n] - } -} - -#[derive(Deserialize, PartialEq, Clone, Copy)] -#[serde(rename_all = "lowercase")] -pub enum Kind { - Regressor, - Classifier, -} - -#[derive(Deserialize)] -pub struct Head { - pub kind: Kind, - baseline: f64, - trees: Vec, -} - -impl Head { - /// Regressor: the prediction. Classifier: P(class 1), like predict_proba[:, 1]. - pub fn predict(&self, x: &[f64]) -> f64 { - let mut raw = self.baseline; - for t in &self.trees { - raw += t.predict(x); - } - match self.kind { - Kind::Regressor => raw, - Kind::Classifier => 1.0 / (1.0 + (-raw).exp()), - } - } -} - -#[derive(Deserialize)] -pub struct Bundle { - pub station_code: String, - pub model_version: String, - pub regression_target: Option, - pub feature_names: Vec, - pub horizons: Vec, - pub sigma: BTreeMap, - pub thresholds: BTreeMap, - pub heads: BTreeMap, -} diff --git a/spikes/rust-forecast/src/main.rs b/spikes/rust-forecast/src/main.rs deleted file mode 100644 index 7dfc578..0000000 --- a/spikes/rust-forecast/src/main.rs +++ /dev/null @@ -1,211 +0,0 @@ -//! Parity + timing check: Rust features and HGB inference vs the Python output -//! recorded in data/fixture_*.json. - -mod features; -mod hgb; - -use chrono::NaiveDateTime; -use serde::Deserialize; -use std::collections::HashMap; -use std::time::Instant; - -#[derive(Deserialize)] -struct StationObs { - water_level: Vec>, - discharge: Vec>, -} - -#[derive(Deserialize)] -struct Fixture { - station: String, - timestamps: Vec, - observed: HashMap, - rain: Vec>, - feature_names: Vec, - features: Vec>>, - predictions: HashMap>, -} - -fn nanify(v: &[Option]) -> Vec { - v.iter().map(|x| x.unwrap_or(f64::NAN)).collect() -} - -fn same(a: f64, b: f64, tol: f64) -> bool { - (a.is_nan() && b.is_nan()) || (a - b).abs() <= tol * (1.0 + b.abs()) -} - -fn matrix(cols: &[(String, Vec)], names: &[String]) -> Vec> { - let by_name: HashMap<&str, &Vec> = cols.iter().map(|(k, v)| (k.as_str(), v)).collect(); - let picked: Vec<&Vec> = names - .iter() - .map(|n| { - *by_name - .get(n.as_str()) - .unwrap_or_else(|| panic!("missing feature {n}")) - }) - .collect(); - let rows = picked[0].len(); - (0..rows) - .map(|i| picked.iter().map(|c| c[i]).collect()) - .collect() -} - -fn main() { - let data = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("data"); - let t = Instant::now(); - let bundle: hgb::Bundle = - serde_json::from_slice(&std::fs::read(data.join("model_P.1.json")).unwrap()).unwrap(); - println!( - "model {} ({}): {} heads, {} features, target={:?}, horizons {:?}, sigma {:?}, thresholds {:?}; loaded in {:?}", - bundle.station_code, - bundle.model_version, - bundle.heads.len(), - bundle.feature_names.len(), - bundle.regression_target, - bundle.horizons, - bundle.sigma, - bundle.thresholds, - t.elapsed() - ); - - for name in ["flood2024", "recent2026"] { - let fx: Fixture = serde_json::from_slice( - &std::fs::read(data.join(format!("fixture_{name}.json"))).unwrap(), - ) - .unwrap(); - assert_eq!(fx.feature_names, bundle.feature_names); - let grid = features::Grid { - times: fx - .timestamps - .iter() - .map(|s| NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S").unwrap()) - .collect(), - level: fx - .observed - .iter() - .map(|(k, v)| (k.clone(), nanify(&v.water_level))) - .collect(), - discharge: fx - .observed - .iter() - .map(|(k, v)| (k.clone(), nanify(&v.discharge))) - .collect(), - }; - let rain = nanify(&fx.rain); - - // 1. feature parity - let t = Instant::now(); - let cols = features::build_features(&grid, &fx.station, Some(&rain)); - let feat_time = t.elapsed(); - let rust_x = matrix(&cols, &bundle.feature_names); - let py_x: Vec> = fx.features.iter().map(|r| nanify(r)).collect(); - let (mut cells, mut exact, mut close, mut worst) = - (0usize, 0usize, 0usize, (0.0f64, String::new())); - for (r, p) in rust_x.iter().zip(&py_x) { - for (j, (a, b)) in r.iter().zip(p).enumerate() { - cells += 1; - if a.to_bits() == b.to_bits() || (a.is_nan() && b.is_nan()) { - exact += 1; - } else if same(*a, *b, 1e-9) { - close += 1; - let d = (a - b).abs(); - if d > worst.0 { - worst = (d, bundle.feature_names[j].clone()); - } - } else { - panic!( - "{name}: feature {} differs: rust {a} vs python {b}", - bundle.feature_names[j] - ); - } - } - } - println!( - "\n[{name}] {} rows x {} features: {exact} bit-identical, {close} within 1e-9 (max |diff| {:.1e} in {}), 0 mismatches; Rust feature build {:?}", - rust_x.len(), - bundle.feature_names.len(), - worst.0, - if worst.1.is_empty() { "-" } else { &worst.1 }, - feat_time - ); - let _ = cells; - - // 2. inference parity, on Python's features and on Rust's own features - let t = Instant::now(); - let mut report = Vec::new(); - for (head_name, head) in &bundle.heads { - let want = &fx.predictions[head_name]; - let (mut max_py, mut max_rs, mut ident) = (0.0f64, 0.0f64, 0usize); - for i in 0..rust_x.len() { - let on_py = head.predict(&py_x[i]); - let on_rs = head.predict(&rust_x[i]); - if on_py.to_bits() == want[i].to_bits() { - ident += 1; - } - max_py = max_py.max((on_py - want[i]).abs()); - max_rs = max_rs.max((on_rs - want[i]).abs()); - } - report.push(format!( - " {head_name:<10} bit-identical {ident}/{} max|diff| on py-features {max_py:.1e}, on rust-features {max_rs:.1e}", - rust_x.len() - )); - } - let infer_time = t.elapsed(); - println!("{}", report.join("\n")); - println!( - " {} heads x {} rows x 2 inference passes in {:?}", - bundle.heads.len(), - rust_x.len(), - infer_time - ); - - // 3. serving-shaped work: 14-day window, forecast for the last hour - let tail = 336.min(grid.times.len()); - let start = grid.times.len() - tail; - let window = features::Grid { - times: grid.times[start..].to_vec(), - level: grid - .level - .iter() - .map(|(k, v)| (k.clone(), v[start..].to_vec())) - .collect(), - discharge: grid - .discharge - .iter() - .map(|(k, v)| (k.clone(), v[start..].to_vec())) - .collect(), - }; - let w_rain = rain[start..].to_vec(); - let reps = 200; - let t = Instant::now(); - let mut last = Vec::new(); - for _ in 0..reps { - let cols = features::build_features(&window, &fx.station, Some(&w_rain)); - let x = matrix(&cols, &bundle.feature_names); - let row = x.last().unwrap(); - last = bundle - .heads - .iter() - .map(|(k, h)| (k.clone(), h.predict(row))) - .collect(); - } - let per = t.elapsed() / reps; - let level_now = window.level[&fx.station] - .iter() - .rev() - .find(|x| !x.is_nan()) - .copied() - .unwrap(); - let rise24 = last.iter().find(|(k, _)| k == "max_24").unwrap().1; - let warn24 = last.iter().find(|(k, _)| k == "warn_24").unwrap().1; - println!( - " serving: 336 h window -> features + {} heads for the last hour: {:?} per station; at {} level {:.2} m, predicted 24 h max {:.2} m, warn_24 head p={:.3}", - bundle.heads.len(), - per, - window.times.last().unwrap(), - level_now, - (rise24 + level_now).max(level_now), - warn24 - ); - } -} diff --git a/sql/init_postgres.sql b/sql/init_postgres.sql deleted file mode 100644 index c4b7b46..0000000 --- a/sql/init_postgres.sql +++ /dev/null @@ -1,162 +0,0 @@ --- Northern Thailand Ping River Monitor - PostgreSQL Database Schema --- This script initializes the database tables for water monitoring data - --- Enable required extensions -CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; - --- Create schema for better organization -CREATE SCHEMA IF NOT EXISTS water_monitor; -SET search_path TO water_monitor, public; - --- Stations table - stores monitoring station information -CREATE TABLE IF NOT EXISTS stations ( - id SERIAL PRIMARY KEY, - station_code VARCHAR(10) UNIQUE NOT NULL, - thai_name VARCHAR(255) NOT NULL, - english_name VARCHAR(255) NOT NULL, - latitude DECIMAL(10,8), - longitude DECIMAL(11,8), - geohash VARCHAR(20), - elevation DECIMAL(8,2), -- meters above sea level - river_basin VARCHAR(100), - province VARCHAR(100), - district VARCHAR(100), - is_active BOOLEAN DEFAULT true, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP -); - --- Water measurements table - stores time series data -CREATE TABLE IF NOT EXISTS water_measurements ( - id BIGSERIAL PRIMARY KEY, - timestamp TIMESTAMP NOT NULL, - station_id INTEGER NOT NULL, - water_level NUMERIC(10,3), -- meters - discharge NUMERIC(10,2), -- cubic meters per second - discharge_percent NUMERIC(5,2), -- percentage of normal discharge - status VARCHAR(20) DEFAULT 'active', - data_quality VARCHAR(20) DEFAULT 'good', -- good, fair, poor, missing - remarks TEXT, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - FOREIGN KEY (station_id) REFERENCES stations(id) ON DELETE CASCADE, - UNIQUE(timestamp, station_id) -); - --- Alert thresholds table - stores warning/danger levels for each station -CREATE TABLE IF NOT EXISTS alert_thresholds ( - id SERIAL PRIMARY KEY, - station_id INTEGER NOT NULL, - threshold_type VARCHAR(20) NOT NULL, -- 'warning', 'danger', 'critical' - water_level_min NUMERIC(10,3), - water_level_max NUMERIC(10,3), - discharge_min NUMERIC(10,2), - discharge_max NUMERIC(10,2), - is_active BOOLEAN DEFAULT true, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - FOREIGN KEY (station_id) REFERENCES stations(id) ON DELETE CASCADE -); - --- Data quality log - tracks data collection issues -CREATE TABLE IF NOT EXISTS data_quality_log ( - id BIGSERIAL PRIMARY KEY, - timestamp TIMESTAMP NOT NULL, - station_id INTEGER, - issue_type VARCHAR(50) NOT NULL, -- 'connection_failed', 'invalid_data', 'missing_data' - description TEXT, - severity VARCHAR(20) DEFAULT 'info', -- 'info', 'warning', 'error', 'critical' - resolved_at TIMESTAMP, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - FOREIGN KEY (station_id) REFERENCES stations(id) ON DELETE SET NULL -); - --- Create indexes for better query performance -CREATE INDEX IF NOT EXISTS idx_water_measurements_timestamp ON water_measurements(timestamp DESC); -CREATE INDEX IF NOT EXISTS idx_water_measurements_station_id ON water_measurements(station_id); -CREATE INDEX IF NOT EXISTS idx_water_measurements_station_timestamp ON water_measurements(station_id, timestamp DESC); -CREATE INDEX IF NOT EXISTS idx_water_measurements_status ON water_measurements(status); -CREATE INDEX IF NOT EXISTS idx_stations_code ON stations(station_code); -CREATE INDEX IF NOT EXISTS idx_stations_active ON stations(is_active); -CREATE INDEX IF NOT EXISTS idx_data_quality_timestamp ON data_quality_log(timestamp DESC); -CREATE INDEX IF NOT EXISTS idx_data_quality_station ON data_quality_log(station_id); - --- Create a view for latest measurements per station -CREATE OR REPLACE VIEW latest_measurements AS -SELECT - s.id as station_id, - s.station_code, - s.english_name, - s.thai_name, - s.latitude, - s.longitude, - s.province, - s.river_basin, - m.timestamp, - m.water_level, - m.discharge, - m.discharge_percent, - m.status, - m.data_quality, - CASE - WHEN m.timestamp > CURRENT_TIMESTAMP - INTERVAL '2 hours' THEN 'online' - WHEN m.timestamp > CURRENT_TIMESTAMP - INTERVAL '24 hours' THEN 'delayed' - ELSE 'offline' - END as station_status -FROM stations s -LEFT JOIN LATERAL ( - SELECT * FROM water_measurements - WHERE station_id = s.id - ORDER BY timestamp DESC - LIMIT 1 -) m ON true -WHERE s.is_active = true -ORDER BY s.station_code; - --- Create a function to update the updated_at timestamp -CREATE OR REPLACE FUNCTION update_modified_column() -RETURNS TRIGGER AS $$ -BEGIN - NEW.updated_at = CURRENT_TIMESTAMP; - RETURN NEW; -END; -$$ language 'plpgsql'; - --- Create triggers to automatically update updated_at -DROP TRIGGER IF EXISTS update_stations_modtime ON stations; -CREATE TRIGGER update_stations_modtime - BEFORE UPDATE ON stations - FOR EACH ROW - EXECUTE FUNCTION update_modified_column(); - --- Insert sample stations (Northern Thailand Ping River stations) -INSERT INTO stations (id, station_code, thai_name, english_name, latitude, longitude, province, river_basin) VALUES -(1, 'P.1', 'เชียงใหม่', 'Chiang Mai', 18.7883, 98.9853, 'Chiang Mai', 'Ping River'), -(2, 'P.4A', 'ท่าแพ', 'Tha Phae', 18.7875, 99.0045, 'Chiang Mai', 'Ping River'), -(3, 'P.12', 'สันป่าตอง', 'San Pa Tong', 18.6167, 98.9500, 'Chiang Mai', 'Ping River'), -(4, 'P.20', 'ลำพูน', 'Lamphun', 18.5737, 99.0081, 'Lamphun', 'Ping River'), -(5, 'P.30', 'ลี้', 'Li', 17.4833, 99.3000, 'Lamphun', 'Ping River'), -(6, 'P.35', 'ป่าซาง', 'Pa Sang', 18.5444, 98.9397, 'Lamphun', 'Ping River'), -(7, 'P.67', 'ตาก', 'Tak', 16.8839, 99.1267, 'Tak', 'Ping River'), -(8, 'P.75', 'สามเงา', 'Sam Ngao', 17.1019, 99.4644, 'Tak', 'Ping River') -ON CONFLICT (id) DO NOTHING; - --- Insert sample alert thresholds -INSERT INTO alert_thresholds (station_id, threshold_type, water_level_min, water_level_max) VALUES -(1, 'warning', 4.5, NULL), -(1, 'danger', 6.0, NULL), -(1, 'critical', 7.5, NULL), -(2, 'warning', 4.0, NULL), -(2, 'danger', 5.5, NULL), -(2, 'critical', 7.0, NULL) -ON CONFLICT DO NOTHING; - --- Grant permissions (adjust as needed for your setup) -GRANT USAGE ON SCHEMA water_monitor TO postgres; -GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA water_monitor TO postgres; -GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA water_monitor TO postgres; - --- Optional: Create a read-only user for reporting --- CREATE USER water_monitor_readonly WITH PASSWORD 'readonly_password'; --- GRANT USAGE ON SCHEMA water_monitor TO water_monitor_readonly; --- GRANT SELECT ON ALL TABLES IN SCHEMA water_monitor TO water_monitor_readonly; - -COMMIT; \ No newline at end of file diff --git a/src/__init__.py b/src/__init__.py index 7476191..3b20d6e 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -1,38 +1,5 @@ -#!/usr/bin/env python3 -""" -Northern Thailand Ping River Monitor Package - -A comprehensive real-time water level monitoring system for the Ping River Basin -in Northern Thailand, covering Royal Irrigation Department (RID) stations. -""" +"""Ping River Monitor: the Python side, i.e. the flood-model training and +evaluation stack (src/ml) and the archive backfills. The service (collection, +forecast, notifications, website/API) is ping-collector in rust/.""" __version__ = "3.1.3" -__author__ = "Ping River Monitor Team" -__description__ = "Northern Thailand Ping River Monitoring System" - -from .config import Config -from .database_adapters import DatabaseAdapter, create_database_adapter -from .exceptions import ( - APIConnectionError, - ConfigurationError, - DatabaseConnectionError, - DataValidationError, - WaterMonitorException, -) -from .models import DatabaseConfig, StationInfo, WaterMeasurement -from .water_scraper_v3 import EnhancedWaterMonitorScraper - -__all__ = [ - "EnhancedWaterMonitorScraper", - "create_database_adapter", - "DatabaseAdapter", - "Config", - "WaterMeasurement", - "StationInfo", - "DatabaseConfig", - "WaterMonitorException", - "DatabaseConnectionError", - "APIConnectionError", - "DataValidationError", - "ConfigurationError", -] diff --git a/src/alerting.py b/src/alerting.py deleted file mode 100644 index 04bb789..0000000 --- a/src/alerting.py +++ /dev/null @@ -1,602 +0,0 @@ -#!/usr/bin/env python3 -""" -Water Level Alerting System with Matrix Integration -""" - -import datetime -import html -import os -import re -from dataclasses import dataclass -from enum import Enum -from typing import Dict, List, Optional - -import requests - -try: - from .config import Config - from .database_adapters import create_database_adapter - from .logging_config import get_logger -except ImportError: - import logging - - from config import Config - from database_adapters import create_database_adapter - - def get_logger(name): - return logging.getLogger(name) - - -logger = get_logger(__name__) - - -_BOLD_RE = re.compile(r"\*\*(.+?)\*\*") -_URL_RE = re.compile(r"(https?://[^\s<]+)") - - -def markdown_to_matrix_html(text: str) -> str: - """Convert the small Markdown subset we emit into Matrix-compatible HTML. - - Matrix clients do NOT render Markdown in the plain ``body`` field; formatting - only shows when an HTML ``formatted_body`` is sent alongside it. We only use - ``**bold**``, bare URLs and newlines, so a minimal converter is sufficient and - avoids adding a Markdown dependency. - """ - # Escape HTML special chars first so station/message data can't inject markup. - result = html.escape(text, quote=False) - result = _BOLD_RE.sub(r"\1", result) - result = _URL_RE.sub(r'\1', result) - result = result.replace("\n", "
") - return result - - -def strip_markdown(text: str) -> str: - """Produce a clean plain-text fallback for the Matrix ``body`` field.""" - return _BOLD_RE.sub(r"\1", text) - - -class AlertLevel(Enum): - INFO = "info" - WARNING = "warning" - CRITICAL = "critical" - EMERGENCY = "emergency" - - -@dataclass -class WaterAlert: - station_code: str - station_name: str - alert_type: str - level: AlertLevel - water_level: float - threshold: float - discharge: Optional[float] = None - timestamp: Optional[datetime.datetime] = None - message: Optional[str] = None - - -class MatrixNotifier: - def __init__(self, homeserver: str, access_token: str, room_id: str): - self.homeserver = homeserver.rstrip("/") - self.access_token = access_token - self.room_id = room_id - self.session = requests.Session() - - def send_message( - self, message: str, msgtype: str = "m.text", markdown: bool = True - ) -> bool: - """Send a message to the Matrix room. - - When ``markdown`` is True (default) the ``message`` is treated as Markdown: - a rendered HTML ``formatted_body`` is sent so clients show real formatting, - with a plain-text ``body`` fallback for clients that ignore HTML. - """ - try: - # Add transaction ID to prevent duplicates - txn_id = datetime.datetime.now().strftime("%Y%m%d_%H%M%S_%f") - url = f"{self.homeserver}/_matrix/client/v3/rooms/{self.room_id}/send/m.room.message/{txn_id}" - - headers = { - "Authorization": f"Bearer {self.access_token}", - "Content-Type": "application/json", - } - - if markdown: - data = { - "msgtype": msgtype, - "body": strip_markdown(message), - "format": "org.matrix.custom.html", - "formatted_body": markdown_to_matrix_html(message), - } - else: - data = {"msgtype": msgtype, "body": message} - - # Matrix API requires PUT when transaction ID is in the URL path - response = self.session.put(url, headers=headers, json=data, timeout=10) - response.raise_for_status() - - logger.info( - f"Matrix message sent successfully: {response.json().get('event_id')}" - ) - return True - - except Exception as e: - logger.error(f"Failed to send Matrix message: {e}") - return False - - def send_alert(self, alert: WaterAlert) -> bool: - """Send formatted water alert to Matrix""" - emoji_map = { - AlertLevel.INFO: "ℹ️", - AlertLevel.WARNING: "⚠️", - AlertLevel.CRITICAL: "🚨", - AlertLevel.EMERGENCY: "🆘", - } - - emoji = emoji_map.get(alert.level, "📊") - - message = f"""{emoji} **WATER LEVEL ALERT** - -**Station:** {alert.station_code} ({alert.station_name}) -**Alert Type:** {alert.alert_type} -**Severity:** {alert.level.value.upper()} - -**Current Level:** {alert.water_level:.2f}m -**Threshold:** {alert.threshold:.2f}m -**Difference:** {(alert.water_level - alert.threshold):+.2f}m -""" - - if alert.discharge: - message += f"**Discharge:** {alert.discharge:.1f} cms\n" - - if alert.timestamp: - message += f"**Time:** {alert.timestamp.strftime('%Y-%m-%d %H:%M:%S')}\n" - - if alert.message: - message += f"\n**Details:** {alert.message}\n" - - # Add live dashboard link - dashboard_url = os.getenv("ALERT_DASHBOARD_URL", "https://water.buildfor.life/") - message += f"\n📈 **View Dashboard:** {dashboard_url}" - - return self.send_message(message) - - -class WaterLevelAlertSystem: - # Stations upstream of Chiang Mai (and CNX itself) to monitor - UPSTREAM_STATIONS = { - "P.20", # Ban Chiang Dao - "P.75", # Ban Chai Lat - "P.92", # Ban Muang Aut - "P.4A", # Ban Mae Taeng - "P.67", # Ban Tae - "P.21", # Ban Rim Tai - "P.103", # Ring Bridge 3 - "P.1", # Nawarat Bridge (Chiang Mai) - } - - def __init__(self): - self.db_adapter = None - self.matrix_notifier = None - self.thresholds = self._load_thresholds() - - # Matrix configuration from environment - matrix_homeserver = os.getenv("MATRIX_HOMESERVER", "https://matrix.org") - matrix_token = os.getenv("MATRIX_ACCESS_TOKEN") - matrix_room = os.getenv("MATRIX_ROOM_ID") - - if matrix_token and matrix_room: - self.matrix_notifier = MatrixNotifier( - matrix_homeserver, matrix_token, matrix_room - ) - logger.info("Matrix notifications enabled") - else: - logger.warning("Matrix configuration missing - notifications disabled") - - def _load_thresholds(self) -> Dict[str, Dict[str, float]]: - """Load alert thresholds from config or database""" - # Default thresholds for Northern Thailand stations - return { - "P.1": { - # Zone-based thresholds for Nawarat Bridge (P.1) - "zone_1": 3.7, - "zone_2": 3.9, - "zone_3": 4.0, - "zone_4": 4.1, - "zone_5": 4.2, - "zone_6": 4.3, - "zone_7": 4.6, - "zone_8": 4.8, - "newedge": 4.8, # Same as zone 8 or adjust as needed - # Keep legacy thresholds for compatibility - "warning": 3.7, - "critical": 4.3, - "emergency": 4.8, - }, - "P.4A": {"warning": 4.5, "critical": 6.0, "emergency": 7.5}, - "P.20": {"warning": 3.0, "critical": 4.5, "emergency": 6.0}, - "P.21": {"warning": 4.0, "critical": 5.5, "emergency": 7.0}, - "P.67": {"warning": 6.0, "critical": 8.0, "emergency": 10.0}, - "P.75": {"warning": 5.5, "critical": 7.5, "emergency": 9.5}, - "P.103": {"warning": 7.0, "critical": 9.0, "emergency": 11.0}, - # Default for unknown stations - "default": {"warning": 4.0, "critical": 6.0, "emergency": 8.0}, - } - - def connect_database(self): - """Initialize database connection""" - try: - db_config = Config.get_database_config() - self.db_adapter = create_database_adapter( - db_config["type"], connection_string=db_config["connection_string"] - ) - - if self.db_adapter.connect(): - logger.info("Database connection established for alerting") - return True - else: - logger.error("Failed to connect to database") - return False - - except Exception as e: - logger.error(f"Database connection error: {e}") - return False - - def check_water_levels(self) -> List[WaterAlert]: - """Check current water levels against thresholds""" - alerts = [] - - if not self.db_adapter: - logger.error("Database not connected") - return alerts - - try: - # Get latest measurements - measurements = self.db_adapter.get_latest_measurements(limit=50) - - for measurement in measurements: - station_code = measurement.get("station_code", "UNKNOWN") - water_level = measurement.get("water_level") - - if not water_level: - continue - - # Only alert for upstream stations and Chiang Mai - if station_code not in self.UPSTREAM_STATIONS: - continue - - # Get thresholds for this station - station_thresholds = self.thresholds.get( - station_code, self.thresholds["default"] - ) - - # Check each threshold level - alert_level = None - threshold_value = None - alert_type = None - - # Special handling for P.1 with zone-based thresholds - if station_code == "P.1" and "zone_1" in station_thresholds: - # Check all zones in reverse order (highest to lowest) - zones = [ - ("zone_8", 4.8, AlertLevel.EMERGENCY, "Zone 8 - Emergency"), - ("newedge", 4.8, AlertLevel.EMERGENCY, "NewEdge Alert Level"), - ("zone_7", 4.6, AlertLevel.CRITICAL, "Zone 7 - Critical"), - ("zone_6", 4.3, AlertLevel.CRITICAL, "Zone 6 - Critical"), - ("zone_5", 4.2, AlertLevel.WARNING, "Zone 5 - Warning"), - ("zone_4", 4.1, AlertLevel.WARNING, "Zone 4 - Warning"), - ("zone_3", 4.0, AlertLevel.WARNING, "Zone 3 - Warning"), - ("zone_2", 3.9, AlertLevel.INFO, "Zone 2 - Info"), - ("zone_1", 3.7, AlertLevel.INFO, "Zone 1 - Info"), - ] - - for ( - zone_name, - zone_threshold, - zone_alert_level, - zone_description, - ) in zones: - if water_level >= zone_threshold: - alert_level = zone_alert_level - threshold_value = zone_threshold - alert_type = zone_description - break - - else: - # Standard threshold checking for other stations - if water_level >= station_thresholds.get("emergency", float("inf")): - alert_level = AlertLevel.EMERGENCY - threshold_value = station_thresholds["emergency"] - alert_type = "Emergency Water Level" - elif water_level >= station_thresholds.get( - "critical", float("inf") - ): - alert_level = AlertLevel.CRITICAL - threshold_value = station_thresholds["critical"] - alert_type = "Critical Water Level" - elif water_level >= station_thresholds.get("warning", float("inf")): - alert_level = AlertLevel.WARNING - threshold_value = station_thresholds["warning"] - alert_type = "High Water Level" - - if alert_level: - alert = WaterAlert( - station_code=station_code, - station_name=measurement.get( - "station_name_th", f"Station {station_code}" - ), - alert_type=alert_type, - level=alert_level, - water_level=water_level, - threshold=threshold_value, - discharge=measurement.get("discharge"), - timestamp=measurement.get("timestamp"), - ) - alerts.append(alert) - - except Exception as e: - logger.error(f"Error checking water levels: {e}") - - return alerts - - def check_data_freshness(self, max_age_hours: int = 12) -> List[WaterAlert]: - """Check if data is fresh enough""" - alerts = [] - - if not self.db_adapter: - return alerts - - try: - measurements = self.db_adapter.get_latest_measurements(limit=20) - cutoff_time = datetime.datetime.now() - datetime.timedelta( - hours=max_age_hours - ) - - for measurement in measurements: - timestamp = measurement.get("timestamp") - if timestamp and timestamp < cutoff_time: - station_code = measurement.get("station_code", "UNKNOWN") - - age_hours = ( - datetime.datetime.now() - timestamp - ).total_seconds() / 3600 - - alert = WaterAlert( - station_code=station_code, - station_name=measurement.get( - "station_name_th", f"Station {station_code}" - ), - alert_type="Stale Data", - level=AlertLevel.WARNING, - water_level=measurement.get("water_level", 0), - threshold=max_age_hours, - timestamp=timestamp, - message=f"No fresh data for {age_hours:.1f} hours", - ) - alerts.append(alert) - - except Exception as e: - logger.error(f"Error checking data freshness: {e}") - - return alerts - - def check_rate_of_change(self, lookback_hours: int = 3) -> List[WaterAlert]: - """Check for rapid water level changes over recent hours""" - alerts = [] - - if not self.db_adapter: - return alerts - - try: - # Define rate-of-change thresholds (meters per hour) - rate_thresholds = { - "P.1": { - "warning": 0.15, # 15cm/hour - moderate rise - "critical": 0.25, # 25cm/hour - rapid rise - "emergency": 0.40, # 40cm/hour - very rapid rise - }, - "default": {"warning": 0.20, "critical": 0.35, "emergency": 0.50}, - } - - # Get recent measurements for each station - cutoff_time = datetime.datetime.now() - datetime.timedelta( - hours=lookback_hours - ) - - # Get unique stations from latest data - latest = self.db_adapter.get_latest_measurements(limit=20) - station_codes = set( - m.get("station_code") for m in latest if m.get("station_code") - ) - - for station_code in station_codes: - try: - # Only alert for upstream stations and Chiang Mai - if station_code not in self.UPSTREAM_STATIONS: - continue - - # Get measurements for this station in the time window - current_time = datetime.datetime.now() - measurements = self.db_adapter.get_measurements_by_timerange( - start_time=cutoff_time, - end_time=current_time, - station_codes=[station_code], - ) - - if len(measurements) < 2: - continue # Need at least 2 points to calculate rate - - # Sort by timestamp - measurements = sorted( - measurements, key=lambda m: m.get("timestamp") - ) - - # Get oldest and newest measurements - oldest = measurements[0] - newest = measurements[-1] - - oldest_time = oldest.get("timestamp") - oldest_level = oldest.get("water_level") - newest_time = newest.get("timestamp") - newest_level = newest.get("water_level") - - # Convert timestamp strings to datetime if needed - if isinstance(oldest_time, str): - oldest_time = datetime.datetime.fromisoformat(oldest_time) - if isinstance(newest_time, str): - newest_time = datetime.datetime.fromisoformat(newest_time) - - # Calculate rate of change - time_diff_hours = (newest_time - oldest_time).total_seconds() / 3600 - if time_diff_hours == 0: - continue - - level_change = newest_level - oldest_level - rate_per_hour = level_change / time_diff_hours - - # Only alert on rising water (positive rate) - if rate_per_hour <= 0: - continue - - # Get station info from latest data - station_info = next( - (m for m in latest if m.get("station_code") == station_code), {} - ) - station_name = station_info.get("station_name_th", station_code) - - # Get thresholds for this station - station_rate_threshold = rate_thresholds.get( - station_code, rate_thresholds["default"] - ) - - alert_level = None - threshold_value = None - alert_type = None - - if rate_per_hour >= station_rate_threshold["emergency"]: - alert_level = AlertLevel.EMERGENCY - threshold_value = station_rate_threshold["emergency"] - alert_type = "Very Rapid Water Level Rise" - elif rate_per_hour >= station_rate_threshold["critical"]: - alert_level = AlertLevel.CRITICAL - threshold_value = station_rate_threshold["critical"] - alert_type = "Rapid Water Level Rise" - elif rate_per_hour >= station_rate_threshold["warning"]: - alert_level = AlertLevel.WARNING - threshold_value = station_rate_threshold["warning"] - alert_type = "Moderate Water Level Rise" - - if alert_level: - message = ( - f"Rising at {rate_per_hour:.2f}m/h over last {time_diff_hours:.1f}h " - f"(change: {level_change:+.2f}m)" - ) - - alert = WaterAlert( - station_code=station_code, - station_name=station_name or f"Station {station_code}", - alert_type=alert_type, - level=alert_level, - water_level=newest_level, - threshold=threshold_value, - timestamp=newest_time, - message=message, - ) - alerts.append(alert) - - except Exception as station_error: - logger.debug( - f"Error checking rate of change for station {station_code}: {station_error}" - ) - continue - - except Exception as e: - logger.error(f"Error checking rate of change: {e}") - - return alerts - - def send_alerts(self, alerts: List[WaterAlert]) -> int: - """Send alerts via configured channels""" - sent_count = 0 - - if not alerts: - return sent_count - - if self.matrix_notifier: - for alert in alerts: - if self.matrix_notifier.send_alert(alert): - sent_count += 1 - - # Could add other notification channels here: - # - Email - # - Discord - # - Telegram - # - SMS - - return sent_count - - def run_alert_check(self) -> Dict[str, int]: - """Run complete alert check cycle""" - if not self.connect_database(): - return {"error": 1} - - # Check water levels - water_alerts = self.check_water_levels() - - # Check data freshness - data_alerts = self.check_data_freshness() - - # Check rate of change (rapid rises) - rate_alerts = self.check_rate_of_change() - - # Combine alerts - all_alerts = water_alerts + rate_alerts - - # Send alerts - sent_count = self.send_alerts(all_alerts) - - logger.info( - f"Alert check complete: {len(all_alerts)} alerts, {sent_count} sent" - ) - - return { - "water_alerts": len(water_alerts), - "data_alerts": len(data_alerts), - "rate_alerts": len(rate_alerts), - "total_alerts": len(all_alerts), - "sent": sent_count, - } - - -def main(): - """Standalone alerting check""" - import argparse - - parser = argparse.ArgumentParser(description="Water Level Alert System") - parser.add_argument("--check", action="store_true", help="Run alert check") - parser.add_argument("--test", action="store_true", help="Send test message") - args = parser.parse_args() - - alerting = WaterLevelAlertSystem() - - if args.test: - if alerting.matrix_notifier: - test_message = ( - "🧪 **Test Alert**\n\nThis is a test message from the Water Level Alert System.\n\n" - "If you received this, Matrix notifications are working correctly!" - ) - success = alerting.matrix_notifier.send_message(test_message) - print(f"Test message sent: {success}") - else: - print("Matrix notifier not configured") - - elif args.check: - results = alerting.run_alert_check() - print(f"Alert check results: {results}") - - else: - print("Use --check or --test") - - -if __name__ == "__main__": - main() diff --git a/src/config.py b/src/config.py index 2bdeb71..36b3139 100644 --- a/src/config.py +++ b/src/config.py @@ -1,354 +1,43 @@ -import os -from typing import Any, Dict +"""Database settings for the Python tools (training, evaluation, backfills). + +The service itself is ping-collector (rust/collector); it reads the same +environment / `.env` file (rust/collector/src/config.rs). PostgreSQL only: +POSTGRES_CONNECTION_STRING, or POSTGRES_HOST / POSTGRES_PORT / POSTGRES_DB / +POSTGRES_USER / POSTGRES_PASSWORD with the same defaults as ping-collector. +""" + +import os +import urllib.parse +from typing import Optional -# Load environment variables from .env file try: from dotenv import load_dotenv load_dotenv() -except ImportError: - # python-dotenv not installed, continue without it +except ImportError: # the systemd units pass the variables in directly pass -try: - from .exceptions import ConfigurationError - from .models import DatabaseType -except ImportError: - # Handle case when running as standalone script - class ConfigurationError(Exception): - pass - from enum import Enum - - class DatabaseType(Enum): - SQLITE = "sqlite" - MYSQL = "mysql" - POSTGRESQL = "postgresql" - INFLUXDB = "influxdb" - VICTORIAMETRICS = "victoriametrics" - - -class Config: - """Configuration class for the Water Level Monitor""" - - # Database settings - DATABASE_PATH = os.getenv("WATER_DB_PATH", "water_levels.db") - - # Website settings - TARGET_URL = "https://hyd-app-db.rid.go.th/hydro1h.html" - API_URL = "https://hyd-app-db.rid.go.th/webservice/getGroupHourlyWaterLevelReportAllHL.ashx" - THAIWATER_API_KEY = os.getenv("THAIWATER_API_KEY") - - # Public flood notifications (ntfy). Off unless NTFY_SERVER is set. - # NTFY_SERVER is what subscribers use (public https URL, shown on the - # dashboard). NTFY_PUBLISH_URL is where the monitor POSTs; defaults to - # NTFY_SERVER, set it to http://127.0.0.1:2586 when ntfy runs on the same - # host so publishing never depends on DNS/proxy/tunnel being up. - NTFY_SERVER = os.getenv("NTFY_SERVER", "").strip() - NTFY_PUBLISH_URL = os.getenv("NTFY_PUBLISH_URL", "").strip() or NTFY_SERVER - NTFY_TOPIC_PREFIX = os.getenv("NTFY_TOPIC_PREFIX", "ping").strip() - NTFY_TOKEN = os.getenv("NTFY_TOKEN", "").strip() # publish token if ACL enabled - PUBLIC_URL = os.getenv("PUBLIC_URL", "https://water.buildfor.life/").strip() - REQUEST_TIMEOUT = int(os.getenv("REQUEST_TIMEOUT", "30")) - USER_AGENT = ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " - "(KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36" +def database_url() -> Optional[str]: + """The PostgreSQL connection string, or None when none is configured.""" + url = os.getenv("POSTGRES_CONNECTION_STRING", "").strip() + if url: + return url + password = os.getenv("POSTGRES_PASSWORD") + if not password: + return None + user = urllib.parse.quote(os.getenv("POSTGRES_USER", "postgres"), safe="") + host = os.getenv("POSTGRES_HOST", "localhost") + port = os.getenv("POSTGRES_PORT", "5432") + name = os.getenv("POSTGRES_DB", "water_monitoring") + return ( + f"postgresql://{user}:{urllib.parse.quote(password, safe='')}" + f"@{host}:{port}/{name}" ) - # Database configuration - # When DB_TYPE is not set explicitly, a configured Postgres connection wins over the sqlite default - DB_TYPE = os.getenv( - "DB_TYPE", - "postgresql" if os.getenv("POSTGRES_CONNECTION_STRING") else "sqlite", - ).lower() - - # VictoriaMetrics settings - # Default to localhost; set VM_HOST in the environment for real deployments - # (avoids committing infrastructure hostnames to the repo). - VM_HOST = os.getenv("VM_HOST", "localhost") - VM_PORT = int(os.getenv("VM_PORT", "443")) - - # Support for HTTPS URLs (e.g., behind reverse proxy) - VM_URL = os.getenv("VM_URL") # Full URL override (e.g., https://vm.example.com) - - # InfluxDB settings - INFLUX_HOST = os.getenv("INFLUX_HOST", "localhost") - INFLUX_PORT = int(os.getenv("INFLUX_PORT", "8086")) - INFLUX_DATABASE = os.getenv("INFLUX_DATABASE", "water_monitoring") - INFLUX_USERNAME = os.getenv("INFLUX_USERNAME") - INFLUX_PASSWORD = os.getenv("INFLUX_PASSWORD") - - # PostgreSQL settings - POSTGRES_CONNECTION_STRING = os.getenv("POSTGRES_CONNECTION_STRING") - POSTGRES_HOST = os.getenv("POSTGRES_HOST", "localhost") - POSTGRES_PORT = int(os.getenv("POSTGRES_PORT", "5432")) - POSTGRES_DB = os.getenv("POSTGRES_DB", "water_monitoring") - POSTGRES_USER = os.getenv("POSTGRES_USER", "postgres") - POSTGRES_PASSWORD = os.getenv("POSTGRES_PASSWORD") - - # MySQL settings - MYSQL_CONNECTION_STRING = os.getenv("MYSQL_CONNECTION_STRING") - - # HII/ThaiWater open api-v3 collection (rainfall + backup water level) - # See docs/DATA_SOURCES.md. Requires a SQL DB_TYPE (sqlite/postgresql/mysql). - ENABLE_HII_COLLECTION = os.getenv("ENABLE_HII_COLLECTION", "true").lower() in ( - "1", - "true", - "yes", - ) - HII_BASIN_CODE = int(os.getenv("HII_BASIN_CODE", "6")) # 6 = Ping Basin - - # RID large-dam daily status (app.rid.go.th/reservoir) — Mae Ngat et al. - ENABLE_RESERVOIR_COLLECTION = os.getenv( - "ENABLE_RESERVOIR_COLLECTION", "true" - ).lower() in ("1", "true", "yes") - # TTL for the /api/hii/*/latest response cache; source data changes hourly - HII_CACHE_TTL_SECONDS = int(os.getenv("HII_CACHE_TTL_SECONDS", "120")) - # TTL for the /measurements/latest response cache (hottest endpoint) - LATEST_CACHE_TTL_SECONDS = int(os.getenv("LATEST_CACHE_TTL_SECONDS", "45")) - - # TTL for /health check results (includes an external RID-API probe) - HEALTH_CACHE_TTL_SECONDS = int(os.getenv("HEALTH_CACHE_TTL_SECONDS", "30")) - - # Thread-pool size for blocking work in the web process (DB queries, - # inference, health probes). Waiting threads are cheap; starving the pool - # stalls every endpoint that needs a thread. - EXECUTOR_THREADS = int(os.getenv("EXECUTOR_THREADS", "48")) - - # Web server worker processes. Above 1, uvicorn forks workers and a - # localhost lock port elects a single background-collection leader. - WEB_WORKERS = int(os.getenv("WEB_WORKERS", "2")) - COLLECTION_LEADER_PORT = int(os.getenv("COLLECTION_LEADER_PORT", "8901")) - - # Umami analytics (self-hosted). The website id is public (it ships in the - # dashboard