diff --git a/.env.example b/.env.example index e058280..e300f92 100644 --- a/.env.example +++ b/.env.example @@ -58,6 +58,10 @@ RID_POLL_OFFSET_MINUTES=5 RID_RETRY_MINUTES=10 # Days of RID holes the leader re-fetches at startup (0 = off) GAP_FILL_DAYS_ON_START=14 +# false when the Rust ping-collector service collects (rust/collector); +# share the status file between both services for /scraping/status +COLLECTION_ENABLED=true +# SCRAPING_STATUS_FILE=/opt/thailand-water-monitor/collector-status.json REQUEST_TIMEOUT=30 MAX_RETRIES=3 RETRY_DELAY_SECONDS=60 diff --git a/.gitea/workflows/rust.yml b/.gitea/workflows/rust.yml new file mode 100644 index 0000000..e40390d --- /dev/null +++ b/.gitea/workflows/rust.yml @@ -0,0 +1,71 @@ +name: Rust + +# rust/ workspace (ping-collector): format, clippy, tests (including the +# database tests against a throwaway PostgreSQL service), release build. +# The Linux binary is uploaded as an artifact for scripts/install_collector.sh. + +on: + push: + branches: [master, develop] + paths: + - "rust/**" + - "src/data/stations.json" + - ".gitea/workflows/rust.yml" + pull_request: + branches: [master] + paths: + - "rust/**" + - "src/data/stations.json" + - ".gitea/workflows/rust.yml" + workflow_dispatch: + +jobs: + rust: + name: fmt, clippy, test, build + runs-on: ubuntu-latest + services: + postgres: + image: postgres:16 + env: + POSTGRES_PASSWORD: ping + POSTGRES_DB: ping + options: >- + --health-cmd pg_isready + --health-interval 5s + --health-timeout 5s + --health-retries 20 + env: + CARGO_TERM_COLOR: always + TEST_DATABASE_URL: postgres://postgres:ping@postgres:5432/ping + defaults: + run: + working-directory: rust + steps: + - uses: actions/checkout@v4 + + - name: System packages (OpenSSL headers for native-tls) + run: | + SUDO=$(command -v sudo || true) + $SUDO apt-get update -qq + $SUDO apt-get install -y -qq pkg-config libssl-dev + + - uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt, clippy + + - name: fmt + run: cargo fmt --all --check + + - name: clippy + run: cargo clippy --all-targets --locked -- -D warnings + + - name: test + run: cargo test --locked + + - name: release build + run: cargo build --release --locked + + - uses: actions/upload-artifact@v3 + with: + name: ping-collector-linux-x86_64 + path: rust/target/release/ping-collector diff --git a/CLAUDE.md b/CLAUDE.md index 98d45ea..567e7bb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -28,6 +28,11 @@ Production: one systemd unit on a small VPS, `/opt/thailand-water-monitor`, user - Do not add Co-Authored-By trailers. - The dashboard is a single file, `src/static/dashboard.html`, EN + TH via the `t()` table: every user-visible string needs both languages. +- Rust (`rust/`, see rust/README.md): `cargo fmt --all`, `cargo clippy --all-targets + -- -D warnings`, `cargo test` (database tests need `TEST_DATABASE_URL`). The Rust + collector writes the same tables as the Python collectors: a change to a table, + upsert or parser on one side must be made on the other until the Python side is + retired (check with a live run of both into two databases). ## Where things are @@ -36,5 +41,7 @@ Production: one systemd unit on a small VPS, `/opt/thailand-water-monitor`, user `rain.py` (Open-Meteo), `dam.py`, `hii_rain.py`. - `scripts/retrain.sh` + `water-monitor-retrain.timer`: monthly retrain with staged promote. `scripts/dev_proxy.py`: serve the working-copy dashboard against the live API. +- `rust/collector` (`ping-collector`): Rust data collector; with it deployed the + Python app runs `COLLECTION_ENABLED=false`. Rewrite plan: `docs/RUST_REWRITE.md`. - `docs/FLOOD_FORECASTING.md` is the authoritative model write-up; `docs/DATA_SOURCES.md` the source catalog. diff --git a/docs/RUST_REWRITE.md b/docs/RUST_REWRITE.md index db67b7b..819c31e 100644 --- a/docs/RUST_REWRITE.md +++ b/docs/RUST_REWRITE.md @@ -1,6 +1,7 @@ # A Rust rewrite: assessment -Status: evaluation, nothing decided. Spike: [spikes/rust-forecast](../spikes/rust-forecast/README.md). +Status: **in progress.** Phase 1 (collector) is built and verified, deploy pending; see +[rust/README.md](../rust/README.md). Spike: [spikes/rust-forecast](../spikes/rust-forecast/README.md). ## Short version @@ -76,23 +77,34 @@ This slows model iteration, which is the most active part of the repo. | One static binary; no venv on the server for the API | Server still needs Python + sklearn for the monthly retrain | | Same stack as proplan | Feature work = Python + Rust change every time | -## Plan, if pursued (each phase ships on its own) +## Plan (each phase ships on its own) + +| Phase | Scope | Status | +| --- | --- | --- | +| 0 | Spike: P.1 features + HGB inference | done, bit-identical | +| 1 | `ping-collector`: RID, HII, dams, Open-Meteo | built; live parity with Python on all 9 tables; deploy pending | +| 2 | Forecast crate + ntfy | next | +| 3 | Read API | | +| 4 | Cleanup | | 0. **Spike: done.** P.1 features + HGB inference, bit-identical. -1. **Collector + notifier daemon** (~1.5k LOC): RID (the fixed semantics: store - late readings, retry while behind, re-fetch yesterday's holes), HII, dams, - Open-Meteo, ntfy, `notification_state`. The Python API runs with collection - off and becomes a stateless reader. Shadow one week first: compare stored - hours against RID upstream. -2. **Forecast crate** (~2k LOC): all 15 stations, dam/HII/rain inputs, heuristic - fallback, probabilities and stages, `forecast_history`; parity CI for every - station on a flood window and a dry window. +1. **Collector daemon** (`rust/collector`, ~1.5k LOC incl. tests): RID with the + fixed semantics (store late readings, retry while behind, re-fetch + yesterday's holes), HII, dams, Open-Meteo, into the same tables. The Python + API runs with `COLLECTION_ENABLED=false` and re-forecasts when the newest + RID reading in the database advances. ntfy moved to phase 2: the outlook + alert needs the forecasts, which are still Python. Cutover and rollback + are one command each (`scripts/install_collector.sh`). +2. **Forecast crate + ntfy** (~2k LOC): all 15 stations, dam/HII/rain inputs, + heuristic fallback, probabilities and stages, `forecast_history`, + `notification_state`; parity CI for every station on a flood window and a + dry window. The Python API then only serves. 3. **Read API** (~2k LOC): routes recorded from the Python API as golden files, then a Caddy switch. Python keeps only training/evaluation. 4. **Cleanup:** drop the multi-backend adapters, demo databases and the Python web stack. -Open questions before phase 1: is Matrix alerting still used? Must `/docs` +Open questions before phase 3: is Matrix alerting still used? Must `/docs` (OpenAPI UI) stay? Is SQLite dev mode needed? Where does the monthly retrain run if the server stops having a Python environment for the API? diff --git a/rust/.gitignore b/rust/.gitignore new file mode 100644 index 0000000..2f7896d --- /dev/null +++ b/rust/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/rust/Cargo.lock b/rust/Cargo.lock new file mode 100644 index 0000000..0b20057 --- /dev/null +++ b/rust/Cargo.lock @@ -0,0 +1,2639 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" +dependencies = [ + "serde_core", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6651c9ed80effdc7db0ff72512157f901af5e3549e341e24b1dd4887d836d838" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "serde", + "windows-link", +] + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + +[[package]] +name = "crossbeam-queue" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03e8bd762f7479489c70ed6c768ddca99d7296857de437a68dcb2a94365b3fae" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", + "subtle", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" +dependencies = [ + "serde", +] + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] + +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" + +[[package]] +name = "flume" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" +dependencies = [ + "futures-core", + "futures-sink", + "spin", +] + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.5", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "hyper" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" +dependencies = [ + "atomic-waker", + "bytes", + "futures-core", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" +dependencies = [ + "base64 0.23.1", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "httparse", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" + +[[package]] +name = "icu_properties" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" + +[[package]] +name = "icu_provider" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "ipnet" +version = "2.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" +dependencies = [ + "spin", +] + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "libredox" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61ff90caf6077a803a240f62fdbe88645a890bbca49ef8174c3cb0404362171d" +dependencies = [ + "bitflags", + "libc", + "plain", + "redox_syscall 0.9.4", +] + +[[package]] +name = "libsqlite3-sys" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +dependencies = [ + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "native-tls" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "openssl" +version = "0.10.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "openssl-sys" +version = "0.9.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.18", + "smallvec", + "windows-link", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "ping-collector" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "dotenvy", + "reqwest", + "serde", + "serde_json", + "sqlx", + "tokio", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_syscall" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "737970939a87c6fa31e7acad13307bccbb017a073b695b6089a2c484f929e20e" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-tls", + "hyper-util", + "js-sys", + "log", + "native-tls", + "percent-encoding", + "pin-project-lite", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-native-tls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rsa" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core", + "signature", + "spki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.23.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "schannel" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cb23c424d8ee841bb4d813677ca237e2a610067d594fec130f87d6093495ab5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" +dependencies = [ + "serde", +] + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "sqlx" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fefb893899429669dcdd979aff487bd78f4064e5e7907e4269081e0ef7d97dc" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" +dependencies = [ + "base64 0.22.1", + "bytes", + "chrono", + "crc", + "crossbeam-queue", + "either", + "event-listener", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashbrown 0.15.5", + "hashlink", + "indexmap", + "log", + "memchr", + "once_cell", + "percent-encoding", + "rustls", + "serde", + "serde_json", + "sha2", + "smallvec", + "thiserror", + "tokio", + "tokio-stream", + "tracing", + "url", + "webpki-roots 0.26.11", +] + +[[package]] +name = "sqlx-macros" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2d452988ccaacfbf5e0bdbc348fb91d7c8af5bee192173ac3636b5fb6e6715d" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 2.0.119", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b" +dependencies = [ + "dotenvy", + "either", + "heck", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn 2.0.119", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags", + "byteorder", + "bytes", + "chrono", + "crc", + "digest", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand", + "rsa", + "serde", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-postgres" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags", + "byteorder", + "chrono", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "rand", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea" +dependencies = [ + "atoi", + "chrono", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "serde_urlencoded", + "sqlx-core", + "thiserror", + "tracing", + "url", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.52.0", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tinystr" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" + +[[package]] +name = "tokio" +version = "1.53.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce3335fa71841cda333a58d7615b03901380ecf09d59b3296d21f8bbac0dde4e" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "unicode-normalization" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75cd3fb03d5288b3ba9a1ee103ac7e9cbb59f0b03d64f8ddfcc04dd42758a85d" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "want" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cbab34de2d982e9b48e18d216d04c4a6f641066ff19ffb699980f591ee3610e" +dependencies = [ + "js-sys", + "tokio", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.9", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88261b9deccee56594c11a3460c462c41f58d148598fe70ad77070126a68aba4" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.9", +] + +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "whoami" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" +dependencies = [ + "libredox", + "wasite", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5fe1f8f1b06191a00962174c61aa5005e0bb391a6d80d07e24d115c01a92ed8" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "863ad3ac83293fb4d740aedbfdc9240dd8d1a50c1099acd76ce80ce7c7230c7f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/rust/Cargo.toml b/rust/Cargo.toml new file mode 100644 index 0000000..454ab99 --- /dev/null +++ b/rust/Cargo.toml @@ -0,0 +1,29 @@ +[workspace] +resolver = "3" +members = ["collector"] + +[workspace.package] +edition = "2024" +license = "MIT" +publish = false +repository = "https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor" + +[workspace.dependencies] +anyhow = "1" +chrono = { version = "0.4", default-features = false, features = ["std", "clock", "serde"] } +clap = { version = "4", features = ["derive"] } +dotenvy = "0.15" +# native-tls (OpenSSL on Linux, SChannel on Windows), not rustls: app.rid.go.th (dams) +# offers only TLS 1.2 CBC suites (ECDHE-RSA-AES256-SHA384), which rustls refuses. +reqwest = { version = "0.12", default-features = false, features = ["native-tls", "json"] } +serde = { version = "1", features = ["derive"] } +serde_json = { version = "1", features = ["float_roundtrip"] } +sqlx = { version = "0.8", default-features = false, features = ["runtime-tokio", "tls-rustls", "postgres", "chrono", "macros"] } +tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "signal"] } +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } + +[profile.release] +lto = "thin" +codegen-units = 1 +strip = true diff --git a/rust/README.md b/rust/README.md new file mode 100644 index 0000000..1610284 --- /dev/null +++ b/rust/README.md @@ -0,0 +1,85 @@ +# Rust services + +Rewrite of the Ping River Monitor in Rust, one service at a time. The plan, +costs and the parity rules are in [docs/RUST_REWRITE.md](../docs/RUST_REWRITE.md). + +| Crate | Replaces | Status | +| --- | --- | --- | +| `collector` (`ping-collector`) | RID, HII/ThaiWater, RID dams and Open-Meteo collection in the Python web app | phase 1: ready to deploy | + +## ping-collector + +Collects into the same PostgreSQL tables as the Python app, so the two can be +swapped (and swapped back) with no data migration. The Python web app keeps +serving the API, forecasting and sending ntfy alerts with +`COLLECTION_ENABLED=false`, and re-forecasts whenever the newest RID reading in +the database advances. + +```text +ping-collector run # the service: startup gap fill, RID at :05 (+10 min retries + # while RID is behind), HII/dams/rain once per clock hour +ping-collector once [--dry-run] # one cycle of every source; --dry-run writes nothing + # and compares the RID fetch with the database +ping-collector fill-gaps # re-fetch RID reports with holes +ping-collector check-db # connect, create missing tables, row counts +``` + +Configuration is the Python app's `.env` (read from the working directory, or +`--env-file`); real environment variables win. Used keys: +`POSTGRES_CONNECTION_STRING` (or `POSTGRES_HOST/PORT/DB/USER/PASSWORD`), +`RID_POLL_OFFSET_MINUTES` (5), `RID_RETRY_MINUTES` (10), +`GAP_FILL_DAYS_ON_START` (14), `ENABLE_HII_COLLECTION`, +`ENABLE_RESERVOIR_COLLECTION`, `ENABLE_RAIN_COLLECTION`, `HII_BASIN_CODE` (6), +`STATION_CONFIG_PATH` (falls back to the bundled `src/data/stations.json`), +`SCRAPING_STATUS_FILE` (status JSON served by the API's `/scraping/status`), +`LOG_LEVEL` / `RUST_LOG`. All timestamps are Asia/Bangkok wall-clock, +whatever the server's time zone. + +Not ported: the one-off archive backfills (`scripts/backfill_*.py`), which +stay in Python. + +### Verified + +- 20 tests: parsers, gap/schedule logic, and database tests against a real + PostgreSQL (late reports, midnight recovery, upsert revisions, NUMERIC + rounding, dam metadata preservation). +- Live parity: the Python collectors and `ping-collector` run back-to-back + against the real upstream APIs into two empty databases produced identical + rows in all 9 tables (16 stations, 1,104 RID readings, 103 + 76 HII rows, + 35 dams x 2 days, 168 rain hours). Rerun with + `scripts/collector_parity.py` after changing a parser, table or upsert on + either side. +- The Python API with `COLLECTION_ENABLED=false` served `/health`, + `/measurements/latest`, `/api/stats`, `/scraping/status` and model + forecasts from a database filled only by `ping-collector`. + +### Develop + +```bash +cd rust +cargo fmt --all && cargo clippy --all-targets -- -D warnings +docker run -d --name ping-pg -e POSTGRES_PASSWORD=ping -e POSTGRES_DB=ping -p 55432:5432 postgres:16 +TEST_DATABASE_URL=postgres://postgres:ping@localhost:55432/ping cargo test +``` + +Without `TEST_DATABASE_URL` the database tests skip. CI +(`.gitea/workflows/rust.yml`) runs them against a PostgreSQL service and +uploads the Linux binary as the `ping-collector-linux-x86_64` artifact. + +HTTP uses native-tls (OpenSSL on Linux), not rustls: app.rid.go.th (dams) +only offers TLS 1.2 CBC cipher suites, which rustls refuses. Building on +Linux needs `pkg-config` and `libssl-dev`. + +### Deploy + +On the server, after `git pull` (the Python side of the switch ships in the +same commit): + +```bash +cargo build --release --locked --manifest-path rust/Cargo.toml # as a normal user, + # or BIN= +sudo bash scripts/install_collector.sh # install + dry run, nothing switched +sudo bash scripts/install_collector.sh --cutover # Rust collects from now on +sudo bash scripts/install_collector.sh --rollback # back to Python collection +journalctl -u ping-collector -f +``` diff --git a/rust/collector/Cargo.toml b/rust/collector/Cargo.toml new file mode 100644 index 0000000..d27749e --- /dev/null +++ b/rust/collector/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "ping-collector" +version = "0.1.0" +description = "Ping River Monitor data collector: RID gauges, HII/ThaiWater, RID dams, Open-Meteo rain -> PostgreSQL" +edition.workspace = true +license.workspace = true +publish.workspace = true +repository.workspace = true + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +clap.workspace = true +dotenvy.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlx.workspace = true +tokio.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true diff --git a/rust/collector/src/config.rs b/rust/collector/src/config.rs new file mode 100644 index 0000000..5841705 --- /dev/null +++ b/rust/collector/src/config.rs @@ -0,0 +1,175 @@ +//! Configuration from the same environment / `.env` file as the Python app, +//! so both services can share `/opt/thailand-water-monitor/.env`. + +use anyhow::{Context, Result, bail}; +use std::env; +use std::path::PathBuf; + +pub const RID_API_URL: &str = + "https://hyd-app-db.rid.go.th/webservice/getGroupHourlyWaterLevelReportAllHL.ashx"; +pub const HII_API_BASE: &str = "https://api-v3.thaiwater.net/api/v1/thaiwater30/public"; +pub const RID_DAMS_URL: &str = "https://app.rid.go.th/reservoir/api/dams"; +pub const OPENMETEO_URL: &str = "https://api.open-meteo.com/v1/forecast"; +/// The RID endpoint serves the same JSON to any client; keep the UA the +/// Python collector has used for years. +pub const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 \ + (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"; + +#[derive(Clone, Debug)] +pub struct Config { + pub database_url: String, + pub rid_api_url: String, + pub hii_api_base: String, + pub dams_url: String, + pub openmeteo_url: String, + pub hii_enabled: bool, + pub dams_enabled: bool, + pub rain_enabled: bool, + pub hii_basin_code: i64, + /// RID is polled this many minutes past each hour. + pub poll_offset_minutes: u32, + /// While the current hour is not published yet, retry this often. + pub retry_minutes: u32, + /// Days of RID holes re-fetched at startup (0 = off). + pub gap_fill_days_on_start: u32, + pub station_config_path: PathBuf, + /// Status JSON read by the Python API's /scraping/status. + pub status_file: Option, +} + +fn var(name: &str) -> Option { + env::var(name) + .ok() + .map(|v| v.trim().to_string()) + .filter(|v| !v.is_empty()) +} + +fn flag(name: &str, default: bool) -> bool { + var(name).map_or(default, |v| { + matches!(v.to_lowercase().as_str(), "1" | "true" | "yes") + }) +} + +fn number(name: &str, default: T) -> Result +where + T::Err: std::fmt::Display, +{ + match var(name) { + None => Ok(default), + Some(v) => v.parse().map_err(|e| anyhow::anyhow!("{name}={v:?}: {e}")), + } +} + +/// Percent-encode everything outside RFC 3986 "unreserved" (as Python's +/// `urllib.parse.quote(password, safe="")`). +fn encode_userinfo(s: &str) -> String { + s.bytes() + .map(|b| match b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'.' | b'_' | b'~' => { + (b as char).to_string() + } + _ => format!("%{b:02X}"), + }) + .collect() +} + +/// SQLAlchemy URLs carry a driver suffix (`postgresql+psycopg2://`) that +/// sqlx does not understand. +pub fn normalize_database_url(url: &str) -> String { + match url.split_once("://") { + Some((scheme, rest)) + if scheme.starts_with("postgresql") || scheme.starts_with("postgres") => + { + format!("postgres://{rest}") + } + _ => url.to_string(), + } +} + +impl Config { + pub fn from_env() -> Result { + let database_url = match var("POSTGRES_CONNECTION_STRING") { + Some(url) => normalize_database_url(&url), + None => { + let password = var("POSTGRES_PASSWORD") + .context("POSTGRES_CONNECTION_STRING or POSTGRES_PASSWORD must be set")?; + format!( + "postgres://{}:{}@{}:{}/{}", + encode_userinfo(&var("POSTGRES_USER").unwrap_or_else(|| "postgres".into())), + encode_userinfo(&password), + var("POSTGRES_HOST").unwrap_or_else(|| "localhost".into()), + var("POSTGRES_PORT").unwrap_or_else(|| "5432".into()), + var("POSTGRES_DB").unwrap_or_else(|| "water_monitoring".into()), + ) + } + }; + if let Some(db_type) = var("DB_TYPE") + && db_type != "postgresql" + { + bail!("ping-collector supports PostgreSQL only (DB_TYPE={db_type})"); + } + let cfg = Config { + database_url, + rid_api_url: var("RID_API_URL").unwrap_or_else(|| RID_API_URL.into()), + hii_api_base: var("HII_API_BASE").unwrap_or_else(|| HII_API_BASE.into()), + dams_url: var("RID_DAMS_URL").unwrap_or_else(|| RID_DAMS_URL.into()), + openmeteo_url: var("OPENMETEO_URL").unwrap_or_else(|| OPENMETEO_URL.into()), + hii_enabled: flag("ENABLE_HII_COLLECTION", true), + dams_enabled: flag("ENABLE_RESERVOIR_COLLECTION", true), + rain_enabled: flag("ENABLE_RAIN_COLLECTION", true), + hii_basin_code: number("HII_BASIN_CODE", 6)?, + poll_offset_minutes: number("RID_POLL_OFFSET_MINUTES", 5)?, + retry_minutes: number("RID_RETRY_MINUTES", 10)?, + gap_fill_days_on_start: number("GAP_FILL_DAYS_ON_START", 14)?, + station_config_path: var("STATION_CONFIG_PATH") + .unwrap_or_else(|| "stations.json".into()) + .into(), + status_file: var("SCRAPING_STATUS_FILE").map(PathBuf::from), + }; + if cfg.poll_offset_minutes >= 60 { + bail!("RID_POLL_OFFSET_MINUTES must be between 0 and 59"); + } + if cfg.retry_minutes == 0 { + bail!("RID_RETRY_MINUTES must be positive"); + } + Ok(cfg) + } + + /// The URL with the password masked, for logs. + pub fn redacted_database_url(&self) -> String { + match (self.database_url.find("://"), self.database_url.rfind('@')) { + (Some(s), Some(at)) if at > s => { + let creds = &self.database_url[s + 3..at]; + let user = creds.split(':').next().unwrap_or(""); + format!( + "{}{user}:***{}", + &self.database_url[..s + 3], + &self.database_url[at..] + ) + } + _ => self.database_url.clone(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sqlalchemy_urls_are_normalized() { + assert_eq!( + normalize_database_url("postgresql+psycopg2://u:p@h:5432/db"), + "postgres://u:p@h:5432/db" + ); + assert_eq!( + normalize_database_url("postgresql://u@h/db"), + "postgres://u@h/db" + ); + } + + #[test] + fn passwords_are_percent_encoded() { + assert_eq!(encode_userinfo("p@ss:w/rd"), "p%40ss%3Aw%2Frd"); + } +} diff --git a/rust/collector/src/dams.rs b/rust/collector/src/dams.rs new file mode 100644 index 0000000..408a278 --- /dev/null +++ b/rust/collector/src/dams.rs @@ -0,0 +1,143 @@ +//! RID large-dam daily status (app.rid.go.th/reservoir): storage, inflow, +//! outflow for every large dam. Port of the hourly path of +//! src/rid_reservoir.py (today + yesterday's late revisions). The one-off +//! archive backfills stay in Python (scripts/backfill_rid_reservoir.py). + +use crate::util::{lenient_f64, scalar_string}; +use anyhow::{Context, Result}; +use chrono::NaiveDate; +use serde_json::Value; + +#[derive(Clone, Debug, PartialEq)] +pub struct DamRecord { + pub dam_id: String, + pub region: Option, + pub name_th: Option, + pub latitude: Option, + pub longitude: Option, + pub capacity_max_mcm: Option, + pub capacity_normal_mcm: Option, + pub date: NaiveDate, + pub storage_mcm: Option, + pub storage_pct: Option, + pub inflow_mcm: Option, + pub outflow_mcm: Option, + pub level_msl: Option, +} + +/// Values at or beyond the column's NUMERIC capacity become NULL instead of +/// failing the whole daily batch (the source publishes e.g. 87798 %). +fn bounded(v: Option, limit: f64) -> Option { + v.filter(|x| x.abs() < limit) +} + +fn string(v: Option<&Value>) -> Option { + v.and_then(Value::as_str) + .filter(|s| !s.is_empty()) + .map(str::to_string) +} + +pub fn parse_dams(payload: &Value) -> Vec { + let mut out = Vec::new(); + for region in payload + .get("regions") + .and_then(Value::as_array) + .into_iter() + .flatten() + { + for dam in region + .get("dams") + .and_then(Value::as_array) + .into_iter() + .flatten() + { + let Some(date) = dam + .get("DMD_Date") + .and_then(Value::as_str) + .and_then(|s| NaiveDate::parse_from_str(s, "%Y-%m-%d").ok()) + else { + continue; + }; + let Some(dam_id) = scalar_string(dam.get("DAM_ID")) else { + continue; + }; + out.push(DamRecord { + dam_id, + region: string(region.get("region_name")), + name_th: string(dam.get("DAM_Name")), + latitude: lenient_f64(dam.get("DAM_Lat")), + longitude: lenient_f64(dam.get("DAM_Lon")), + capacity_max_mcm: lenient_f64(dam.get("DAM_QMax")), + capacity_normal_mcm: lenient_f64(dam.get("DAM_QStore")), + date, + storage_mcm: bounded(lenient_f64(dam.get("DMD_QUse")), 1e8), + storage_pct: bounded(lenient_f64(dam.get("PERCENT_DMD_QUse")), 1e6), + inflow_mcm: bounded(lenient_f64(dam.get("DMD_Inflow")), 1e8), + outflow_mcm: bounded(lenient_f64(dam.get("DMD_Outflow")), 1e8), + level_msl: bounded(lenient_f64(dam.get("DMD_Q")), 1e6), + }); + } + } + out +} + +pub struct DamsClient { + http: reqwest::Client, + url: String, +} + +impl DamsClient { + pub fn new(http: reqwest::Client, url: String) -> Self { + Self { http, url } + } + + /// Every large dam on `date` (None = today). POST only; the body may be + /// empty but must be sent. + pub async fn fetch_day(&self, date: Option) -> Result> { + let date = date + .map(|d| d.format("%Y-%m-%d").to_string()) + .unwrap_or_default(); + let payload: Value = self + .http + .post(&self.url) + .header("X-Requested-With", "XMLHttpRequest") + .form(&[("date", date.as_str())]) + .timeout(std::time::Duration::from_secs(120)) + .send() + .await + .context("RID dams request failed")? + .error_for_status()? + .json() + .await + .context("RID dams response is not JSON")?; + Ok(parse_dams(&payload)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn parses_regions_and_cleans_numbers() { + let payload = json!({"regions": [{"region_name": "ภาคเหนือ", "dams": [ + {"DAM_ID": "200103", "DAM_Name": "แม่งัด", "DMD_Date": "2026-10-17", "DAM_Lat": "19.17", + "DAM_QMax": "265.00", "DMD_QUse": "1,234.5", "PERCENT_DMD_QUse": "87798000", + "DMD_Inflow": " - ", "DMD_Outflow": 1.2, "DMD_Q": " - "}, + {"DAM_ID": "", "DMD_Date": "2026-10-17"}, + {"DAM_ID": 100602, "DMD_Date": "not a date"} + ]}]}); + let r = parse_dams(&payload); + assert_eq!(r.len(), 1); + let d = &r[0]; + assert_eq!(d.dam_id, "200103"); + assert_eq!(d.region.as_deref(), Some("ภาคเหนือ")); + assert_eq!(d.storage_mcm, Some(1234.5)); + assert_eq!(d.storage_pct, None); // junk beyond NUMERIC(8,2) + assert_eq!(d.inflow_mcm, None); + assert_eq!(d.outflow_mcm, Some(1.2)); + assert_eq!(d.level_msl, None); + assert_eq!(d.latitude, Some(19.17)); + } +} diff --git a/rust/collector/src/db.rs b/rust/collector/src/db.rs new file mode 100644 index 0000000..4974ad6 --- /dev/null +++ b/rust/collector/src/db.rs @@ -0,0 +1,476 @@ +//! PostgreSQL access: the same tables and upsert semantics as the Python +//! collectors (src/database_adapters.py, hii_collector.py, rid_reservoir.py, +//! ml/rain.py), written as one set-based statement per table (UNNEST) instead +//! of a statement per row. + +use crate::dams::DamRecord; +use crate::hii::{RainRecord, WaterLevelRecord}; +use crate::rain::RainRow; +use crate::rid::Reading; +use crate::stations::Stations; +use anyhow::{Context, Result}; +use chrono::{NaiveDate, NaiveDateTime, NaiveTime}; +use sqlx::PgPool; +use sqlx::postgres::PgPoolOptions; +use std::collections::{BTreeMap, HashMap, HashSet}; + +pub async fn connect(url: &str) -> Result { + PgPoolOptions::new() + .max_connections(4) + .acquire_timeout(std::time::Duration::from_secs(30)) + .connect(url) + .await + .context("connecting to PostgreSQL") +} + +/// Idempotent DDL, identical to what the Python app creates (so either side +/// can start first against an empty database). +const SCHEMA: &[&str] = &[ + "CREATE TABLE IF NOT EXISTS stations ( + id SERIAL PRIMARY KEY, + station_code VARCHAR(10) UNIQUE NOT NULL, + thai_name VARCHAR(255) NOT NULL, + english_name VARCHAR(255) NOT NULL, + latitude DECIMAL(10,8), + longitude DECIMAL(11,8), + geohash VARCHAR(20), + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP)", + "CREATE TABLE IF NOT EXISTS water_measurements ( + id BIGSERIAL PRIMARY KEY, + timestamp TIMESTAMP NOT NULL, + station_id INTEGER NOT NULL, + water_level NUMERIC(10,3), + discharge NUMERIC(10,2), + discharge_percent NUMERIC(5,2), + status VARCHAR(20) DEFAULT 'active', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (station_id) REFERENCES stations(id), + UNIQUE(timestamp, station_id))", + "CREATE INDEX IF NOT EXISTS idx_timestamp ON water_measurements(timestamp)", + "CREATE INDEX IF NOT EXISTS idx_station_timestamp ON water_measurements(station_id, timestamp DESC)", + "CREATE TABLE IF NOT EXISTS hii_rain_stations ( + id INTEGER PRIMARY KEY, + oldcode VARCHAR(60), + name_th VARCHAR(255), + name_en VARCHAR(255), + latitude NUMERIC(10,6), + longitude NUMERIC(10,6), + sub_basin_id VARCHAR(10), + agency VARCHAR(40), + updated_at TIMESTAMP)", + "CREATE TABLE IF NOT EXISTS hii_rainfall ( + station_id INTEGER NOT NULL, + timestamp TIMESTAMP NOT NULL, + rain_1h NUMERIC(7,2), + rain_24h NUMERIC(8,2), + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (station_id, timestamp))", + "CREATE TABLE IF NOT EXISTS hii_wl_stations ( + id INTEGER PRIMARY KEY, + oldcode VARCHAR(60), + rid_code VARCHAR(10), + name_th VARCHAR(255), + name_en VARCHAR(255), + latitude NUMERIC(10,6), + longitude NUMERIC(10,6), + agency VARCHAR(40), + river_name VARCHAR(255), + offset_msl NUMERIC(8,3), + ground_level_msl NUMERIC(8,3), + min_bank_msl NUMERIC(8,3), + critical_level_msl NUMERIC(8,3), + critical_level_m NUMERIC(8,3), + qmax NUMERIC(10,2), + is_key_station BOOLEAN, + updated_at TIMESTAMP)", + "CREATE TABLE IF NOT EXISTS hii_waterlevel ( + station_id INTEGER NOT NULL, + timestamp TIMESTAMP NOT NULL, + wl_msl NUMERIC(8,3), + wl_m NUMERIC(8,3), + discharge NUMERIC(10,2), + flow_rate NUMERIC(10,2), + storage_percent NUMERIC(6,2), + situation_level INTEGER, + diff_wl_bank NUMERIC(8,3), + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (station_id, timestamp))", + "CREATE INDEX IF NOT EXISTS idx_hii_rainfall_ts ON hii_rainfall(timestamp)", + "CREATE INDEX IF NOT EXISTS idx_hii_waterlevel_ts ON hii_waterlevel(timestamp)", + "CREATE TABLE IF NOT EXISTS rid_dams ( + dam_id VARCHAR(10) PRIMARY KEY, + region VARCHAR(40), + name_th VARCHAR(255), + latitude NUMERIC(10,6), + longitude NUMERIC(10,6), + capacity_max_mcm NUMERIC(10,2), + capacity_normal_mcm NUMERIC(10,2), + updated_at TIMESTAMP)", + "CREATE TABLE IF NOT EXISTS rid_reservoir_daily ( + dam_id VARCHAR(10) NOT NULL, + date DATE NOT NULL, + storage_mcm NUMERIC(10,2), + storage_pct NUMERIC(8,2), + inflow_mcm NUMERIC(10,2), + outflow_mcm NUMERIC(10,2), + level_msl NUMERIC(8,2), + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (dam_id, date))", + "CREATE INDEX IF NOT EXISTS idx_rid_reservoir_date ON rid_reservoir_daily(date)", + "CREATE TABLE IF NOT EXISTS openmeteo_rain ( + timestamp TIMESTAMP PRIMARY KEY, + chiang_dao NUMERIC(6,2), mae_taeng NUMERIC(6,2), mae_ngat NUMERIC(6,2), + mae_rim NUMERIC(6,2), chiang_mai NUMERIC(6,2), + catchment_mean NUMERIC(6,2), + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP)", +]; + +pub async fn ensure_schema(pool: &PgPool) -> Result<()> { + for ddl in SCHEMA { + sqlx::query(ddl) + .execute(pool) + .await + .with_context(|| format!("DDL failed: {ddl}"))?; + } + Ok(()) +} + +/// Keep the last record per key (a later report revision wins), preserving +/// first-seen order; Postgres refuses to upsert the same key twice in one +/// statement. +fn dedupe_last(items: &[T], key: impl Fn(&T) -> K) -> Vec<&T> { + let mut index: HashMap = HashMap::new(); + let mut out: Vec<&T> = Vec::new(); + for item in items { + let k = key(item); + match index.get(&k) { + Some(&i) => out[i] = item, + None => { + index.insert(k, out.len()); + out.push(item); + } + } + } + out +} + +pub async fn save_rid(pool: &PgPool, readings: &[Reading], stations: &Stations) -> Result { + let rows = dedupe_last(readings, |r| (r.timestamp, r.station_num)); + let station_nums: Vec = rows + .iter() + .map(|r| r.station_num) + .collect::>() + .into_iter() + .collect(); + let infos: Vec<_> = station_nums.iter().map(|&n| stations.get(n)).collect(); + + let mut tx = pool.begin().await?; + sqlx::query( + "INSERT INTO stations (id, station_code, thai_name, english_name, latitude, longitude, geohash, updated_at) + SELECT *, NOW() FROM UNNEST($1::int4[], $2::text[], $3::text[], $4::text[], $5::float8[], $6::float8[], $7::text[]) + ON CONFLICT (id) DO UPDATE SET + thai_name = EXCLUDED.thai_name, english_name = EXCLUDED.english_name, + latitude = EXCLUDED.latitude, longitude = EXCLUDED.longitude, + geohash = EXCLUDED.geohash, updated_at = NOW()", + ) + .bind(station_nums.iter().map(|&n| n as i32).collect::>()) + .bind(infos.iter().map(|s| s.code.clone()).collect::>()) + .bind(infos.iter().map(|s| s.thai_name.clone()).collect::>()) + .bind(infos.iter().map(|s| s.english_name.clone()).collect::>()) + .bind(infos.iter().map(|s| s.latitude).collect::>()) + .bind(infos.iter().map(|s| s.longitude).collect::>()) + .bind(infos.iter().map(|s| s.geohash.clone()).collect::>()) + .execute(&mut *tx) + .await + .context("upserting stations")?; + + sqlx::query( + "INSERT INTO water_measurements (timestamp, station_id, water_level, discharge, discharge_percent, status) + SELECT *, 'active' FROM UNNEST($1::timestamp[], $2::int4[], $3::float8[], $4::float8[], $5::float8[]) + ON CONFLICT (timestamp, station_id) DO UPDATE SET + water_level = EXCLUDED.water_level, discharge = EXCLUDED.discharge, + discharge_percent = EXCLUDED.discharge_percent, status = EXCLUDED.status", + ) + .bind(rows.iter().map(|r| r.timestamp).collect::>()) + .bind(rows.iter().map(|r| r.station_num as i32).collect::>()) + .bind(rows.iter().map(|r| r.water_level).collect::>()) + .bind(rows.iter().map(|r| r.discharge).collect::>()) + .bind(rows.iter().map(|r| r.discharge_percent).collect::>()) + .execute(&mut *tx) + .await + .context("upserting water_measurements")?; + tx.commit().await?; + Ok(rows.len()) +} + +/// Day -> hours (0-23) that have at least one RID reading, for [start, end]. +pub async fn recorded_hours_by_day( + pool: &PgPool, + start: NaiveDate, + end: NaiveDate, +) -> Result>> { + let rows: Vec<(NaiveDate, i32)> = sqlx::query_as( + "SELECT CAST(timestamp AS DATE) AS day, CAST(EXTRACT(HOUR FROM timestamp) AS INTEGER) AS hour + FROM water_measurements WHERE timestamp >= $1 AND timestamp < $2 + GROUP BY 1, 2", + ) + .bind(start.and_time(NaiveTime::MIN)) + .bind((end + chrono::Duration::days(1)).and_time(NaiveTime::MIN)) + .fetch_all(pool) + .await?; + let mut out: HashMap> = HashMap::new(); + for (day, hour) in rows { + out.entry(day).or_default().insert(hour as u32); + } + Ok(out) +} + +pub async fn earliest_rid_timestamp(pool: &PgPool) -> Result> { + Ok( + sqlx::query_scalar("SELECT MIN(timestamp) FROM water_measurements") + .fetch_one(pool) + .await?, + ) +} + +pub async fn latest_rid_timestamp(pool: &PgPool) -> Result> { + Ok( + sqlx::query_scalar("SELECT MAX(timestamp) FROM water_measurements") + .fetch_one(pool) + .await?, + ) +} + +/// How many of `readings` are already stored (dry-run comparison). +pub async fn count_stored_rid(pool: &PgPool, readings: &[Reading]) -> Result { + Ok(sqlx::query_scalar( + "SELECT COUNT(*) FROM water_measurements w + JOIN UNNEST($1::timestamp[], $2::int4[]) AS k(ts, sid) ON w.timestamp = k.ts AND w.station_id = k.sid", + ) + .bind(readings.iter().map(|r| r.timestamp).collect::>()) + .bind(readings.iter().map(|r| r.station_num as i32).collect::>()) + .fetch_one(pool) + .await?) +} + +pub async fn save_hii_rain( + pool: &PgPool, + records: &[RainRecord], + now: NaiveDateTime, +) -> Result { + if records.is_empty() { + return Ok(0); + } + let st = dedupe_last(records, |r| r.station_id); + let ms = dedupe_last(records, |r| (r.station_id, r.timestamp)); + let mut tx = pool.begin().await?; + sqlx::query( + "INSERT INTO hii_rain_stations (id, oldcode, name_th, name_en, latitude, longitude, sub_basin_id, agency, updated_at) + SELECT * FROM UNNEST($1::int4[], $2::text[], $3::text[], $4::text[], $5::float8[], $6::float8[], $7::text[], $8::text[], $9::timestamp[]) + ON CONFLICT (id) DO UPDATE SET + oldcode = EXCLUDED.oldcode, name_th = EXCLUDED.name_th, name_en = EXCLUDED.name_en, + latitude = EXCLUDED.latitude, longitude = EXCLUDED.longitude, + sub_basin_id = EXCLUDED.sub_basin_id, agency = EXCLUDED.agency, updated_at = EXCLUDED.updated_at", + ) + .bind(st.iter().map(|r| r.station_id as i32).collect::>()) + .bind(st.iter().map(|r| r.oldcode.clone()).collect::>()) + .bind(st.iter().map(|r| r.name_th.clone()).collect::>()) + .bind(st.iter().map(|r| r.name_en.clone()).collect::>()) + .bind(st.iter().map(|r| r.latitude).collect::>()) + .bind(st.iter().map(|r| r.longitude).collect::>()) + .bind(st.iter().map(|r| r.sub_basin_id.clone()).collect::>()) + .bind(st.iter().map(|r| r.agency.clone()).collect::>()) + .bind(vec![now; st.len()]) + .execute(&mut *tx) + .await + .context("upserting hii_rain_stations")?; + sqlx::query( + "INSERT INTO hii_rainfall (station_id, timestamp, rain_1h, rain_24h) + SELECT * FROM UNNEST($1::int4[], $2::timestamp[], $3::float8[], $4::float8[]) + ON CONFLICT (station_id, timestamp) DO UPDATE SET + rain_1h = EXCLUDED.rain_1h, rain_24h = EXCLUDED.rain_24h", + ) + .bind(ms.iter().map(|r| r.station_id as i32).collect::>()) + .bind(ms.iter().map(|r| r.timestamp).collect::>()) + .bind(ms.iter().map(|r| r.rain_1h).collect::>()) + .bind(ms.iter().map(|r| r.rain_24h).collect::>()) + .execute(&mut *tx) + .await + .context("upserting hii_rainfall")?; + tx.commit().await?; + Ok(ms.len()) +} + +pub async fn save_hii_waterlevel( + pool: &PgPool, + records: &[WaterLevelRecord], + now: NaiveDateTime, +) -> Result { + if records.is_empty() { + return Ok(0); + } + let st = dedupe_last(records, |r| r.station_id); + let ms = dedupe_last(records, |r| (r.station_id, r.timestamp)); + let mut tx = pool.begin().await?; + sqlx::query( + "INSERT INTO hii_wl_stations (id, oldcode, rid_code, name_th, name_en, latitude, longitude, agency, + river_name, offset_msl, ground_level_msl, min_bank_msl, critical_level_msl, critical_level_m, + qmax, is_key_station, updated_at) + SELECT * FROM UNNEST($1::int4[], $2::text[], $3::text[], $4::text[], $5::text[], $6::float8[], + $7::float8[], $8::text[], $9::text[], $10::float8[], $11::float8[], $12::float8[], $13::float8[], + $14::float8[], $15::float8[], $16::bool[], $17::timestamp[]) + ON CONFLICT (id) DO UPDATE SET + oldcode = EXCLUDED.oldcode, rid_code = EXCLUDED.rid_code, name_th = EXCLUDED.name_th, + name_en = EXCLUDED.name_en, latitude = EXCLUDED.latitude, longitude = EXCLUDED.longitude, + agency = EXCLUDED.agency, river_name = EXCLUDED.river_name, offset_msl = EXCLUDED.offset_msl, + ground_level_msl = EXCLUDED.ground_level_msl, min_bank_msl = EXCLUDED.min_bank_msl, + critical_level_msl = EXCLUDED.critical_level_msl, critical_level_m = EXCLUDED.critical_level_m, + qmax = EXCLUDED.qmax, is_key_station = EXCLUDED.is_key_station, updated_at = EXCLUDED.updated_at", + ) + .bind(st.iter().map(|r| r.station_id as i32).collect::>()) + .bind(st.iter().map(|r| r.oldcode.clone()).collect::>()) + .bind(st.iter().map(|r| r.rid_code.clone()).collect::>()) + .bind(st.iter().map(|r| r.name_th.clone()).collect::>()) + .bind(st.iter().map(|r| r.name_en.clone()).collect::>()) + .bind(st.iter().map(|r| r.latitude).collect::>()) + .bind(st.iter().map(|r| r.longitude).collect::>()) + .bind(st.iter().map(|r| r.agency.clone()).collect::>()) + .bind(st.iter().map(|r| r.river_name.clone()).collect::>()) + .bind(st.iter().map(|r| r.offset_msl).collect::>()) + .bind(st.iter().map(|r| r.ground_level_msl).collect::>()) + .bind(st.iter().map(|r| r.min_bank_msl).collect::>()) + .bind(st.iter().map(|r| r.critical_level_msl).collect::>()) + .bind(st.iter().map(|r| r.critical_level_m).collect::>()) + .bind(st.iter().map(|r| r.qmax).collect::>()) + .bind(st.iter().map(|r| r.is_key_station).collect::>()) + .bind(vec![now; st.len()]) + .execute(&mut *tx) + .await + .context("upserting hii_wl_stations")?; + sqlx::query( + "INSERT INTO hii_waterlevel (station_id, timestamp, wl_msl, wl_m, discharge, flow_rate, + storage_percent, situation_level, diff_wl_bank) + SELECT * FROM UNNEST($1::int4[], $2::timestamp[], $3::float8[], $4::float8[], $5::float8[], + $6::float8[], $7::float8[], $8::int4[], $9::float8[]) + ON CONFLICT (station_id, timestamp) DO UPDATE SET + wl_msl = EXCLUDED.wl_msl, wl_m = EXCLUDED.wl_m, discharge = EXCLUDED.discharge, + flow_rate = EXCLUDED.flow_rate, storage_percent = EXCLUDED.storage_percent, + situation_level = EXCLUDED.situation_level, diff_wl_bank = EXCLUDED.diff_wl_bank", + ) + .bind(ms.iter().map(|r| r.station_id as i32).collect::>()) + .bind(ms.iter().map(|r| r.timestamp).collect::>()) + .bind(ms.iter().map(|r| r.wl_msl).collect::>()) + .bind(ms.iter().map(|r| r.wl_m).collect::>()) + .bind(ms.iter().map(|r| r.discharge).collect::>()) + .bind(ms.iter().map(|r| r.flow_rate).collect::>()) + .bind(ms.iter().map(|r| r.storage_percent).collect::>()) + .bind(ms.iter().map(|r| r.situation_level).collect::>()) + .bind(ms.iter().map(|r| r.diff_wl_bank).collect::>()) + .execute(&mut *tx) + .await + .context("upserting hii_waterlevel")?; + tx.commit().await?; + Ok(ms.len()) +} + +/// rid_dams metadata never blanks a stored value (COALESCE), and the daily +/// `level_msl` is preserved too: only the range backfill carries a level, the +/// daily snapshot would NULL it on every cycle. +pub async fn save_dams(pool: &PgPool, records: &[DamRecord], now: NaiveDateTime) -> Result { + if records.is_empty() { + return Ok(0); + } + let dams = dedupe_last(records, |r| r.dam_id.clone()); + let ms = dedupe_last(records, |r| (r.dam_id.clone(), r.date)); + let mut tx = pool.begin().await?; + sqlx::query( + "INSERT INTO rid_dams (dam_id, region, name_th, latitude, longitude, capacity_max_mcm, + capacity_normal_mcm, updated_at) + SELECT * FROM UNNEST($1::text[], $2::text[], $3::text[], $4::float8[], $5::float8[], + $6::float8[], $7::float8[], $8::timestamp[]) + ON CONFLICT (dam_id) DO UPDATE SET + region = COALESCE(EXCLUDED.region, rid_dams.region), + name_th = COALESCE(EXCLUDED.name_th, rid_dams.name_th), + latitude = COALESCE(EXCLUDED.latitude, rid_dams.latitude), + longitude = COALESCE(EXCLUDED.longitude, rid_dams.longitude), + capacity_max_mcm = COALESCE(EXCLUDED.capacity_max_mcm, rid_dams.capacity_max_mcm), + capacity_normal_mcm = COALESCE(EXCLUDED.capacity_normal_mcm, rid_dams.capacity_normal_mcm), + updated_at = EXCLUDED.updated_at", + ) + .bind(dams.iter().map(|r| r.dam_id.clone()).collect::>()) + .bind(dams.iter().map(|r| r.region.clone()).collect::>()) + .bind(dams.iter().map(|r| r.name_th.clone()).collect::>()) + .bind(dams.iter().map(|r| r.latitude).collect::>()) + .bind(dams.iter().map(|r| r.longitude).collect::>()) + .bind(dams.iter().map(|r| r.capacity_max_mcm).collect::>()) + .bind(dams.iter().map(|r| r.capacity_normal_mcm).collect::>()) + .bind(vec![now; dams.len()]) + .execute(&mut *tx) + .await + .context("upserting rid_dams")?; + sqlx::query( + "INSERT INTO rid_reservoir_daily (dam_id, date, storage_mcm, storage_pct, inflow_mcm, outflow_mcm, level_msl) + SELECT * FROM UNNEST($1::text[], $2::date[], $3::float8[], $4::float8[], $5::float8[], $6::float8[], $7::float8[]) + ON CONFLICT (dam_id, date) DO UPDATE SET + storage_mcm = EXCLUDED.storage_mcm, storage_pct = EXCLUDED.storage_pct, + inflow_mcm = EXCLUDED.inflow_mcm, outflow_mcm = EXCLUDED.outflow_mcm, + level_msl = COALESCE(EXCLUDED.level_msl, rid_reservoir_daily.level_msl)", + ) + .bind(ms.iter().map(|r| r.dam_id.clone()).collect::>()) + .bind(ms.iter().map(|r| r.date).collect::>()) + .bind(ms.iter().map(|r| r.storage_mcm).collect::>()) + .bind(ms.iter().map(|r| r.storage_pct).collect::>()) + .bind(ms.iter().map(|r| r.inflow_mcm).collect::>()) + .bind(ms.iter().map(|r| r.outflow_mcm).collect::>()) + .bind(ms.iter().map(|r| r.level_msl).collect::>()) + .execute(&mut *tx) + .await + .context("upserting rid_reservoir_daily")?; + tx.commit().await?; + Ok(ms.len()) +} + +pub async fn save_rain(pool: &PgPool, rows: &[RainRow]) -> Result { + if rows.is_empty() { + return Ok(0); + } + let rows = dedupe_last(rows, |r| r.timestamp); + let col = |i: usize| rows.iter().map(|r| r.points[i]).collect::>(); + sqlx::query( + "INSERT INTO openmeteo_rain (timestamp, chiang_dao, mae_taeng, mae_ngat, mae_rim, chiang_mai, catchment_mean) + SELECT * FROM UNNEST($1::timestamp[], $2::float8[], $3::float8[], $4::float8[], $5::float8[], $6::float8[], $7::float8[]) + ON CONFLICT (timestamp) DO UPDATE SET + chiang_dao = EXCLUDED.chiang_dao, mae_taeng = EXCLUDED.mae_taeng, mae_ngat = EXCLUDED.mae_ngat, + mae_rim = EXCLUDED.mae_rim, chiang_mai = EXCLUDED.chiang_mai, catchment_mean = EXCLUDED.catchment_mean", + ) + .bind(rows.iter().map(|r| r.timestamp).collect::>()) + .bind(col(0)) + .bind(col(1)) + .bind(col(2)) + .bind(col(3)) + .bind(col(4)) + .bind(rows.iter().map(|r| r.catchment_mean).collect::>()) + .execute(pool) + .await + .context("upserting openmeteo_rain")?; + Ok(rows.len()) +} + +/// Row counts per table, for `check-db`. +pub async fn table_counts(pool: &PgPool) -> Result> { + let mut out = BTreeMap::new(); + for table in [ + "water_measurements", + "hii_rainfall", + "hii_waterlevel", + "rid_reservoir_daily", + "openmeteo_rain", + ] { + let n: i64 = sqlx::query_scalar(&format!("SELECT COUNT(*) FROM {table}")) + .fetch_one(pool) + .await?; + out.insert(table, n); + } + Ok(out) +} diff --git a/rust/collector/src/db_tests.rs b/rust/collector/src/db_tests.rs new file mode 100644 index 0000000..00deca2 --- /dev/null +++ b/rust/collector/src/db_tests.rs @@ -0,0 +1,332 @@ +//! Database tests against a real PostgreSQL. Each test runs in its own +//! throwaway schema. Set TEST_DATABASE_URL to run them, e.g. +//! `postgres://postgres:ping@localhost:55432/ping`; without it they are +//! skipped (reported as passed with a note on stderr). + +use crate::dams::DamRecord; +use crate::db; +use crate::hii::{RainRecord, WaterLevelRecord}; +use crate::rain::RainRow; +use crate::rid::{self, Reading, RidSource}; +use crate::stations::Stations; +use anyhow::Result; +use chrono::{Duration, NaiveDate, NaiveDateTime, NaiveTime}; +use sqlx::PgPool; +use sqlx::postgres::{PgConnectOptions, PgPoolOptions}; +use std::collections::HashMap; +use std::str::FromStr; +use std::sync::Mutex; + +struct TestDb { + pool: PgPool, + admin: PgPool, + schema: String, +} + +impl TestDb { + async fn new() -> Option { + let Ok(url) = std::env::var("TEST_DATABASE_URL") else { + eprintln!("TEST_DATABASE_URL not set: skipping database test"); + return None; + }; + let schema = format!( + "t_{}_{}", + std::process::id(), + chrono::Utc::now().timestamp_nanos_opt().unwrap() + ); + let admin = PgPool::connect(&url).await.expect("connect"); + sqlx::query(&format!("CREATE SCHEMA {schema}")) + .execute(&admin) + .await + .unwrap(); + let opts = PgConnectOptions::from_str(&url) + .unwrap() + .options([("search_path", schema.as_str())]); + let pool = PgPoolOptions::new() + .max_connections(2) + .connect_with(opts) + .await + .unwrap(); + db::ensure_schema(&pool).await.expect("schema"); + Some(TestDb { + pool, + admin, + schema, + }) + } + + async fn drop(self) { + self.pool.close().await; + sqlx::query(&format!("DROP SCHEMA {} CASCADE", self.schema)) + .execute(&self.admin) + .await + .unwrap(); + } +} + +fn d(y: i32, m: u32, day: u32) -> NaiveDate { + NaiveDate::from_ymd_opt(y, m, day).unwrap() +} +fn t(y: i32, m: u32, day: u32, h: u32, min: u32) -> NaiveDateTime { + d(y, m, day).and_hms_opt(h, min, 0).unwrap() +} + +/// Canned RID: date -> last published hour (1..=24); P.1 (column 8) and P.20 +/// (column 1). Records which reports were requested. +struct FakeRid { + published: HashMap, + calls: Mutex>, +} + +impl FakeRid { + fn new(published: &[(NaiveDate, i64)]) -> Self { + FakeRid { + published: published.iter().copied().collect(), + calls: Mutex::new(Vec::new()), + } + } + fn calls(&self) -> Vec { + self.calls.lock().unwrap().clone() + } +} + +impl RidSource for FakeRid { + async fn fetch_report(&self, date: NaiveDate) -> Result> { + self.calls.lock().unwrap().push(date); + let last = self.published.get(&date).copied().unwrap_or(0); + Ok((1..=last) + .flat_map(|h| { + let ts = date.and_time(NaiveTime::MIN) + Duration::hours(h); + [8u32, 1].map(|n| Reading { + timestamp: ts, + station_num: n, + water_level: 1.0 + h as f64 / 100.0, + discharge: Some(50.0), + discharge_percent: Some(12.5), + }) + }) + .collect()) + } +} + +async fn stored_hours(pool: &PgPool, day: NaiveDate) -> Vec { + let rec = db::recorded_hours_by_day(pool, day, day).await.unwrap(); + let mut h: Vec = rec + .get(&day) + .cloned() + .unwrap_or_default() + .into_iter() + .collect(); + h.sort(); + h +} + +#[tokio::test] +async fn late_report_is_stored_and_reported_behind() { + let Some(db) = TestDb::new().await else { + return; + }; + let today = d(2026, 10, 17); + let yesterday = today - Duration::days(1); + let stations = Stations::bundled(); + // yesterday complete already + let full = FakeRid::new(&[(yesterday, 24)]); + rid::fill_gaps(&full, &db.pool, &stations, Some(1), t(2026, 10, 17, 0, 30)) + .await + .unwrap(); + + // 21:30 and RID has published up to 20:00 + let src = FakeRid::new(&[(today, 20)]); + let o = rid::run_cycle(&src, &db.pool, &stations, t(2026, 10, 17, 21, 30)).await; + assert!(!o.current); + assert_eq!(o.latest, Some(t(2026, 10, 17, 20, 0))); + assert_eq!(o.saved, 40); + assert_eq!(src.calls(), vec![today]); + assert_eq!( + stored_hours(&db.pool, today).await, + (0..=20).collect::>() + ); + + // the 21:00 reading lands; the next poll is current + let src = FakeRid::new(&[(today, 21)]); + let o = rid::run_cycle(&src, &db.pool, &stations, t(2026, 10, 17, 21, 40)).await; + assert!(o.current); + db.drop().await; +} + +#[tokio::test] +async fn yesterdays_late_hours_are_recovered_after_midnight() { + let Some(db) = TestDb::new().await else { + return; + }; + let today = d(2026, 10, 17); + let yesterday = today - Duration::days(1); + let stations = Stations::bundled(); + // stored up to 21:00 yesterday (the evening RID published late) + let partial = FakeRid::new(&[(yesterday, 21)]); + rid::run_cycle(&partial, &db.pool, &stations, t(2026, 10, 16, 21, 5)).await; + assert_eq!( + stored_hours(&db.pool, yesterday).await, + (1..=21).collect::>() + ); + + // 02:10: RID now has all of yesterday and today up to 02:00 + let src = FakeRid::new(&[(yesterday, 24), (today, 2)]); + let o = rid::run_cycle(&src, &db.pool, &stations, t(2026, 10, 17, 2, 10)).await; + assert!(o.current); + assert_eq!(src.calls(), vec![yesterday, today]); + assert_eq!( + stored_hours(&db.pool, yesterday).await, + (1..=23).collect::>() + ); + assert_eq!(stored_hours(&db.pool, today).await, vec![0, 1, 2]); + + // complete now: the next cycle fetches only today's report + let src = FakeRid::new(&[(yesterday, 24), (today, 3)]); + rid::run_cycle(&src, &db.pool, &stations, t(2026, 10, 17, 3, 5)).await; + assert_eq!(src.calls(), vec![today]); + db.drop().await; +} + +#[tokio::test] +async fn rid_upsert_revises_values_and_rounds_like_numeric() { + let Some(db) = TestDb::new().await else { + return; + }; + let stations = Stations::bundled(); + let r = |level: f64| Reading { + timestamp: t(2026, 10, 17, 9, 0), + station_num: 8, + water_level: level, + discharge: None, + discharge_percent: None, + }; + db::save_rid(&db.pool, &[r(1.0700000524520874)], &stations) + .await + .unwrap(); + db::save_rid(&db.pool, &[r(1.0900000333786)], &stations) + .await + .unwrap(); + let rows: Vec<(String, f64, Option, String)> = sqlx::query_as( + "SELECT s.station_code, w.water_level::float8, w.discharge::float8, w.status + FROM water_measurements w JOIN stations s ON s.id = w.station_id", + ) + .fetch_all(&db.pool) + .await + .unwrap(); + assert_eq!(rows, vec![("P.1".into(), 1.09, None, "active".into())]); + db.drop().await; +} + +#[tokio::test] +async fn hii_dams_and_rain_round_trip() { + let Some(db) = TestDb::new().await else { + return; + }; + let now = t(2026, 10, 17, 21, 5); + let rain = RainRecord { + station_id: 11, + oldcode: Some("CHM001".into()), + name_th: None, + name_en: Some("Chiang Mai 1".into()), + latitude: Some(18.79), + longitude: Some(99.0), + sub_basin_id: Some("601".into()), + agency: Some("HII".into()), + timestamp: t(2026, 10, 17, 21, 0), + rain_1h: Some(1.234), + rain_24h: Some(12.0), + }; + assert_eq!( + db::save_hii_rain(&db.pool, &[rain.clone(), rain.clone()], now) + .await + .unwrap(), + 1 + ); + let (r1h,): (f64,) = sqlx::query_as("SELECT rain_1h::float8 FROM hii_rainfall") + .fetch_one(&db.pool) + .await + .unwrap(); + assert_eq!(r1h, 1.23); + + let wl = WaterLevelRecord { + station_id: 5, + oldcode: Some("ridhydro_P.1".into()), + rid_code: Some("P.1".into()), + name_th: Some("สะพานนวรัฐ".into()), + name_en: None, + latitude: None, + longitude: None, + agency: None, + river_name: None, + offset_msl: Some(300.0), + ground_level_msl: None, + min_bank_msl: None, + critical_level_msl: None, + critical_level_m: None, + qmax: None, + is_key_station: true, + timestamp: t(2026, 10, 17, 21, 0), + wl_msl: Some(301.2), + wl_m: None, + discharge: None, + flow_rate: None, + storage_percent: None, + situation_level: Some(2), + diff_wl_bank: None, + }; + db::save_hii_waterlevel(&db.pool, &[wl], now).await.unwrap(); + let (code, key, sit): (String, bool, i32) = sqlx::query_as( + "SELECT s.rid_code, s.is_key_station, w.situation_level + FROM hii_waterlevel w JOIN hii_wl_stations s ON s.id = w.station_id", + ) + .fetch_one(&db.pool) + .await + .unwrap(); + assert_eq!((code.as_str(), key, sit), ("P.1", true, 2)); + + let dam = |name: Option<&str>, level: Option| DamRecord { + dam_id: "200103".into(), + region: Some("north".into()), + name_th: name.map(str::to_string), + latitude: Some(19.17), + longitude: None, + capacity_max_mcm: Some(265.0), + capacity_normal_mcm: None, + date: d(2026, 10, 17), + storage_mcm: Some(200.0), + storage_pct: Some(75.5), + inflow_mcm: None, + outflow_mcm: Some(1.2), + level_msl: level, + }; + db::save_dams(&db.pool, &[dam(Some("แม่งัด"), Some(395.1))], now) + .await + .unwrap(); + // a later snapshot without name / level must not blank them + db::save_dams(&db.pool, &[dam(None, None)], now) + .await + .unwrap(); + let (name, level): (Option, Option) = sqlx::query_as( + "SELECT d.name_th, r.level_msl::float8 FROM rid_reservoir_daily r JOIN rid_dams d USING (dam_id)", + ) + .fetch_one(&db.pool) + .await + .unwrap(); + assert_eq!(name.as_deref(), Some("แม่งัด")); + assert_eq!(level, Some(395.1)); + + let rows = vec![RainRow { + timestamp: t(2026, 10, 18, 3, 0), + points: [Some(1.0), None, Some(2.0), None, Some(0.333)], + catchment_mean: Some(1.111), + }]; + db::save_rain(&db.pool, &rows).await.unwrap(); + let (mean, missing): (f64, Option) = + sqlx::query_as("SELECT catchment_mean::float8, mae_taeng::float8 FROM openmeteo_rain") + .fetch_one(&db.pool) + .await + .unwrap(); + assert_eq!((mean, missing), (1.11, None)); + db.drop().await; +} diff --git a/rust/collector/src/hii.rs b/rust/collector/src/hii.rs new file mode 100644 index 0000000..317a0ca --- /dev/null +++ b/rust/collector/src/hii.rs @@ -0,0 +1,262 @@ +//! HII / ThaiWater open api-v3 (no key): Ping-basin rain gauges and an +//! independent water-level network (m MSL). Port of src/hii_collector.py. + +use crate::util::{lenient_f64, lenient_i64, scalar_string, truthy}; +use anyhow::{Context, Result}; +use chrono::NaiveDateTime; +use serde_json::Value; + +#[derive(Clone, Debug, PartialEq)] +pub struct RainRecord { + pub station_id: i64, + pub oldcode: Option, + pub name_th: Option, + pub name_en: Option, + pub latitude: Option, + pub longitude: Option, + pub sub_basin_id: Option, + pub agency: Option, + pub timestamp: NaiveDateTime, + pub rain_1h: Option, + pub rain_24h: Option, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct WaterLevelRecord { + pub station_id: i64, + pub oldcode: Option, + pub rid_code: Option, + pub name_th: Option, + pub name_en: Option, + pub latitude: Option, + pub longitude: Option, + pub agency: Option, + pub river_name: Option, + pub offset_msl: Option, + pub ground_level_msl: Option, + pub min_bank_msl: Option, + pub critical_level_msl: Option, + pub critical_level_m: Option, + pub qmax: Option, + pub is_key_station: bool, + pub timestamp: NaiveDateTime, + pub wl_msl: Option, + pub wl_m: Option, + pub discharge: Option, + pub flow_rate: Option, + pub storage_percent: Option, + pub situation_level: Option, + pub diff_wl_bank: Option, +} + +/// Thai local time, '2026-08-11 13:00' or with seconds. +fn parse_datetime(v: Option<&Value>) -> Option { + let s = v?.as_str()?; + NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M") + .or_else(|_| NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S")) + .ok() +} + +/// `tele_station_name` is {th, en} or a bare (Thai) string. +fn name(station: &Value, lang: &str) -> Option { + match station.get("tele_station_name")? { + Value::Object(o) => o.get(lang).and_then(Value::as_str).map(str::to_string), + Value::String(s) if lang == "th" => Some(s.clone()), + _ => None, + } +} + +fn string(v: Option<&Value>) -> Option { + v.and_then(Value::as_str).map(str::to_string) +} + +fn agency(row: &Value) -> Option { + string(row.pointer("/agency/agency_shortname/en")) +} + +/// 'P.1', 'ridhydro_P.67' -> 'P.1' / 'P.67'; 'ridtele_TUP.14' -> None. +/// (Python: `(?:^|_)(P\.\d+[A-Z]?)$`.) +pub fn rid_code_from_oldcode(oldcode: &str) -> Option { + let seg = oldcode.rsplit('_').next()?; + let digits = seg.strip_prefix("P.")?; + let (num, suffix) = match digits.char_indices().find(|(_, c)| !c.is_ascii_digit()) { + Some((i, _)) => digits.split_at(i), + None => (digits, ""), + }; + let suffix_ok = suffix.is_empty() + || (suffix.chars().count() == 1 && suffix.chars().all(|c| c.is_ascii_uppercase())); + (!num.is_empty() && suffix_ok).then(|| seg.to_string()) +} + +fn in_basin(row: &Value, basin_code: i64) -> bool { + row.pointer("/basin/basin_code").and_then(Value::as_i64) == Some(basin_code) +} + +pub fn parse_rain(payload: &Value, basin_code: i64) -> Vec { + let rows = payload + .get("data") + .and_then(Value::as_array) + .cloned() + .unwrap_or_default(); + rows.iter() + .filter(|row| in_basin(row, basin_code)) + .filter_map(|row| { + let station = row.get("station").unwrap_or(&Value::Null); + Some(RainRecord { + station_id: lenient_i64(station.get("id"))?, + timestamp: parse_datetime(row.get("rainfall_datetime"))?, + oldcode: string(station.get("tele_station_oldcode")), + name_th: name(station, "th"), + name_en: name(station, "en"), + latitude: lenient_f64(station.get("tele_station_lat")), + longitude: lenient_f64(station.get("tele_station_long")), + // Python: str(x or "") or None + sub_basin_id: truthy(station.get("sub_basin_id")) + .then(|| scalar_string(station.get("sub_basin_id"))) + .flatten(), + agency: agency(row), + rain_1h: lenient_f64(row.get("rain_1h")), + rain_24h: lenient_f64(row.get("rain_24h")), + }) + }) + .collect() +} + +pub fn parse_waterlevel(payload: &Value, basin_code: i64) -> Vec { + let rows = payload + .pointer("/waterlevel_data/data") + .and_then(Value::as_array) + .cloned() + .unwrap_or_default(); + rows.iter() + .filter(|row| in_basin(row, basin_code)) + .filter_map(|row| { + let station = row.get("station").unwrap_or(&Value::Null); + let oldcode = string(station.get("tele_station_oldcode")); + Some(WaterLevelRecord { + station_id: lenient_i64(station.get("id"))?, + timestamp: parse_datetime(row.get("waterlevel_datetime"))?, + rid_code: oldcode.as_deref().and_then(rid_code_from_oldcode), + oldcode, + name_th: name(station, "th"), + name_en: name(station, "en"), + latitude: lenient_f64(station.get("tele_station_lat")), + longitude: lenient_f64(station.get("tele_station_long")), + agency: agency(row), + river_name: string(row.get("river_name")), + offset_msl: lenient_f64(station.get("offset")), + ground_level_msl: lenient_f64(station.get("ground_level")), + min_bank_msl: lenient_f64(station.get("min_bank")), + critical_level_msl: lenient_f64(station.get("critical_level_msl")), + critical_level_m: lenient_f64(station.get("critical_level_m")), + qmax: lenient_f64(station.get("qmax")), + is_key_station: truthy(station.get("is_key_station")), + wl_msl: lenient_f64(row.get("waterlevel_msl")), + wl_m: lenient_f64(row.get("waterlevel_m")), + discharge: lenient_f64(row.get("discharge")), + flow_rate: lenient_f64(row.get("flow_rate")), + storage_percent: lenient_f64(row.get("storage_percent")), + situation_level: lenient_i64(row.get("situation_level")).map(|v| v as i32), + diff_wl_bank: lenient_f64(row.get("diff_wl_bank")), + }) + }) + .collect() +} + +pub struct HiiClient { + http: reqwest::Client, + base: String, +} + +impl HiiClient { + pub fn new(http: reqwest::Client, base: String) -> Self { + Self { + http, + base: base.trim_end_matches('/').to_string(), + } + } + + async fn get(&self, endpoint: &str) -> Result { + self.http + .get(format!("{}/{endpoint}", self.base)) + .timeout(std::time::Duration::from_secs(90)) + .send() + .await + .with_context(|| format!("HII {endpoint} request failed"))? + .error_for_status()? + .json() + .await + .with_context(|| format!("HII {endpoint} response is not JSON")) + } + + pub async fn fetch_rain(&self, basin_code: i64) -> Result> { + Ok(parse_rain(&self.get("rain_24h").await?, basin_code)) + } + + pub async fn fetch_waterlevel(&self, basin_code: i64) -> Result> { + Ok(parse_waterlevel( + &self.get("waterlevel_load").await?, + basin_code, + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn rid_codes() { + assert_eq!(rid_code_from_oldcode("P.1").as_deref(), Some("P.1")); + assert_eq!( + rid_code_from_oldcode("ridhydro_P.67").as_deref(), + Some("P.67") + ); + assert_eq!(rid_code_from_oldcode("x_P.4A").as_deref(), Some("P.4A")); + assert_eq!(rid_code_from_oldcode("ridtele_TUP.14"), None); + assert_eq!(rid_code_from_oldcode("P.4AB"), None); + assert_eq!(rid_code_from_oldcode("CHM001"), None); + } + + #[test] + fn rain_rows_filtered_to_basin() { + let payload = json!({"data": [ + {"basin": {"basin_code": 6}, "station": {"id": 11, "tele_station_oldcode": "CHM001", + "tele_station_name": {"th": "ชม1", "en": "Chiang Mai 1"}, "tele_station_lat": "18.79", + "tele_station_long": 99.0, "sub_basin_id": 601}, + "agency": {"agency_shortname": {"en": "HII"}}, + "rainfall_datetime": "2026-08-11 13:00", "rain_1h": "1.5", "rain_24h": 12}, + {"basin": {"basin_code": 7}, "station": {"id": 12}, "rainfall_datetime": "2026-08-11 13:00"}, + {"basin": {"basin_code": 6}, "station": {"id": 13}, "rainfall_datetime": "bad"} + ]}); + let r = parse_rain(&payload, 6); + assert_eq!(r.len(), 1); + assert_eq!(r[0].station_id, 11); + assert_eq!(r[0].name_en.as_deref(), Some("Chiang Mai 1")); + assert_eq!(r[0].sub_basin_id.as_deref(), Some("601")); + assert_eq!(r[0].agency.as_deref(), Some("HII")); + assert_eq!(r[0].rain_1h, Some(1.5)); + assert_eq!(r[0].latitude, Some(18.79)); + } + + #[test] + fn waterlevel_rows() { + let payload = json!({"waterlevel_data": {"data": [ + {"basin": {"basin_code": 6}, + "station": {"id": 5, "tele_station_oldcode": "ridhydro_P.1", "tele_station_name": "สะพานนวรัฐ", + "offset": "300.0", "is_key_station": 1}, + "waterlevel_datetime": "2026-08-11 13:00:00", "waterlevel_msl": "301.2", + "situation_level": "2", "river_name": "แม่น้ำปิง"} + ]}}); + let r = parse_waterlevel(&payload, 6); + assert_eq!(r.len(), 1); + assert_eq!(r[0].rid_code.as_deref(), Some("P.1")); + assert_eq!(r[0].name_th.as_deref(), Some("สะพานนวรัฐ")); + assert_eq!(r[0].name_en, None); + assert!(r[0].is_key_station); + assert_eq!(r[0].situation_level, Some(2)); + assert_eq!(r[0].wl_msl, Some(301.2)); + assert_eq!(r[0].offset_msl, Some(300.0)); + } +} diff --git a/rust/collector/src/main.rs b/rust/collector/src/main.rs new file mode 100644 index 0000000..f8e4861 --- /dev/null +++ b/rust/collector/src/main.rs @@ -0,0 +1,117 @@ +//! ping-collector: the Ping River Monitor's data collection service. +//! +//! Replaces the collection half of the Python web app (RID hourly gauges, +//! HII/ThaiWater rain + water level, RID dams, Open-Meteo rain). Writes the +//! same PostgreSQL tables; the Python API keeps forecasting and notifying +//! with `COLLECTION_ENABLED=false`. See docs/RUST_REWRITE.md. + +mod config; +mod dams; +mod db; +#[cfg(test)] +mod db_tests; +mod hii; +mod rain; +mod rid; +mod service; +mod stations; +mod util; + +use anyhow::{Result, bail}; +use clap::{Parser, Subcommand}; +use std::path::PathBuf; + +#[derive(Parser)] +#[command(version, about)] +struct Cli { + /// .env file to load (default: ./.env when present). Real environment + /// variables take precedence. + #[arg(long, global = true)] + env_file: Option, + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand)] +enum Command { + /// Run the collection service (what systemd starts) + Run, + /// One cycle of every source, then exit + Once { + /// Fetch and compare with the database, write nothing + #[arg(long)] + dry_run: bool, + }, + /// Re-fetch RID reports that have holes: a number of days back, or "all" + FillGaps { days: String }, + /// Connect, create missing tables, print row counts and the newest RID reading + CheckDb, +} + +#[tokio::main] +async fn main() -> Result<()> { + let cli = Cli::parse(); + match &cli.env_file { + Some(path) => { + dotenvy::from_path(path)?; + } + None => { + dotenvy::dotenv().ok(); + } + } + let level = std::env::var("RUST_LOG") + .or_else(|_| std::env::var("LOG_LEVEL").map(|l| l.to_lowercase())) + .unwrap_or_else(|_| "info".into()); + tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::new(format!( + "warn,ping_collector={level}" + ))) + .with_target(false) + .init(); + + let cfg = config::Config::from_env()?; + tracing::info!( + "ping-collector {} -> {}", + env!("CARGO_PKG_VERSION"), + cfg.redacted_database_url() + ); + let pool = db::connect(&cfg.database_url).await?; + let stations = stations::Stations::load(&cfg.station_config_path)?; + let collector = service::Collector::new(cfg, pool, stations)?; + + match cli.command { + Command::Run => collector.run().await, + Command::Once { dry_run } => collector.once(dry_run).await, + Command::FillGaps { days } => { + let days = match days.as_str() { + "all" => None, + n => match n.parse::() { + Ok(d) => Some(d), + Err(_) => bail!("fill-gaps takes a number of days or \"all\", got {n:?}"), + }, + }; + db::ensure_schema(&collector.pool).await?; + let filled = rid::fill_gaps( + &collector.rid, + &collector.pool, + &collector.stations, + days, + util::bangkok_now(), + ) + .await?; + println!("{filled} readings re-fetched"); + Ok(()) + } + Command::CheckDb => { + db::ensure_schema(&collector.pool).await?; + for (table, n) in db::table_counts(&collector.pool).await? { + println!("{table:<22} {n:>10} rows"); + } + println!( + "newest RID reading: {:?}", + db::latest_rid_timestamp(&collector.pool).await? + ); + Ok(()) + } + } +} diff --git a/rust/collector/src/rain.rs b/rust/collector/src/rain.rs new file mode 100644 index 0000000..431d381 --- /dev/null +++ b/rust/collector/src/rain.rs @@ -0,0 +1,133 @@ +//! Open-Meteo hourly precipitation at five upper-Ping catchment points +//! (recent days + next 48 h) into `openmeteo_rain`. Port of the live path of +//! src/ml/rain.py (`fetch_forecast` + `save_to_db`). Times are Asia/Bangkok. + +use anyhow::{Context, Result}; +use chrono::NaiveDateTime; +use serde_json::Value; +use std::collections::BTreeMap; + +/// (column, lat, lon), headwaters to city. Order = column order in the table. +pub const CATCHMENT_POINTS: [(&str, f64, f64); 5] = [ + ("chiang_dao", 19.37, 98.97), + ("mae_taeng", 19.12, 98.94), + ("mae_ngat", 19.17, 99.05), + ("mae_rim", 18.92, 98.92), + ("chiang_mai", 18.79, 99.00), +]; + +#[derive(Clone, Debug, PartialEq)] +pub struct RainRow { + pub timestamp: NaiveDateTime, + pub points: [Option; 5], + /// Mean of the points that have a value (pandas `mean(axis=1)`). + pub catchment_mean: Option, +} + +/// A multi-coordinate request returns a list, one result per point. +pub fn parse(payload: &Value) -> Vec { + let results: Vec<&Value> = match payload { + Value::Array(a) => a.iter().collect(), + other => vec![other], + }; + let mut by_time: BTreeMap; 5]> = BTreeMap::new(); + for (i, result) in results.iter().take(CATCHMENT_POINTS.len()).enumerate() { + let hourly = result.get("hourly"); + let times = hourly.and_then(|h| h.get("time")).and_then(Value::as_array); + let values = hourly + .and_then(|h| h.get("precipitation")) + .and_then(Value::as_array); + let (Some(times), Some(values)) = (times, values) else { + continue; + }; + for (t, v) in times.iter().zip(values) { + let Some(ts) = t + .as_str() + .and_then(|s| NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M").ok()) + else { + continue; + }; + by_time.entry(ts).or_insert([None; 5])[i] = v.as_f64().filter(|x| x.is_finite()); + } + } + by_time + .into_iter() + .map(|(timestamp, points)| { + let present: Vec = points.iter().flatten().copied().collect(); + let catchment_mean = + (!present.is_empty()).then(|| present.iter().sum::() / present.len() as f64); + RainRow { + timestamp, + points, + catchment_mean, + } + }) + .collect() +} + +pub struct OpenMeteoClient { + http: reqwest::Client, + url: String, +} + +impl OpenMeteoClient { + pub fn new(http: reqwest::Client, url: String) -> Self { + Self { http, url } + } + + pub async fn fetch_forecast(&self, past_days: u32, forecast_days: u32) -> Result> { + let join = |f: fn(&(&str, f64, f64)) -> f64| { + CATCHMENT_POINTS + .iter() + .map(|p| f(p).to_string()) + .collect::>() + .join(",") + }; + let payload: Value = self + .http + .get(&self.url) + .query(&[ + ("latitude", join(|p| p.1)), + ("longitude", join(|p| p.2)), + ("hourly", "precipitation".into()), + ("timezone", "Asia/Bangkok".into()), + ("past_days", past_days.to_string()), + ("forecast_days", forecast_days.to_string()), + ]) + .timeout(std::time::Duration::from_secs(60)) + .send() + .await + .context("Open-Meteo request failed")? + .error_for_status()? + .json() + .await + .context("Open-Meteo response is not JSON")?; + Ok(parse(&payload)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn aligns_points_and_averages_present_values() { + let point = |vals: Value| json!({"hourly": {"time": ["2026-10-17T00:00", "2026-10-17T01:00"], "precipitation": vals}}); + let payload = json!([ + point(json!([1.0, null])), + point(json!([3.0, null])), + point(json!([null, null])), + point(json!([2.0, null])), + point(json!([0.0, 4.0])), + ]); + let rows = parse(&payload); + assert_eq!(rows.len(), 2); + assert_eq!( + rows[0].points, + [Some(1.0), Some(3.0), None, Some(2.0), Some(0.0)] + ); + assert_eq!(rows[0].catchment_mean, Some(1.5)); + assert_eq!(rows[1].catchment_mean, Some(4.0)); + } +} diff --git a/rust/collector/src/rid.rs b/rust/collector/src/rid.rs new file mode 100644 index 0000000..ec64830 --- /dev/null +++ b/rust/collector/src/rid.rs @@ -0,0 +1,435 @@ +//! RID hourly water-level/discharge report (16 Ping gauges). +//! +//! `POST getGroupHourlyWaterLevelReportAllHL.ashx` with a Buddhist-era date +//! returns 24 rows (`hourlytime` "1.00".."24.00"; hour 24 is 00:00 of the next +//! day) with `wlvaluesN` / `qvaluesN` / `QPercentN` for report column N=1..16. +//! Hours not published yet are null. +//! +//! The report often lags 1-2 h in the evening. Each cycle therefore stores +//! every reading it gets (not only "the current hour"), and re-fetches +//! yesterday's report while the store still has holes in it: hours 22-24 of a +//! day are reachable only through that day's report. + +use crate::db; +use crate::stations::Stations; +use crate::util::{floor_hour, lenient_f64}; +use anyhow::{Context, Result}; +use chrono::{Datelike, Duration, NaiveDate, NaiveDateTime, NaiveTime}; +use serde_json::Value; +use sqlx::PgPool; +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::future::Future; + +pub const N_STATIONS: u32 = 16; + +// Plausibility ranges (src/validators.py) +const WATER_LEVEL_RANGE: (f64, f64) = (-10.0, 50.0); +const DISCHARGE_RANGE: (f64, f64) = (0.0, 10_000.0); +// % of channel capacity; Oct 2024 peaked at 226 %. Out of range -> NULL, +// the reading itself is kept. +const DISCHARGE_PERCENT_RANGE: (f64, f64) = (0.0, 500.0); + +#[derive(Clone, Debug, PartialEq)] +pub struct Reading { + pub timestamp: NaiveDateTime, + /// RID report column 1..16 (= stations.id in the database) + pub station_num: u32, + pub water_level: f64, + pub discharge: Option, + pub discharge_percent: Option, +} + +fn within(x: f64, (lo, hi): (f64, f64)) -> bool { + lo <= x && x <= hi +} + +/// Parse one report. Rows with an unparsable hour, a missing/implausible +/// water level or an implausible discharge are dropped, as in Python. +pub fn parse_report(report_date: NaiveDate, payload: &Value) -> Vec { + let mut out = Vec::new(); + let Some(rows) = payload.get("rows").and_then(Value::as_array) else { + return out; + }; + for row in rows { + let hour = match row.get("hourlytime") { + Some(Value::String(s)) => s.trim().parse::().ok(), + Some(Value::Number(n)) => n.as_f64(), + _ => None, + }; + let Some(hour) = hour + .map(|h| h.trunc() as i64) + .filter(|h| (1..=24).contains(h)) + else { + continue; + }; + let timestamp = report_date.and_time(NaiveTime::MIN) + Duration::hours(hour); + + for n in 1..=N_STATIONS { + let Some(water_level) = lenient_f64(row.get(format!("wlvalues{n}"))) else { + continue; + }; + if !within(water_level, WATER_LEVEL_RANGE) { + tracing::warn!( + "RID {timestamp} column {n}: water level {water_level} out of range" + ); + continue; + } + let discharge = lenient_f64(row.get(format!("qvalues{n}"))); + if let Some(q) = discharge + && !within(q, DISCHARGE_RANGE) + { + tracing::warn!("RID {timestamp} column {n}: discharge {q} out of range"); + continue; + } + let discharge_percent = discharge + .and(lenient_f64(row.get(format!("QPercent{n}")))) + .filter(|p| within(*p, DISCHARGE_PERCENT_RANGE)); + out.push(Reading { + timestamp, + station_num: n, + water_level, + discharge, + discharge_percent, + }); + } + } + out +} + +/// Anything that can produce a parsed RID report (HTTP in production, +/// canned rows in tests). +pub trait RidSource { + fn fetch_report(&self, date: NaiveDate) -> impl Future>> + Send; +} + +pub struct RidClient { + http: reqwest::Client, + url: String, +} + +impl RidClient { + pub fn new(http: reqwest::Client, url: String) -> Self { + Self { http, url } + } +} + +impl RidSource for RidClient { + async fn fetch_report(&self, date: NaiveDate) -> Result> { + let thai_date = format!( + "{:02}/{:02}/{}", + date.day(), + date.month(), + date.year() + 543 + ); + let nd = chrono::Utc::now().timestamp_millis().to_string(); + let form = [ + ("DW[UtokID]", "1"), + ("DW[BasinID]", "6"), + ("DW[TimeCurrent]", thai_date.as_str()), + ("_search", "false"), + ("nd", nd.as_str()), + ("rows", "100"), + ("page", "1"), + ("sidx", "indexhourly"), + ("sord", "asc"), + ]; + let payload: Value = self + .http + .post(&self.url) + .header("X-Requested-With", "XMLHttpRequest") + .header("Accept", "application/json, text/javascript, */*; q=0.01") + .form(&form) + .timeout(std::time::Duration::from_secs(30)) + .send() + .await + .context("RID request failed")? + .error_for_status()? + .json() + .await + .context("RID response is not JSON")?; + Ok(parse_report(date, &payload)) + } +} + +/// Report dates whose re-fetch would fill holes in [start, end]. +/// +/// Hours 1-23 of day D come from D's report, hour 0 from the previous day's +/// report ("hour 24"). For today only hours already passed are expected. +pub fn gap_fetch_dates( + start: NaiveDate, + end: NaiveDate, + now: NaiveDateTime, + recorded: &HashMap>, +) -> BTreeSet { + let mut dates = BTreeSet::new(); + let empty = HashSet::new(); + let mut day = start; + while day <= end { + let expected: Vec = if day == now.date() { + (0..chrono::Timelike::hour(&now)).collect() + } else { + (0..24).collect() + }; + let have = recorded.get(&day).unwrap_or(&empty); + let missing: Vec = expected.into_iter().filter(|h| !have.contains(h)).collect(); + if missing.iter().any(|&h| h >= 1) { + dates.insert(day); + } + if missing.contains(&0) && day > start { + dates.insert(day - Duration::days(1)); + } + day += Duration::days(1); + } + dates +} + +/// Reports to fetch in one cycle: the one holding the current hour (today's, +/// or before 01:00 yesterday's) plus yesterday's while it has holes. +pub async fn cycle_dates(pool: &PgPool, now: NaiveDateTime) -> Vec { + let today = now.date(); + let yesterday = today - Duration::days(1); + let mut dates = BTreeSet::from([if chrono::Timelike::hour(&now) >= 1 { + today + } else { + yesterday + }]); + match db::recorded_hours_by_day(pool, yesterday, today).await { + Ok(recorded) => dates.extend(gap_fetch_dates(yesterday, today, now, &recorded)), + Err(e) => tracing::warn!("gap detection failed, fetching the current report only: {e:#}"), + } + dates.into_iter().collect() +} + +#[derive(Clone, Debug, Default)] +pub struct CycleOutcome { + pub dates: Vec, + pub fetched: usize, + pub saved: usize, + pub latest: Option, + /// The current hour's reading is in: poll again next hour. + pub current: bool, + pub error: Option, +} + +pub async fn fetch_dates( + src: &impl RidSource, + dates: &[NaiveDate], +) -> (Vec, Option) { + let mut readings = Vec::new(); + let mut error = None; + for &d in dates { + match src.fetch_report(d).await { + Ok(r) => readings.extend(r), + Err(e) => { + tracing::error!("RID report {d}: {e:#}"); + error = Some(format!("{e:#}")); + } + } + } + (readings, error) +} + +/// One RID poll: fetch, store everything, report whether the feed is current. +pub async fn run_cycle( + src: &impl RidSource, + pool: &PgPool, + stations: &Stations, + now: NaiveDateTime, +) -> CycleOutcome { + let mut out = CycleOutcome { + dates: cycle_dates(pool, now).await, + ..Default::default() + }; + let (readings, fetch_error) = fetch_dates(src, &out.dates).await; + out.fetched = readings.len(); + out.error = fetch_error; + if readings.is_empty() { + out.error.get_or_insert_with(|| "no data fetched".into()); + tracing::warn!("RID: no data fetched for {:?}", out.dates); + return out; + } + if let Err(e) = db::save_rid(pool, &readings, stations).await { + tracing::error!("RID save failed: {e:#}"); + out.error = Some(format!("save failed: {e:#}")); + return out; + } + out.saved = readings.len(); + let latest = readings + .iter() + .map(|r| r.timestamp) + .max() + .expect("non-empty"); + let current_hour = floor_hour(now); + out.latest = Some(latest); + out.current = latest >= current_hour; + if out.current { + tracing::info!("RID: {} readings saved, latest {latest}", out.saved); + } else { + tracing::warn!( + "RID: {} readings saved up to {latest}; {current_hour} not published yet ({} h behind), will retry", + out.saved, + (current_hour - latest).num_hours() + ); + } + out +} + +/// Re-fetch every report with holes in the last `days` days (None = whole +/// history). Returns readings re-saved. +pub async fn fill_gaps( + src: &impl RidSource, + pool: &PgPool, + stations: &Stations, + days: Option, + now: NaiveDateTime, +) -> Result { + let end = now.date(); + let start = match days { + Some(d) => end - Duration::days(d as i64), + None => match db::earliest_rid_timestamp(pool).await? { + Some(t) => t.date(), + None => { + tracing::warn!("database is empty; nothing to gap-fill"); + return Ok(0); + } + }, + }; + let recorded = db::recorded_hours_by_day(pool, start, end).await?; + let dates = gap_fetch_dates(start, end, now, &recorded); + tracing::info!( + "gap fill {start}..{end}: {} report(s) to re-fetch", + dates.len() + ); + let mut filled = 0; + for (i, d) in dates.iter().enumerate() { + if i > 0 { + tokio::time::sleep(std::time::Duration::from_secs(1)).await; // be polite + } + match src.fetch_report(*d).await { + Ok(readings) if !readings.is_empty() => { + db::save_rid(pool, &readings, stations).await?; + filled += readings.len(); + } + Ok(_) => tracing::info!("gap fill {d}: RID has no data"), + Err(e) => tracing::warn!("gap fill {d}: {e:#}"), + } + } + tracing::info!("gap fill done: {filled} readings re-saved"); + Ok(filled) +} + +/// Next RID poll: `offset` minutes past the next hour when current, else in +/// `retry` minutes (never later than that hourly poll). +pub fn next_poll_time( + now: NaiveDateTime, + current: bool, + offset_min: u32, + retry_min: u32, +) -> NaiveDateTime { + let on_schedule = floor_hour(now) + Duration::hours(1) + Duration::minutes(offset_min as i64); + if current { + on_schedule + } else { + on_schedule.min(now + Duration::minutes(retry_min as i64)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn d(y: i32, m: u32, day: u32) -> NaiveDate { + NaiveDate::from_ymd_opt(y, m, day).unwrap() + } + fn t(y: i32, m: u32, day: u32, h: u32, min: u32) -> NaiveDateTime { + d(y, m, day).and_hms_opt(h, min, 0).unwrap() + } + + #[test] + fn parses_levels_discharge_and_percent() { + let p = json!({"rows": [{"hourlytime": "9.00", "wlvalues1": "3.50", "qvalues1": "120.5", "QPercent1": 45.2}]}); + let r = parse_report(d(2026, 7, 22), &p); + assert_eq!( + r, + vec![Reading { + timestamp: t(2026, 7, 22, 9, 0), + station_num: 1, + water_level: 3.5, + discharge: Some(120.5), + discharge_percent: Some(45.2), + }] + ); + } + + #[test] + fn hour_24_is_next_day_midnight_and_junk_hours_are_skipped() { + let p = json!({"rows": [ + {"hourlytime": "24.00", "wlvalues1": 3.0}, + {"hourlytime": "25.00", "wlvalues1": 3.0}, + {"hourlytime": "0.00", "wlvalues1": 3.0}, + {"hourlytime": "", "wlvalues1": 3.0} + ]}); + let r = parse_report(d(2026, 7, 22), &p); + assert_eq!(r.len(), 1); + assert_eq!(r[0].timestamp, t(2026, 7, 23, 0, 0)); + } + + #[test] + fn missing_or_junk_values() { + let p = json!({"rows": [{"hourlytime": "10.00", + "wlvalues1": null, + "wlvalues2": 2.5, "qvalues2": "***", "QPercent2": 10, + "wlvalues3": 80.0, + "wlvalues4": 1.0, "qvalues4": "20", "QPercent4": 900, + "wlvalues5": 1.0, "qvalues5": "-5"}]}); + let r = parse_report(d(2026, 7, 22), &p); + let by: HashMap = r.iter().map(|x| (x.station_num, x)).collect(); + assert!(!by.contains_key(&1)); // no level + assert_eq!(by[&2].discharge, None); // "***" + assert_eq!(by[&2].discharge_percent, None); // percent needs a discharge + assert!(!by.contains_key(&3)); // level out of range + assert_eq!(by[&4].discharge_percent, None); // 900 % nulled, reading kept + assert!(!by.contains_key(&5)); // negative discharge drops the reading + } + + #[test] + fn missing_rows_key_is_empty() { + assert!(parse_report(d(2026, 7, 22), &json!({"unexpected": true})).is_empty()); + } + + fn hours(spec: &[(NaiveDate, std::ops::Range)]) -> HashMap> { + spec.iter() + .map(|(day, r)| (*day, r.clone().collect())) + .collect() + } + + #[test] + fn gap_dates_late_evening_and_midnight() { + let today = d(2026, 10, 17); + let yesterday = d(2026, 10, 16); + // 02:10, yesterday has 0..22, today only hour 1: yesterday's report holds + // 22:00, 23:00 and today's 00:00 ("hour 24") + let rec = hours(&[(yesterday, 0..22), (today, 1..2)]); + let got = gap_fetch_dates(yesterday, today, t(2026, 10, 17, 2, 10), &rec); + assert_eq!(got.into_iter().collect::>(), vec![yesterday]); + // today's 01:00 missing as well -> today's report too + let rec = hours(&[(yesterday, 0..22)]); + let got = gap_fetch_dates(yesterday, today, t(2026, 10, 17, 2, 10), &rec); + assert_eq!(got.into_iter().collect::>(), vec![yesterday, today]); + // complete yesterday, today complete up to now -> nothing + let rec = hours(&[(yesterday, 0..24), (today, 0..15)]); + assert!(gap_fetch_dates(yesterday, today, t(2026, 10, 17, 15, 5), &rec).is_empty()); + } + + #[test] + fn poll_schedule() { + let now = t(2026, 10, 17, 21, 6); + assert_eq!(next_poll_time(now, true, 5, 10), t(2026, 10, 17, 22, 5)); + assert_eq!(next_poll_time(now, false, 5, 10), t(2026, 10, 17, 21, 16)); + assert_eq!( + next_poll_time(t(2026, 10, 17, 21, 58), false, 5, 10), + t(2026, 10, 17, 22, 5) + ); + } +} diff --git a/rust/collector/src/service.rs b/rust/collector/src/service.rs new file mode 100644 index 0000000..d4b00c4 --- /dev/null +++ b/rust/collector/src/service.rs @@ -0,0 +1,273 @@ +//! The long-running collection loop and the one-shot commands. + +use crate::config::{Config, USER_AGENT}; +use crate::dams::DamsClient; +use crate::db; +use crate::hii::HiiClient; +use crate::rain::OpenMeteoClient; +use crate::rid::{self, CycleOutcome, RidClient}; +use crate::stations::Stations; +use crate::util::{bangkok_now, floor_hour}; +use anyhow::Result; +use chrono::{Duration, NaiveDateTime}; +use serde_json::json; +use sqlx::PgPool; + +pub struct Collector { + pub cfg: Config, + pub pool: PgPool, + pub stations: Stations, + pub rid: RidClient, + hii: HiiClient, + dams: DamsClient, + rain: OpenMeteoClient, +} + +#[derive(Default)] +struct Stats { + total_runs: u64, + successful_runs: u64, + failed_runs: u64, + last_run: Option, + next_run: Option, + feed_current: Option, + latest_reading: Option, + readings_saved: usize, +} + +fn iso(t: Option) -> serde_json::Value { + t.map_or(serde_json::Value::Null, |t| { + json!(t.format("%Y-%m-%dT%H:%M:%S").to_string()) + }) +} + +impl Collector { + pub fn new(cfg: Config, pool: PgPool, stations: Stations) -> Result { + let http = reqwest::Client::builder() + .user_agent(USER_AGENT) + .connect_timeout(std::time::Duration::from_secs(20)) + .build()?; + Ok(Self { + rid: RidClient::new(http.clone(), cfg.rid_api_url.clone()), + hii: HiiClient::new(http.clone(), cfg.hii_api_base.clone()), + dams: DamsClient::new(http.clone(), cfg.dams_url.clone()), + rain: OpenMeteoClient::new(http, cfg.openmeteo_url.clone()), + cfg, + pool, + stations, + }) + } + + /// HII rain + water level, RID dams (today and yesterday), Open-Meteo. + /// Each source is failure-isolated. + pub async fn collect_hourly(&self, now: NaiveDateTime) { + if self.cfg.hii_enabled { + match self.hii.fetch_rain(self.cfg.hii_basin_code).await { + Ok(r) => match db::save_hii_rain(&self.pool, &r, now).await { + Ok(n) => tracing::info!("HII rainfall: {n} rows saved"), + Err(e) => tracing::error!("HII rainfall save failed: {e:#}"), + }, + Err(e) => tracing::error!("HII rainfall fetch failed: {e:#}"), + } + match self.hii.fetch_waterlevel(self.cfg.hii_basin_code).await { + Ok(r) => match db::save_hii_waterlevel(&self.pool, &r, now).await { + Ok(n) => tracing::info!("HII water level: {n} rows saved"), + Err(e) => tracing::error!("HII water level save failed: {e:#}"), + }, + Err(e) => tracing::error!("HII water level fetch failed: {e:#}"), + } + } + if self.cfg.dams_enabled { + for date in [None, Some(now.date() - Duration::days(1))] { + match self.dams.fetch_day(date).await { + Ok(r) => match db::save_dams(&self.pool, &r, now).await { + Ok(n) => tracing::info!("RID dams {date:?}: {n} dam-day rows saved"), + Err(e) => tracing::error!("RID dams save failed: {e:#}"), + }, + Err(e) => tracing::error!("RID dams fetch {date:?} failed: {e:#}"), + } + } + } + if self.cfg.rain_enabled { + match self.rain.fetch_forecast(5, 2).await { + Ok(r) => match db::save_rain(&self.pool, &r).await { + Ok(n) => tracing::info!("Open-Meteo rain: {n} hourly rows upserted"), + Err(e) => tracing::error!("Open-Meteo save failed: {e:#}"), + }, + Err(e) => tracing::error!("Open-Meteo fetch failed: {e:#}"), + } + } + } + + fn write_status(&self, stats: &Stats, running: bool) { + let Some(path) = &self.cfg.status_file else { + return; + }; + let body = json!({ + "is_running": running, + "total_runs": stats.total_runs, + "successful_runs": stats.successful_runs, + "failed_runs": stats.failed_runs, + "last_run": iso(stats.last_run), + "next_run": iso(stats.next_run), + "feed_current": stats.feed_current, + "latest_reading": iso(stats.latest_reading), + "readings_saved": stats.readings_saved, + "collector": concat!("ping-collector ", env!("CARGO_PKG_VERSION")), + }); + let tmp = path.with_extension("json.tmp"); + let result = + std::fs::write(&tmp, body.to_string()).and_then(|_| std::fs::rename(&tmp, path)); + if let Err(e) = result { + tracing::warn!("could not write status file {}: {e}", path.display()); + } + } + + /// The service: startup gap fill, then RID polls on the hour (+offset) + /// with retries while RID is behind, and the hourly sources once per + /// clock hour. Returns when a shutdown signal arrives. + pub async fn run(&self) -> Result<()> { + db::ensure_schema(&self.pool).await?; + let shutdown = shutdown_signal(); + tokio::pin!(shutdown); + + if self.cfg.gap_fill_days_on_start > 0 { + let days = Some(self.cfg.gap_fill_days_on_start); + tokio::select! { + r = rid::fill_gaps(&self.rid, &self.pool, &self.stations, days, bangkok_now()) => { + if let Err(e) = r { tracing::error!("startup gap fill failed: {e:#}") } + } + _ = &mut shutdown => return Ok(()), + } + } + + let mut stats = Stats::default(); + let mut last_hourly_slot: Option = None; + loop { + let start = bangkok_now(); + self.write_status(&stats, true); + let outcome: CycleOutcome = + rid::run_cycle(&self.rid, &self.pool, &self.stations, start).await; + stats.total_runs += 1; + stats.last_run = Some(start); + if outcome.current { + stats.successful_runs += 1; + } else { + stats.failed_runs += 1; + } + stats.feed_current = Some(outcome.current); + stats.readings_saved = outcome.saved; + if outcome.latest.is_some() { + stats.latest_reading = outcome.latest.max(stats.latest_reading); + } + + let slot = floor_hour(bangkok_now()); + if last_hourly_slot != Some(slot) { + last_hourly_slot = Some(slot); + self.collect_hourly(bangkok_now()).await; + } + + let now = bangkok_now(); + let next = rid::next_poll_time( + now, + outcome.current, + self.cfg.poll_offset_minutes, + self.cfg.retry_minutes, + ); + stats.next_run = Some(next); + self.write_status(&stats, false); + tracing::info!( + "next RID poll at {}{}", + next.format("%H:%M"), + if outcome.current { + "" + } else { + " (current hour not published yet)" + } + ); + let wait = (next - now) + .to_std() + .unwrap_or_default() + .max(std::time::Duration::from_secs(30)); + tokio::select! { + _ = tokio::time::sleep(wait) => {} + _ = &mut shutdown => { + tracing::info!("shutdown signal received"); + return Ok(()); + } + } + } + } + + /// One cycle of every source; with `dry_run` nothing is written and the + /// RID fetch is compared against what the database already holds. + pub async fn once(&self, dry_run: bool) -> Result<()> { + let now = bangkok_now(); + if !dry_run { + db::ensure_schema(&self.pool).await?; + let o = rid::run_cycle(&self.rid, &self.pool, &self.stations, now).await; + println!( + "RID: reports {:?}, {} fetched, {} saved, latest {:?}, current hour in: {}{}", + o.dates, + o.fetched, + o.saved, + o.latest, + o.current, + o.error.map(|e| format!(", error: {e}")).unwrap_or_default() + ); + self.collect_hourly(now).await; + return Ok(()); + } + + let dates = rid::cycle_dates(&self.pool, now).await; + let (readings, error) = rid::fetch_dates(&self.rid, &dates).await; + let stored = db::count_stored_rid(&self.pool, &readings).await?; + let latest = readings.iter().map(|r| r.timestamp).max(); + let db_latest = db::latest_rid_timestamp(&self.pool).await?; + println!( + "RID (dry run): reports {dates:?}: {} readings fetched, {stored} already stored, {} would be added; latest fetched {latest:?}, latest stored {db_latest:?}{}", + readings.len(), + readings.len() as i64 - stored, + error.map(|e| format!(", error: {e}")).unwrap_or_default() + ); + if self.cfg.hii_enabled { + match self.hii.fetch_rain(self.cfg.hii_basin_code).await { + Ok(r) => println!("HII rainfall (dry run): {} Ping-basin stations", r.len()), + Err(e) => println!("HII rainfall (dry run): {e:#}"), + } + match self.hii.fetch_waterlevel(self.cfg.hii_basin_code).await { + Ok(r) => println!("HII water level (dry run): {} Ping-basin stations", r.len()), + Err(e) => println!("HII water level (dry run): {e:#}"), + } + } + if self.cfg.dams_enabled { + match self.dams.fetch_day(None).await { + Ok(r) => println!("RID dams (dry run): {} dams today", r.len()), + Err(e) => println!("RID dams (dry run): {e:#}"), + } + } + if self.cfg.rain_enabled { + match self.rain.fetch_forecast(5, 2).await { + Ok(r) => println!("Open-Meteo (dry run): {} hourly rows", r.len()), + Err(e) => println!("Open-Meteo (dry run): {e:#}"), + } + } + Ok(()) + } +} + +async fn shutdown_signal() { + #[cfg(unix)] + { + use tokio::signal::unix::{SignalKind, signal}; + let mut term = signal(SignalKind::terminate()).expect("SIGTERM handler"); + tokio::select! { + _ = tokio::signal::ctrl_c() => {} + _ = term.recv() => {} + } + } + #[cfg(not(unix))] + { + let _ = tokio::signal::ctrl_c().await; + } +} diff --git a/rust/collector/src/stations.rs b/rust/collector/src/stations.rs new file mode 100644 index 0000000..4bfa9cc --- /dev/null +++ b/rust/collector/src/stations.rs @@ -0,0 +1,84 @@ +//! RID station mapping: report column number (1..16) -> station metadata. +//! Same precedence as the Python scraper: the runtime file +//! (STATION_CONFIG_PATH, edited through the station CRUD API) if it exists, +//! else the defaults bundled in src/data/stations.json. + +use anyhow::{Context, Result}; +use serde::Deserialize; +use std::collections::BTreeMap; +use std::path::Path; + +const BUNDLED: &str = include_str!("../../../src/data/stations.json"); + +#[derive(Clone, Debug, Deserialize)] +pub struct StationInfo { + pub code: String, + pub thai_name: String, + pub english_name: String, + pub latitude: Option, + pub longitude: Option, + pub geohash: Option, +} + +#[derive(Clone, Debug)] +pub struct Stations(BTreeMap); + +impl Stations { + pub fn load(runtime_path: &Path) -> Result { + if runtime_path.exists() { + let text = std::fs::read_to_string(runtime_path) + .with_context(|| format!("reading {}", runtime_path.display()))?; + let stations = Self::parse(&text) + .with_context(|| format!("parsing {}", runtime_path.display()))?; + tracing::info!( + "loaded {} stations from {}", + stations.0.len(), + runtime_path.display() + ); + return Ok(stations); + } + let stations = Self::bundled(); + tracing::info!("loaded {} bundled default stations", stations.0.len()); + Ok(stations) + } + + pub fn bundled() -> Self { + Self::parse(BUNDLED).expect("bundled stations.json is valid") + } + + fn parse(text: &str) -> Result { + let raw: BTreeMap = serde_json::from_str(text)?; + let map = raw + .into_iter() + .map(|(k, v)| Ok((k.trim().parse::()?, v))) + .collect::>>()?; + Ok(Stations(map)) + } + + /// Metadata for report column `n`, with the Python fallback for unknown + /// columns (`P.{19+n}` / `Station n`). + pub fn get(&self, n: u32) -> StationInfo { + self.0.get(&n).cloned().unwrap_or_else(|| StationInfo { + code: format!("P.{}", 19 + n), + thai_name: format!("Station {n}"), + english_name: format!("Station {n}"), + latitude: None, + longitude: None, + geohash: None, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn bundled_mapping_has_the_16_rid_gauges() { + let s = Stations::bundled(); + assert_eq!(s.0.len(), 16); + assert_eq!(s.get(1).code, "P.20"); + assert_eq!(s.get(8).code, "P.1"); + assert_eq!(s.get(99).code, "P.118"); // Python fallback + } +} diff --git a/rust/collector/src/util.rs b/rust/collector/src/util.rs new file mode 100644 index 0000000..cd195f5 --- /dev/null +++ b/rust/collector/src/util.rs @@ -0,0 +1,86 @@ +//! Small shared helpers: Bangkok wall-clock time and lenient JSON numerics. + +use chrono::{Duration, NaiveDateTime, Timelike, Utc}; +use serde_json::Value; + +/// Asia/Bangkok wall-clock time (UTC+7, no DST), without an offset. Every +/// timestamp in the database is stored this way, independent of the +/// server's own time zone. +pub fn bangkok_now() -> NaiveDateTime { + (Utc::now() + Duration::hours(7)).naive_utc() +} + +pub fn floor_hour(t: NaiveDateTime) -> NaiveDateTime { + t.with_minute(0) + .and_then(|t| t.with_second(0)) + .and_then(|t| t.with_nanosecond(0)) + .expect("valid time") +} + +/// Upstream APIs send numbers as JSON numbers, numeric strings ("335.00", +/// "1,234.5"), placeholders ("", "-", "***") or null. Anything that is not a +/// finite number becomes None. +pub fn lenient_f64(v: Option<&Value>) -> Option { + let x = match v? { + Value::Number(n) => n.as_f64()?, + Value::String(s) => s.trim().replace(',', "").parse::().ok()?, + _ => return None, + }; + x.is_finite().then_some(x) +} + +pub fn lenient_i64(v: Option<&Value>) -> Option { + match v? { + Value::Number(n) => n.as_i64().or_else(|| n.as_f64().map(|f| f as i64)), + Value::String(s) => s.trim().parse::().ok().map(|f| f as i64), + _ => None, + } +} + +/// A JSON scalar as a string (ids arrive as numbers or strings). +pub fn scalar_string(v: Option<&Value>) -> Option { + match v? { + Value::String(s) if !s.is_empty() => Some(s.clone()), + Value::Number(n) => Some(n.to_string()), + _ => None, + } +} + +/// Python truthiness of a JSON value (`bool(x)`). +pub fn truthy(v: Option<&Value>) -> bool { + match v { + None | Some(Value::Null) => false, + Some(Value::Bool(b)) => *b, + Some(Value::Number(n)) => n.as_f64() != Some(0.0), + Some(Value::String(s)) => !s.is_empty(), + Some(Value::Array(a)) => !a.is_empty(), + Some(Value::Object(o)) => !o.is_empty(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn lenient_numbers() { + assert_eq!(lenient_f64(Some(&json!("335.00"))), Some(335.0)); + assert_eq!(lenient_f64(Some(&json!(" 1,234.5 "))), Some(1234.5)); + assert_eq!(lenient_f64(Some(&json!(2.5))), Some(2.5)); + for junk in [json!(""), json!("-"), json!("***"), json!(null), json!([1])] { + assert_eq!(lenient_f64(Some(&junk)), None, "{junk}"); + } + assert_eq!(lenient_i64(Some(&json!("2"))), Some(2)); + assert_eq!(lenient_i64(Some(&json!(3))), Some(3)); + } + + #[test] + fn python_truthiness() { + assert!(truthy(Some(&json!(true)))); + assert!(truthy(Some(&json!(1)))); + assert!(!truthy(Some(&json!(0)))); + assert!(!truthy(Some(&json!("")))); + assert!(!truthy(None)); + } +} diff --git a/scripts/collector_parity.py b/scripts/collector_parity.py new file mode 100644 index 0000000..771ec4b --- /dev/null +++ b/scripts/collector_parity.py @@ -0,0 +1,150 @@ +"""Live parity check: Python collectors vs the Rust ping-collector. + +Runs one collection of every source with the Python code into PY_DB and with +`ping-collector` into RS_DB (both should start empty), back to back against the +real upstream APIs, then diffs all nine collector tables row by row (ids and +created/updated timestamps ignored). Exit status 1 on any difference. + + uv run python scripts/collector_parity.py \ + --py-db postgresql://postgres:ping@localhost:55432/py \ + --rs-db postgresql://postgres:ping@localhost:55432/rs \ + --collector rust/target/debug/ping-collector + +Upstream snapshots can move between the two runs (a new HII hour, a revised +RID value); rerun before treating a small difference as a bug. +""" + +import argparse +import os +import subprocess +import sys +from pathlib import Path + +REPO = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO)) + +TABLES = { + "stations": ( + "id", + "id, station_code, thai_name, english_name, latitude, longitude, geohash", + ), + "water_measurements": ( + "timestamp, station_id", + "timestamp, station_id, water_level, discharge, discharge_percent, status", + ), + "hii_rain_stations": ( + "id", + "id, oldcode, name_th, name_en, latitude, longitude, sub_basin_id, agency", + ), + "hii_rainfall": ( + "station_id, timestamp", + "station_id, timestamp, rain_1h, rain_24h", + ), + "hii_wl_stations": ( + "id", + "id, oldcode, rid_code, name_th, name_en, latitude, longitude, agency, " + "river_name, offset_msl, ground_level_msl, min_bank_msl, " + "critical_level_msl, critical_level_m, qmax, is_key_station", + ), + "hii_waterlevel": ( + "station_id, timestamp", + "station_id, timestamp, wl_msl, wl_m, discharge, flow_rate, " + "storage_percent, situation_level, diff_wl_bank", + ), + "rid_dams": ( + "dam_id", + "dam_id, region, name_th, latitude, longitude, capacity_max_mcm, " + "capacity_normal_mcm", + ), + "rid_reservoir_daily": ( + "dam_id, date", + "dam_id, date, storage_mcm, storage_pct, inflow_mcm, outflow_mcm, level_msl", + ), + "openmeteo_rain": ( + "timestamp", + "timestamp, chiang_dao, mae_taeng, mae_ngat, mae_rim, chiang_mai, " + "catchment_mean", + ), +} + + +def run_python(db_url: str, gap_days: int): + os.environ.update( + DB_TYPE="postgresql", POSTGRES_CONNECTION_STRING=db_url, LOG_LEVEL="WARNING" + ) + from sqlalchemy import create_engine + + from src.config import Config + from src.hii_collector import HiiCollector + from src.ml import rain + from src.rid_reservoir import RidReservoirCollector + from src.water_scraper_v3 import EnhancedWaterMonitorScraper + + cfg = Config.get_database_config() + scraper = EnhancedWaterMonitorScraper(cfg) + scraper.fill_data_gaps(gap_days) + scraper.run_scraping_cycle() + HiiCollector(cfg).run_cycle() + RidReservoirCollector(cfg).run_cycle() + rain.save_to_db(rain.fetch_forecast(), create_engine(db_url), "postgresql") + + +def run_rust(db_url: str, collector: str, gap_days: int): + env = {**os.environ, "POSTGRES_CONNECTION_STRING": db_url, "DB_TYPE": "postgresql"} + for args in (["fill-gaps", str(gap_days)], ["once"]): + subprocess.run([collector, *args], env=env, check=True) + + +def table_rows(db_url: str, table: str, key: str, cols: str) -> dict: + import psycopg2 + + with psycopg2.connect(db_url) as conn: + cur = conn.cursor() + cur.execute(f"SELECT {cols} FROM {table} ORDER BY {key}") + nkey = len(key.split(",")) + return {row[:nkey]: row for row in cur.fetchall()} + + +def diff(py_db: str, rs_db: str) -> bool: + identical = True + for table, (key, cols) in TABLES.items(): + a = table_rows(py_db, table, key, cols) + b = table_rows(rs_db, table, key, cols) + only_py, only_rs = sorted(set(a) - set(b)), sorted(set(b) - set(a)) + differ = sorted(k for k in set(a) & set(b) if a[k] != b[k]) + same = not (only_py or only_rs or differ) + identical &= same + print( + f"{table:<20} py {len(a):>6} rs {len(b):>6} only-py {len(only_py):>4} " + f"only-rs {len(only_rs):>4} differing {len(differ):>4} " + f"{'identical' if same else 'DIFFERENT'}" + ) + for k in only_py[:3]: + print(" only py:", a[k]) + for k in only_rs[:3]: + print(" only rs:", b[k]) + for k in differ[:3]: + print(" py:", a[k], "\n rs:", b[k]) + return identical + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--py-db", required=True) + parser.add_argument("--rs-db", required=True) + parser.add_argument( + "--collector", default=str(REPO / "rust/target/debug/ping-collector") + ) + parser.add_argument("--gap-days", type=int, default=2) + parser.add_argument("--diff-only", action="store_true") + args = parser.parse_args() + if not args.diff_only: + run_python(args.py_db, args.gap_days) + run_rust(args.rs_db, args.collector, args.gap_days) + ok = diff(args.py_db, args.rs_db) + print("ALL IDENTICAL" if ok else "differences found") + sys.exit(0 if ok else 1) + + +if __name__ == "__main__": + main() diff --git a/scripts/install_collector.sh b/scripts/install_collector.sh new file mode 100755 index 0000000..b4ca649 --- /dev/null +++ b/scripts/install_collector.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Install the Rust collector (rust/collector, `ping-collector`) and switch +# data collection over to it, or back. +# +# sudo bash scripts/install_collector.sh # install binary + unit, dry-run check +# sudo bash scripts/install_collector.sh --cutover # install, then Rust collects: +# # COLLECTION_ENABLED=false, start +# # ping-collector, restart water-monitor +# sudo bash scripts/install_collector.sh --rollback # Python collects again +# +# The binary comes from BIN=/path/to/ping-collector (e.g. the "ping-collector- +# linux-x86_64" CI artifact) or from rust/target/release/ping-collector, built +# beforehand as a normal user: +# cargo build --release --locked --manifest-path rust/Cargo.toml +# (needs rustup, pkg-config and libssl-dev). +set -euo pipefail + +APP_DIR="${APP_DIR:-/opt/thailand-water-monitor}" +ENV_FILE="${APP_DIR}/.env" +STATUS_FILE="${APP_DIR}/collector-status.json" +REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +MODE="${1:-install}" + +log() { printf '\033[1;32m==>\033[0m %s\n' "$*"; } +die() { printf '\033[1;31m[error]\033[0m %s\n' "$*" >&2; exit 1; } + +[ "$(id -u)" -eq 0 ] || die "run as root (sudo)" +[ -f "$ENV_FILE" ] || die "$ENV_FILE not found" + +set_env() { + local key="$1" value="$2" + if grep -q "^${key}=" "$ENV_FILE"; then + sed -i "s|^${key}=.*|${key}=${value}|" "$ENV_FILE" + else + printf '%s=%s\n' "$key" "$value" >>"$ENV_FILE" + fi +} + +if [ "$MODE" = "--rollback" ]; then + log "Stopping ping-collector; the Python app collects again" + systemctl disable --now ping-collector.service 2>/dev/null || true + set_env COLLECTION_ENABLED true + systemctl restart water-monitor.service + log "Done. Check: journalctl -u water-monitor -n 50" + exit 0 +fi +case "$MODE" in install | --cutover) ;; *) die "unknown option: $MODE" ;; esac + +BIN="${BIN:-${REPO_DIR}/rust/target/release/ping-collector}" +[ -x "$BIN" ] || die "no binary at $BIN: build it (see header) or pass BIN=/path/to/ping-collector" + +log "Installing $BIN -> ${APP_DIR}/bin/ping-collector ($("$BIN" --version))" +install -D -m 0755 -o root -g root "$BIN" "${APP_DIR}/bin/ping-collector" +install -m 0644 "${REPO_DIR}/scripts/ping-collector.service" /etc/systemd/system/ping-collector.service +systemctl daemon-reload + +# Both services read and write /scraping/status through this file (inside +# the app dir: the units use PrivateTmp). +set_env SCRAPING_STATUS_FILE "$STATUS_FILE" +touch "$STATUS_FILE" +chown "$(stat -c %U:%G "$APP_DIR")" "$STATUS_FILE" + +log "Dry run against the configured database (writes nothing)" +(cd "$APP_DIR" && "${APP_DIR}/bin/ping-collector" --env-file "$ENV_FILE" once --dry-run) + +if [ "$MODE" = "--cutover" ]; then + log "Cutover: ping-collector collects, the Python app forecasts and notifies" + set_env COLLECTION_ENABLED false + systemctl enable --now ping-collector.service + systemctl restart water-monitor.service + sleep 20 + systemctl --no-pager --lines=0 status ping-collector.service || true + journalctl -u ping-collector -n 20 --no-pager || true + log "Rollback if needed: sudo bash scripts/install_collector.sh --rollback" +else + log "Installed, not started. Switch over with: sudo bash scripts/install_collector.sh --cutover" +fi diff --git a/scripts/ping-collector.service b/scripts/ping-collector.service new file mode 100644 index 0000000..6c79fd4 --- /dev/null +++ b/scripts/ping-collector.service @@ -0,0 +1,35 @@ +[Unit] +Description=Ping River Monitor data collector (Rust: RID, HII, dams, Open-Meteo) +Documentation=https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/src/branch/master/rust +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=water-monitor +Group=water-monitor +WorkingDirectory=/opt/thailand-water-monitor +ExecStart=/opt/thailand-water-monitor/bin/ping-collector run +Restart=always +RestartSec=30 +TimeoutStopSec=20 + +# Same .env as water-monitor.service (POSTGRES_*, RID_*, GAP_FILL_*, +# SCRAPING_STATUS_FILE). The Python app must run with COLLECTION_ENABLED=false. +EnvironmentFile=/opt/thailand-water-monitor/.env + +# Security settings +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=/opt/thailand-water-monitor +CapabilityBoundingSet= +MemoryMax=256M + +StandardOutput=journal +StandardError=journal +SyslogIdentifier=ping-collector + +[Install] +WantedBy=multi-user.target diff --git a/src/config.py b/src/config.py index b65cd85..e1191ad 100644 --- a/src/config.py +++ b/src/config.py @@ -142,8 +142,20 @@ class Config: # On startup the collection leader re-fetches RID reports with holes from # the last N days (0 = off). Older history: `run.py --fill-gaps all`. GAP_FILL_DAYS_ON_START = int(os.getenv("GAP_FILL_DAYS_ON_START", "14")) - # Leader -> other workers hand-off for /scraping/status (default: temp dir) + # Leader -> other workers hand-off for /scraping/status (default: temp dir). + # With ping-collector, point both services at the same file inside the app + # dir (the systemd units use PrivateTmp). SCRAPING_STATUS_FILE = os.getenv("SCRAPING_STATUS_FILE", "") + # false: data collection runs in the Rust ping-collector service + # (rust/collector); this app then only serves, forecasts and notifies, + # re-forecasting when the newest RID reading advances (polled every + # FORECAST_POLL_SECONDS) and at least once per clock hour. + COLLECTION_ENABLED = os.getenv("COLLECTION_ENABLED", "true").lower() in ( + "1", + "true", + "yes", + ) + FORECAST_POLL_SECONDS = int(os.getenv("FORECAST_POLL_SECONDS", "60")) # Logging settings LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO") @@ -293,6 +305,7 @@ class Config: "RID_POLL_OFFSET_MINUTES": cls.RID_POLL_OFFSET_MINUTES, "RID_RETRY_MINUTES": cls.RID_RETRY_MINUTES, "GAP_FILL_DAYS_ON_START": cls.GAP_FILL_DAYS_ON_START, + "COLLECTION_ENABLED": cls.COLLECTION_ENABLED, "LOG_LEVEL": cls.LOG_LEVEL, "LOG_FILE": cls.LOG_FILE, "DATA_RETENTION_DAYS": cls.DATA_RETENTION_DAYS, diff --git a/src/web_api.py b/src/web_api.py index fc09158..0fd1848 100644 --- a/src/web_api.py +++ b/src/web_api.py @@ -218,8 +218,15 @@ async def lifespan(app: FastAPI): app_state["notify"] = None if app_state["leader_lock"]: app_state["notify"] = _init_notifications() - app_state["scraping_task"] = asyncio.create_task(background_scraping_task()) - logger.info("This worker is the background-collection leader") + if Config.COLLECTION_ENABLED: + app_state["scraping_task"] = asyncio.create_task(background_scraping_task()) + logger.info("This worker is the background-collection leader") + else: + app_state["scraping_task"] = asyncio.create_task(forecast_notify_task()) + logger.info( + "COLLECTION_ENABLED=false: data collection runs in ping-collector " + "(rust/collector); this worker only refreshes forecasts and notifies" + ) else: logger.info( "Another worker holds collection leadership; " @@ -587,6 +594,58 @@ async def background_scraping_task(): await asyncio.sleep(60) # Wait a minute before retrying +def _latest_rid_reading(): + """Newest RID reading in the database (collected by another process).""" + from sqlalchemy import text + + engine = app_state["scraper"].db_adapter.engine + with engine.connect() as conn: + return conn.execute( + text("SELECT MAX(timestamp) FROM water_measurements") + ).scalar() + + +async def forecast_notify_task(): + """Leader loop when COLLECTION_ENABLED=false (ping-collector collects). + + Polls the newest RID reading every FORECAST_POLL_SECONDS; when it + advances, and once per clock hour regardless (so a stopped feed is still + noticed and reported), refreshes the forecast cache/history and pushes + ntfy transitions. + """ + last_hourly_slot = None + newest_reading = None + while True: + try: + try: + latest = await asyncio.to_thread(_latest_rid_reading) + except Exception as e: + logger.warning(f"Could not read the newest RID reading: {e}") + latest = None + new_readings = latest is not None and ( + newest_reading is None or latest > newest_reading + ) + if new_readings: + newest_reading = latest + + slot = datetime.now().replace(minute=0, second=0, microsecond=0) + hourly_due = slot != last_hourly_slot + if hourly_due: + last_hourly_slot = slot + + if new_readings or hourly_due: + await _precompute_forecasts() + await _notify_transitions() + + await asyncio.sleep(Config.FORECAST_POLL_SECONDS) + except asyncio.CancelledError: + logger.info("Forecast/notify task cancelled") + break + except Exception as e: + logger.error(f"Error in forecast/notify task: {e}") + await asyncio.sleep(60) + + # Umami server-side API tracking (fire-and-forget; never blocks a response) _UMAMI_TRACK_PREFIXES = ( @@ -1658,6 +1717,11 @@ async def trigger_scraping(background_tasks: BackgroundTasks): """Trigger manual data scraping""" increment_counter("api_requests", labels={"endpoint": "scrape_trigger"}) + if not Config.COLLECTION_ENABLED: + raise HTTPException( + status_code=409, + detail="Collection runs in ping-collector (COLLECTION_ENABLED=false)", + ) if app_state["is_scraping"]: raise HTTPException(status_code=409, detail="Scraping already in progress") @@ -1696,7 +1760,8 @@ async def get_scraping_status(): increment_counter("api_requests", labels={"endpoint": "scraping_status"}) # Only the leader worker collects; the others report what it published. - if app_state.get("leader_lock"): + # With COLLECTION_ENABLED=false ping-collector writes the same file. + if Config.COLLECTION_ENABLED and app_state.get("leader_lock"): stats = dict(app_state["scraping_stats"], is_running=app_state["is_scraping"]) else: stats = _read_scraping_status() or {"is_running": False} diff --git a/tests/test_collection_disabled.py b/tests/test_collection_disabled.py new file mode 100644 index 0000000..b785284 --- /dev/null +++ b/tests/test_collection_disabled.py @@ -0,0 +1,96 @@ +"""COLLECTION_ENABLED=false: the Rust ping-collector collects, this app only +re-forecasts and notifies when the database's newest RID reading advances +(and once per clock hour), and /scraping/status reports the collector's file.""" + +import asyncio +import datetime +import json + +from src import web_api +from src.config import Config + + +def test_forecasts_follow_new_readings_and_the_clock_hour(monkeypatch): + clock = {"now": datetime.datetime(2026, 10, 17, 21, 30)} + + class FixedClock(datetime.datetime): + @classmethod + def now(cls, tz=None): + return clock["now"] + + # newest reading per poll: unchanged, unchanged, advances, next hour + latest = iter( + [ + datetime.datetime(2026, 10, 17, 20), + datetime.datetime(2026, 10, 17, 20), + datetime.datetime(2026, 10, 17, 21), + datetime.datetime(2026, 10, 17, 21), + ] + ) + calls = {"precompute": 0, "notify": 0} + sleeps = [] + + async def precompute(): + calls["precompute"] += 1 + + async def notify(): + calls["notify"] += 1 + + async def fake_sleep(seconds): + sleeps.append(seconds) + if len(sleeps) == 3: # the last poll happens in the next clock hour + clock["now"] = datetime.datetime(2026, 10, 17, 22, 1) + if len(sleeps) == 4: + raise asyncio.CancelledError + + monkeypatch.setattr(web_api, "datetime", FixedClock) + monkeypatch.setattr(web_api.asyncio, "sleep", fake_sleep) + monkeypatch.setattr(web_api, "_latest_rid_reading", lambda: next(latest)) + monkeypatch.setattr(web_api, "_precompute_forecasts", precompute) + monkeypatch.setattr(web_api, "_notify_transitions", notify) + monkeypatch.setattr(Config, "FORECAST_POLL_SECONDS", 60) + + asyncio.run(web_api.forecast_notify_task()) + + # first poll (new + hourly), third (new reading), fourth (new clock hour) + assert calls == {"precompute": 3, "notify": 3} + assert sleeps == [60, 60, 60, 60] + + +def test_status_comes_from_the_collector_file(tmp_path, monkeypatch): + status = tmp_path / "collector-status.json" + status.write_text( + json.dumps( + { + "is_running": False, + "total_runs": 4, + "successful_runs": 3, + "failed_runs": 1, + "last_run": "2026-10-17T22:05:01", + "next_run": "2026-10-17T23:05:00", + "feed_current": True, + "latest_reading": "2026-10-17T22:00:00", + "readings_saved": 352, + "collector": "ping-collector 0.1.0", + } + ) + ) + monkeypatch.setattr(Config, "COLLECTION_ENABLED", False) + monkeypatch.setattr(Config, "SCRAPING_STATUS_FILE", str(status)) + monkeypatch.setitem(web_api.app_state, "leader_lock", object()) + + resp = asyncio.run(web_api.get_scraping_status()) + + assert resp.total_runs == 4 + assert resp.feed_current is True + assert resp.latest_reading == datetime.datetime(2026, 10, 17, 22) + + +def test_manual_trigger_is_refused_when_collection_runs_elsewhere(monkeypatch): + import pytest + from fastapi import BackgroundTasks, HTTPException + + monkeypatch.setattr(Config, "COLLECTION_ENABLED", False) + with pytest.raises(HTTPException) as err: + asyncio.run(web_api.trigger_scraping(BackgroundTasks())) + assert err.value.status_code == 409