feat(collector): forecast and ntfy in ping-collector; ThaiWater keeps alerts current at night
CI / Format & lint (push) Successful in 22s
CI / Test suite (push) Successful in 59s
Docs / Validate documentation (push) Successful in 25s
Rust / fmt, clippy, test, build (push) Successful in 10m2s
Security / Dependency vulnerabilities (push) Successful in 2m11s
Security / Static analysis (push) Successful in 22s
Security / License report (push) Successful in 1m37s

Phase 2b/2c of docs/RUST_REWRITE.md. With FORECAST_IN_COLLECTOR=true (shared
.env) ping-collector, after each cycle that brought newer RID or HII readings
and at least hourly: loads the last 336 h (RID + HII mirror fill), fetches the
rain forecast, runs every exported model (rust/forecast), stores
forecast_history and forecast_current, and sends the ntfy notifications.
The Python API then serves forecast_current as /forecast (computing in-process
only when it is older than FORECAST_MAX_AGE_SECONDS) and neither forecasts nor
notifies.

ntfy (collector/src/notify.rs): a port of src/notify.py with the same topics,
texts, hysteresis, capacity guard and notification_state keys, checked word
for word on 751 messages generated by Python (testdata/gen_notify_cases.py).
The change: for stations whose ThaiWater twin is the same sensor, a reading
newer than RID's drives the level alerts (marked as ThaiWater telemetry), and
the feed counts as stale only when neither source has a reading for 3 h. RID
alone lagging is reported once after RID_STALE_ALERT_HOURS (18). This ends the
nightly "gauge feed stale" / "recovered" pairs caused by RID's late reports.

Commands: `forecast [--json] [--save] [--now] [--rain-file]`,
`notify --dry-run`. scripts/forecast_parity.py compares Python and the
collector on the live database (same moment, same rain); locally 48/48 rows
identical. install_collector.sh: models export + that check on every install,
--cutover-forecast (refuses on a failed check) and --rollback-forecast.

Trees are packed into 24-byte nodes: 70 MB peak with all 15 models (was 140).
Forecast values are written to NUMERIC as shortest round-trip text, so they
round as psycopg2's do. DB test schemas get a counter (parallel tests collided
on Windows' coarse clock).
This commit is contained in:
grabowski committed 2026-10-18 14:43:39 +07:00
1 parent 5c4f464383
commit 6ae67dec63
27 files changed
+2656 -118

No files matched your search

+191 -2
View File
@@ -29,10 +29,13 @@ impl TestDb {
eprintln!("TEST_DATABASE_URL not set: skipping database test");
return None;
};
// the counter: tests run in parallel and the clock is coarse on Windows
static N: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0);
let schema = format!(
"t_{}_{}",
"t_{}_{}_{}",
std::process::id(),
chrono::Utc::now().timestamp_nanos_opt().unwrap()
chrono::Utc::now().timestamp_nanos_opt().unwrap(),
N.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
);
let admin = PgPool::connect(&url).await.expect("connect");
sqlx::query(&format!("CREATE SCHEMA {schema}"))
@@ -330,3 +333,189 @@ async fn hii_dams_and_rain_round_trip() {
assert_eq!((mean, missing), (1.11, None));
db.drop().await;
}
fn forecast_row(code: &str, h: u32, p: f64) -> ping_forecast::Forecast {
ping_forecast::Forecast {
station_code: code.into(),
horizon_hours: h,
p_warning: p,
p_danger: p / 2.0,
predicted_max_level: 3.123456,
current_level: 1.23,
as_of: t(2026, 10, 18, 13, 0),
model_version: "hgb-v3+test".into(),
trained_at: Some("2026-10-01T03:30:00".into()),
source: "model".into(),
threshold_warning: 3.7,
threshold_danger: 4.2,
stages: None,
}
}
#[tokio::test]
async fn forecast_outputs_round_trip() {
let Some(db) = TestDb::new().await else {
return;
};
// 0.123455 + 1 ulp: Python's repr gives 17 digits and NUMERIC(7,5)
// rounds that up; a float8 -> numeric cast (15 digits) would round down
let tricky = f64::from_bits(0.123455f64.to_bits() + 1);
let rows = [
forecast_row("P.1", 24, tricky),
forecast_row("P.1", 24, 0.2),
forecast_row("P.20", 6, 1e-7),
];
assert_eq!(db::save_forecast_history(&db.pool, &rows).await.unwrap(), 2);
db::save_forecast_history(&db.pool, &rows[..1])
.await
.unwrap();
let got: Vec<(String, i32, f64, f64, f64)> = sqlx::query_as(
"SELECT station_code, horizon_hours, p_warning::float8, predicted_max_level::float8,
current_level::float8 FROM forecast_history ORDER BY station_code",
)
.fetch_all(&db.pool)
.await
.unwrap();
assert_eq!(
got,
vec![
("P.1".into(), 24, 0.12346, 3.123, 1.23),
("P.20".into(), 6, 0.0, 3.123, 1.23)
]
);
let now = t(2026, 10, 18, 13, 7);
db::save_forecast_current(&db.pool, now, &rows[1..])
.await
.unwrap();
db::save_forecast_current(&db.pool, now, &rows[..1])
.await
.unwrap();
let (at, versions, payload): (NaiveDateTime, String, String) =
sqlx::query_as("SELECT computed_at, model_versions, payload FROM forecast_current")
.fetch_one(&db.pool)
.await
.unwrap();
assert_eq!((at, versions.as_str()), (now, "hgb-v3+test"));
let v: serde_json::Value = serde_json::from_str(&payload).unwrap();
assert_eq!(v[0]["as_of"], "2026-10-18T13:00:00");
assert_eq!(v[0]["p_warning"].as_f64(), Some(tricky));
assert!(v[0].get("stages").is_none());
db.drop().await;
}
#[tokio::test]
async fn notification_state_round_trip() {
let Some(db) = TestDb::new().await else {
return;
};
let now = t(2026, 10, 18, 2, 0);
db::set_notification_state(&db.pool, "level:P.1", "warning", Some(3.7512), now)
.await
.unwrap();
db::set_notification_state(&db.pool, "feed", "stale", None, now)
.await
.unwrap();
db::set_notification_state(&db.pool, "level:P.1", "clear", Some(3.5), now)
.await
.unwrap();
let states = db::notification_states(&db.pool).await.unwrap();
assert_eq!(states.len(), 2);
assert_eq!(states["level:P.1"], "clear");
assert_eq!(states["feed"], "stale");
db.drop().await;
}
#[tokio::test]
async fn forecast_inputs_load_like_python() {
let Some(db) = TestDb::new().await else {
return;
};
let stations = Stations::bundled();
let r = |h: u32, level: f64| Reading {
timestamp: t(2026, 10, 18, h, 0),
station_num: 8, // P.1
water_level: level,
discharge: Some(51.25),
discharge_percent: Some(40.5),
};
db::save_rid(&db.pool, &[r(10, 1.1), r(11, 1.2), r(12, 1.3)], &stations)
.await
.unwrap();
let rows = db::load_rid_window(&db.pool, t(2026, 10, 18, 11, 0), t(2026, 10, 18, 12, 30))
.await
.unwrap();
assert_eq!(rows.len(), 2);
assert_eq!(
(rows[0].station_code.as_str(), rows[0].water_level),
("P.1", 1.2)
);
assert_eq!(rows[1].discharge, 51.25);
let latest = db::latest_rid_readings(&db.pool).await.unwrap();
assert_eq!(latest.len(), 1);
assert_eq!(
(
latest[0].timestamp,
latest[0].water_level,
latest[0].discharge_percent
),
(t(2026, 10, 18, 12, 0), Some(1.3), Some(40.5))
);
let wl = |minute: u32, v: f64| WaterLevelRecord {
station_id: 5,
oldcode: None,
rid_code: Some("P.1".into()),
name_th: None,
name_en: None,
latitude: None,
longitude: None,
agency: None,
river_name: None,
offset_msl: None,
ground_level_msl: None,
min_bank_msl: None,
critical_level_msl: None,
critical_level_m: None,
qmax: None,
is_key_station: true,
timestamp: t(2026, 10, 18, 13, minute),
wl_msl: Some(v),
wl_m: None,
discharge: Some(52.0),
flow_rate: None,
storage_percent: None,
situation_level: None,
diff_wl_bank: None,
};
db::save_hii_waterlevel(
&db.pool,
&[wl(0, 301.4), wl(40, 301.45)],
t(2026, 10, 18, 13, 45),
)
.await
.unwrap();
let codes = vec!["P.1".to_string(), "P.103".to_string()];
let hii = db::load_hii_window(
&db.pool,
&codes,
t(2026, 10, 18, 0, 0),
t(2026, 10, 18, 14, 0),
)
.await
.unwrap();
assert_eq!(hii.len(), 2);
assert_eq!(ping_forecast::hii::prepare(&hii)[0].wl_msl, 301.45);
assert_eq!(
db::latest_hii_timestamp(&db.pool, &codes).await.unwrap(),
Some(t(2026, 10, 18, 13, 40))
);
assert!(
db::load_dam_days(&db.pool, "200103", d(2026, 10, 1))
.await
.unwrap()
.is_empty()
);
db.drop().await;
}