feat(collector): forecast and ntfy in ping-collector; ThaiWater keeps alerts current at night
CI / Format & lint (push) Successful in 22s
CI / Test suite (push) Successful in 59s
Docs / Validate documentation (push) Successful in 25s
Rust / fmt, clippy, test, build (push) Successful in 10m2s
Security / Dependency vulnerabilities (push) Successful in 2m11s
Security / Static analysis (push) Successful in 22s
Security / License report (push) Successful in 1m37s
CI / Format & lint (push) Successful in 22s
CI / Test suite (push) Successful in 59s
Docs / Validate documentation (push) Successful in 25s
Rust / fmt, clippy, test, build (push) Successful in 10m2s
Security / Dependency vulnerabilities (push) Successful in 2m11s
Security / Static analysis (push) Successful in 22s
Security / License report (push) Successful in 1m37s
Phase 2b/2c of docs/RUST_REWRITE.md. With FORECAST_IN_COLLECTOR=true (shared .env) ping-collector, after each cycle that brought newer RID or HII readings and at least hourly: loads the last 336 h (RID + HII mirror fill), fetches the rain forecast, runs every exported model (rust/forecast), stores forecast_history and forecast_current, and sends the ntfy notifications. The Python API then serves forecast_current as /forecast (computing in-process only when it is older than FORECAST_MAX_AGE_SECONDS) and neither forecasts nor notifies. ntfy (collector/src/notify.rs): a port of src/notify.py with the same topics, texts, hysteresis, capacity guard and notification_state keys, checked word for word on 751 messages generated by Python (testdata/gen_notify_cases.py). The change: for stations whose ThaiWater twin is the same sensor, a reading newer than RID's drives the level alerts (marked as ThaiWater telemetry), and the feed counts as stale only when neither source has a reading for 3 h. RID alone lagging is reported once after RID_STALE_ALERT_HOURS (18). This ends the nightly "gauge feed stale" / "recovered" pairs caused by RID's late reports. Commands: `forecast [--json] [--save] [--now] [--rain-file]`, `notify --dry-run`. scripts/forecast_parity.py compares Python and the collector on the live database (same moment, same rain); locally 48/48 rows identical. install_collector.sh: models export + that check on every install, --cutover-forecast (refuses on a failed check) and --rollback-forecast. Trees are packed into 24-byte nodes: 70 MB peak with all 15 models (was 140). Forecast values are written to NUMERIC as shortest round-trip text, so they round as psycopg2's do. DB test schemas get a counter (parallel tests collided on Windows' coarse clock).
This commit is contained in:
1 parent
5c4f464383
commit
6ae67dec63
27 files changed
+2656
-118
No files matched your search
@@ -29,10 +29,13 @@ impl TestDb {
|
||||
eprintln!("TEST_DATABASE_URL not set: skipping database test");
|
||||
return None;
|
||||
};
|
||||
// the counter: tests run in parallel and the clock is coarse on Windows
|
||||
static N: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0);
|
||||
let schema = format!(
|
||||
"t_{}_{}",
|
||||
"t_{}_{}_{}",
|
||||
std::process::id(),
|
||||
chrono::Utc::now().timestamp_nanos_opt().unwrap()
|
||||
chrono::Utc::now().timestamp_nanos_opt().unwrap(),
|
||||
N.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
|
||||
);
|
||||
let admin = PgPool::connect(&url).await.expect("connect");
|
||||
sqlx::query(&format!("CREATE SCHEMA {schema}"))
|
||||
@@ -330,3 +333,189 @@ async fn hii_dams_and_rain_round_trip() {
|
||||
assert_eq!((mean, missing), (1.11, None));
|
||||
db.drop().await;
|
||||
}
|
||||
|
||||
fn forecast_row(code: &str, h: u32, p: f64) -> ping_forecast::Forecast {
|
||||
ping_forecast::Forecast {
|
||||
station_code: code.into(),
|
||||
horizon_hours: h,
|
||||
p_warning: p,
|
||||
p_danger: p / 2.0,
|
||||
predicted_max_level: 3.123456,
|
||||
current_level: 1.23,
|
||||
as_of: t(2026, 10, 18, 13, 0),
|
||||
model_version: "hgb-v3+test".into(),
|
||||
trained_at: Some("2026-10-01T03:30:00".into()),
|
||||
source: "model".into(),
|
||||
threshold_warning: 3.7,
|
||||
threshold_danger: 4.2,
|
||||
stages: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn forecast_outputs_round_trip() {
|
||||
let Some(db) = TestDb::new().await else {
|
||||
return;
|
||||
};
|
||||
// 0.123455 + 1 ulp: Python's repr gives 17 digits and NUMERIC(7,5)
|
||||
// rounds that up; a float8 -> numeric cast (15 digits) would round down
|
||||
let tricky = f64::from_bits(0.123455f64.to_bits() + 1);
|
||||
let rows = [
|
||||
forecast_row("P.1", 24, tricky),
|
||||
forecast_row("P.1", 24, 0.2),
|
||||
forecast_row("P.20", 6, 1e-7),
|
||||
];
|
||||
assert_eq!(db::save_forecast_history(&db.pool, &rows).await.unwrap(), 2);
|
||||
db::save_forecast_history(&db.pool, &rows[..1])
|
||||
.await
|
||||
.unwrap();
|
||||
let got: Vec<(String, i32, f64, f64, f64)> = sqlx::query_as(
|
||||
"SELECT station_code, horizon_hours, p_warning::float8, predicted_max_level::float8,
|
||||
current_level::float8 FROM forecast_history ORDER BY station_code",
|
||||
)
|
||||
.fetch_all(&db.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
got,
|
||||
vec![
|
||||
("P.1".into(), 24, 0.12346, 3.123, 1.23),
|
||||
("P.20".into(), 6, 0.0, 3.123, 1.23)
|
||||
]
|
||||
);
|
||||
|
||||
let now = t(2026, 10, 18, 13, 7);
|
||||
db::save_forecast_current(&db.pool, now, &rows[1..])
|
||||
.await
|
||||
.unwrap();
|
||||
db::save_forecast_current(&db.pool, now, &rows[..1])
|
||||
.await
|
||||
.unwrap();
|
||||
let (at, versions, payload): (NaiveDateTime, String, String) =
|
||||
sqlx::query_as("SELECT computed_at, model_versions, payload FROM forecast_current")
|
||||
.fetch_one(&db.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!((at, versions.as_str()), (now, "hgb-v3+test"));
|
||||
let v: serde_json::Value = serde_json::from_str(&payload).unwrap();
|
||||
assert_eq!(v[0]["as_of"], "2026-10-18T13:00:00");
|
||||
assert_eq!(v[0]["p_warning"].as_f64(), Some(tricky));
|
||||
assert!(v[0].get("stages").is_none());
|
||||
db.drop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn notification_state_round_trip() {
|
||||
let Some(db) = TestDb::new().await else {
|
||||
return;
|
||||
};
|
||||
let now = t(2026, 10, 18, 2, 0);
|
||||
db::set_notification_state(&db.pool, "level:P.1", "warning", Some(3.7512), now)
|
||||
.await
|
||||
.unwrap();
|
||||
db::set_notification_state(&db.pool, "feed", "stale", None, now)
|
||||
.await
|
||||
.unwrap();
|
||||
db::set_notification_state(&db.pool, "level:P.1", "clear", Some(3.5), now)
|
||||
.await
|
||||
.unwrap();
|
||||
let states = db::notification_states(&db.pool).await.unwrap();
|
||||
assert_eq!(states.len(), 2);
|
||||
assert_eq!(states["level:P.1"], "clear");
|
||||
assert_eq!(states["feed"], "stale");
|
||||
db.drop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn forecast_inputs_load_like_python() {
|
||||
let Some(db) = TestDb::new().await else {
|
||||
return;
|
||||
};
|
||||
let stations = Stations::bundled();
|
||||
let r = |h: u32, level: f64| Reading {
|
||||
timestamp: t(2026, 10, 18, h, 0),
|
||||
station_num: 8, // P.1
|
||||
water_level: level,
|
||||
discharge: Some(51.25),
|
||||
discharge_percent: Some(40.5),
|
||||
};
|
||||
db::save_rid(&db.pool, &[r(10, 1.1), r(11, 1.2), r(12, 1.3)], &stations)
|
||||
.await
|
||||
.unwrap();
|
||||
let rows = db::load_rid_window(&db.pool, t(2026, 10, 18, 11, 0), t(2026, 10, 18, 12, 30))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(rows.len(), 2);
|
||||
assert_eq!(
|
||||
(rows[0].station_code.as_str(), rows[0].water_level),
|
||||
("P.1", 1.2)
|
||||
);
|
||||
assert_eq!(rows[1].discharge, 51.25);
|
||||
|
||||
let latest = db::latest_rid_readings(&db.pool).await.unwrap();
|
||||
assert_eq!(latest.len(), 1);
|
||||
assert_eq!(
|
||||
(
|
||||
latest[0].timestamp,
|
||||
latest[0].water_level,
|
||||
latest[0].discharge_percent
|
||||
),
|
||||
(t(2026, 10, 18, 12, 0), Some(1.3), Some(40.5))
|
||||
);
|
||||
|
||||
let wl = |minute: u32, v: f64| WaterLevelRecord {
|
||||
station_id: 5,
|
||||
oldcode: None,
|
||||
rid_code: Some("P.1".into()),
|
||||
name_th: None,
|
||||
name_en: None,
|
||||
latitude: None,
|
||||
longitude: None,
|
||||
agency: None,
|
||||
river_name: None,
|
||||
offset_msl: None,
|
||||
ground_level_msl: None,
|
||||
min_bank_msl: None,
|
||||
critical_level_msl: None,
|
||||
critical_level_m: None,
|
||||
qmax: None,
|
||||
is_key_station: true,
|
||||
timestamp: t(2026, 10, 18, 13, minute),
|
||||
wl_msl: Some(v),
|
||||
wl_m: None,
|
||||
discharge: Some(52.0),
|
||||
flow_rate: None,
|
||||
storage_percent: None,
|
||||
situation_level: None,
|
||||
diff_wl_bank: None,
|
||||
};
|
||||
db::save_hii_waterlevel(
|
||||
&db.pool,
|
||||
&[wl(0, 301.4), wl(40, 301.45)],
|
||||
t(2026, 10, 18, 13, 45),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let codes = vec!["P.1".to_string(), "P.103".to_string()];
|
||||
let hii = db::load_hii_window(
|
||||
&db.pool,
|
||||
&codes,
|
||||
t(2026, 10, 18, 0, 0),
|
||||
t(2026, 10, 18, 14, 0),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(hii.len(), 2);
|
||||
assert_eq!(ping_forecast::hii::prepare(&hii)[0].wl_msl, 301.45);
|
||||
assert_eq!(
|
||||
db::latest_hii_timestamp(&db.pool, &codes).await.unwrap(),
|
||||
Some(t(2026, 10, 18, 13, 40))
|
||||
);
|
||||
assert!(
|
||||
db::load_dam_days(&db.pool, "200103", d(2026, 10, 1))
|
||||
.await
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
db.drop().await;
|
||||
}
|
||||
Reference in new issue
Block a user