feat(forecast): Rust forecast crate, bit-for-bit with the Python serving path
CI / Test suite (push) Successful in 1m2s
CI / Format & lint (push) Successful in 22s
Rust / fmt, clippy, test, build (push) Successful in 9m53s
Security / Dependency vulnerabilities (push) Successful in 1m52s
Security / Static analysis (push) Successful in 20s
Security / License report (push) Successful in 1m41s
CI / Test suite (push) Successful in 1m2s
CI / Format & lint (push) Successful in 22s
Rust / fmt, clippy, test, build (push) Successful in 9m53s
Security / Dependency vulnerabilities (push) Successful in 1m52s
Security / Static analysis (push) Successful in 20s
Security / License report (push) Successful in 1m41s
Phase 2a of docs/RUST_REWRITE.md. rust/forecast (ping-forecast) ports the serving half of src/ml: the HII mirror gap fill, the hourly grid, every feature (pandas' Kahan rolling sums included), the Mae Ngat dam frame, HistGradientBoosting tree inference, the heuristic fallback, stale-threshold handling and the P.1 flood-stage probabilities. Hand-over from Python (src/ml/export.py): - models/flood_<station>.json.gz next to each joblib bundle; train_all() writes it, retrain.sh promotes it, `python -m src.ml.export` exports existing ones. - rust/forecast/flood_config.json: thresholds, stages, leads and notify constants, compiled in. tests/test_model_export.py fails while it differs from the Python source. Parity (rust/forecast/tests/parity.rs, fixtures from testdata/gen_forecast_fixtures.py on the production code path): four scenarios incl. the 2024 flood with RID lagging behind HII. Merged readings, all features and all levels identical; with all 15 local models 377/384 probabilities bit-identical, the rest within 2 ulp (exp). Python refactor without behaviour change: predict.forecasts_from_long, and get_latest_forecasts/load_latest/serving_frame take now/rain for reproducible comparisons; the HII query orders its rows.
This commit is contained in:
1 parent
6d4beae65a
commit
5c4f464383
31 files changed
+2889
-29
No files matched your search
@@ -0,0 +1,224 @@
|
||||
//! Gap fill from the HII/ThaiWater mirror telemetry (`data.fill_from_hii`).
|
||||
//!
|
||||
//! For stations whose HII twin is the same sensor, hours missing from the RID
|
||||
//! series (or carrying no level) are filled from HII, shifted by the median
|
||||
//! offset between the two series over their overlap. This is also what keeps
|
||||
//! the forecast current at night while RID's report lags.
|
||||
|
||||
use crate::config::config;
|
||||
use crate::grid::{Row, floor_hour};
|
||||
use chrono::NaiveDateTime;
|
||||
use std::collections::{BTreeMap, BTreeSet, HashSet};
|
||||
|
||||
/// One `hii_waterlevel` row joined to its RID station code; NaN = missing.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct HiiRow {
|
||||
pub timestamp: NaiveDateTime,
|
||||
pub station_code: String,
|
||||
pub wl_msl: f64,
|
||||
pub discharge: f64,
|
||||
}
|
||||
|
||||
/// `data._prepare_hii_levels`: rows without a level dropped, one row per
|
||||
/// station-hour, the latest reading of the hour winning (stable on ties).
|
||||
pub fn prepare(rows: &[HiiRow]) -> Vec<HiiRow> {
|
||||
let mut kept: Vec<&HiiRow> = rows.iter().filter(|r| !r.wl_msl.is_nan()).collect();
|
||||
kept.sort_by_key(|r| r.timestamp); // stable
|
||||
let mut by_hour: BTreeMap<(String, NaiveDateTime), HiiRow> = BTreeMap::new();
|
||||
for r in kept {
|
||||
let t = floor_hour(r.timestamp);
|
||||
by_hour.insert(
|
||||
(r.station_code.clone(), t),
|
||||
HiiRow {
|
||||
timestamp: t,
|
||||
..r.clone()
|
||||
},
|
||||
);
|
||||
}
|
||||
by_hour.into_values().collect()
|
||||
}
|
||||
|
||||
/// The HII station codes worth fetching for these RID rows.
|
||||
pub fn fill_candidates(rid: &[Row]) -> Vec<String> {
|
||||
let present: HashSet<&str> = rid.iter().map(|r| r.station_code.as_str()).collect();
|
||||
config()
|
||||
.hii_fill_stations
|
||||
.iter()
|
||||
.filter(|c| present.contains(c.as_str()))
|
||||
.cloned()
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Per-station result of the fill, for logs and the notifier.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct FillInfo {
|
||||
pub station_code: String,
|
||||
pub offset: f64,
|
||||
pub filled_hours: usize,
|
||||
}
|
||||
|
||||
/// pandas `Series.median` (numpy: mean of the two middle values when even).
|
||||
fn median(mut v: Vec<f64>) -> f64 {
|
||||
v.sort_by(f64::total_cmp);
|
||||
let n = v.len();
|
||||
if n % 2 == 1 {
|
||||
v[n / 2]
|
||||
} else {
|
||||
(v[n / 2 - 1] + v[n / 2]) / 2.0
|
||||
}
|
||||
}
|
||||
|
||||
/// `data.fill_from_hii`: `rid` as loaded (already one row per station-hour),
|
||||
/// `hii` from [`prepare`]. Returns the merged rows (fill rows replace RID rows
|
||||
/// that have no level) and what was filled.
|
||||
pub fn fill(rid: &[Row], hii: &[HiiRow]) -> (Vec<Row>, Vec<FillInfo>) {
|
||||
let cfg = config();
|
||||
let candidates: BTreeSet<String> = fill_candidates(rid).into_iter().collect();
|
||||
let mut mirrors: BTreeMap<&str, Vec<&HiiRow>> = BTreeMap::new();
|
||||
for h in hii {
|
||||
if candidates.contains(&h.station_code) {
|
||||
mirrors.entry(h.station_code.as_str()).or_default().push(h);
|
||||
}
|
||||
}
|
||||
let mut fills: Vec<Row> = Vec::new();
|
||||
let mut info = Vec::new();
|
||||
for (code, mirror) in mirrors {
|
||||
let base: BTreeMap<NaiveDateTime, f64> = rid
|
||||
.iter()
|
||||
.filter(|r| r.station_code == code)
|
||||
.map(|r| (r.timestamp, r.water_level))
|
||||
.collect();
|
||||
let diffs: Vec<f64> = mirror
|
||||
.iter()
|
||||
.filter_map(|m| match base.get(&m.timestamp) {
|
||||
Some(level) if !level.is_nan() => Some(m.wl_msl - level),
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
if diffs.len() < cfg.hii_min_overlap_hours {
|
||||
continue;
|
||||
}
|
||||
let offset = median(diffs);
|
||||
let exact = cfg.hii_exact_mirrors.iter().any(|c| c == code);
|
||||
let before = fills.len();
|
||||
for m in mirror {
|
||||
if base.get(&m.timestamp).is_some_and(|l| !l.is_nan()) {
|
||||
continue;
|
||||
}
|
||||
fills.push(Row {
|
||||
timestamp: m.timestamp,
|
||||
station_code: code.to_string(),
|
||||
water_level: m.wl_msl - offset,
|
||||
discharge: if exact { m.discharge } else { f64::NAN },
|
||||
});
|
||||
}
|
||||
if fills.len() > before {
|
||||
info.push(FillInfo {
|
||||
station_code: code.to_string(),
|
||||
offset,
|
||||
filled_hours: fills.len() - before,
|
||||
});
|
||||
}
|
||||
}
|
||||
if fills.is_empty() {
|
||||
return (rid.to_vec(), info);
|
||||
}
|
||||
// concat + drop_duplicates(keep="last"): a fill replaces a level-less RID row
|
||||
let mut merged: BTreeMap<(String, NaiveDateTime), Row> = BTreeMap::new();
|
||||
for r in rid.iter().chain(fills.iter()) {
|
||||
merged.insert((r.station_code.clone(), r.timestamp), r.clone());
|
||||
}
|
||||
(merged.into_values().collect(), info)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use chrono::{Duration, NaiveDate};
|
||||
|
||||
fn t0() -> NaiveDateTime {
|
||||
NaiveDate::from_ymd_opt(2026, 10, 1)
|
||||
.unwrap()
|
||||
.and_hms_opt(0, 0, 0)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn rid(code: &str, hours: std::ops::Range<i64>) -> Vec<Row> {
|
||||
hours
|
||||
.map(|h| Row {
|
||||
timestamp: t0() + Duration::hours(h),
|
||||
station_code: code.into(),
|
||||
water_level: 1.0 + h as f64 * 0.01,
|
||||
discharge: 50.0,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn mirror(code: &str, hours: std::ops::Range<i64>, offset: f64) -> Vec<HiiRow> {
|
||||
hours
|
||||
.map(|h| HiiRow {
|
||||
timestamp: t0() + Duration::hours(h),
|
||||
station_code: code.into(),
|
||||
wl_msl: 1.0 + h as f64 * 0.01 + offset,
|
||||
discharge: 51.0,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trailing_hours_filled_with_the_overlap_offset() {
|
||||
let r = rid("P.1", 0..200);
|
||||
let h = prepare(&mirror("P.1", 0..203, 300.0));
|
||||
let (merged, info) = fill(&r, &h);
|
||||
assert_eq!(merged.len(), 203);
|
||||
assert_eq!(info[0].filled_hours, 3);
|
||||
let last = merged.last().unwrap();
|
||||
assert_eq!(last.timestamp, t0() + Duration::hours(202));
|
||||
assert!((last.water_level - 3.02).abs() < 1e-9);
|
||||
assert_eq!(last.discharge, 51.0); // exact mirror keeps discharge
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn short_overlap_or_other_sensor_is_not_merged() {
|
||||
let r = rid("P.1", 0..100);
|
||||
assert_eq!(
|
||||
fill(&r, &prepare(&mirror("P.1", 0..103, 300.0))).0.len(),
|
||||
100
|
||||
);
|
||||
let r = rid("P.77", 0..200);
|
||||
assert_eq!(
|
||||
fill(&r, &prepare(&mirror("P.77", 0..203, 300.0))).0.len(),
|
||||
200
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn p81_is_bias_corrected_without_discharge() {
|
||||
let (merged, _) = fill(&rid("P.81", 0..200), &prepare(&mirror("P.81", 0..201, 5.0)));
|
||||
assert!(merged.last().unwrap().discharge.is_nan());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn latest_reading_of_an_hour_wins() {
|
||||
let mut rows = mirror("P.1", 0..1, 300.0);
|
||||
rows.push(HiiRow {
|
||||
timestamp: t0() + Duration::minutes(40),
|
||||
wl_msl: 302.0,
|
||||
..rows[0].clone()
|
||||
});
|
||||
rows.push(HiiRow {
|
||||
timestamp: t0() + Duration::minutes(10),
|
||||
wl_msl: 301.5,
|
||||
..rows[0].clone()
|
||||
});
|
||||
rows.push(HiiRow {
|
||||
wl_msl: f64::NAN,
|
||||
timestamp: t0() + Duration::minutes(50),
|
||||
..rows[0].clone()
|
||||
});
|
||||
let p = prepare(&rows);
|
||||
assert_eq!(p.len(), 1);
|
||||
assert_eq!(p[0].wl_msl, 302.0);
|
||||
assert_eq!(p[0].timestamp, t0());
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user