feat(collector): forecast and ntfy in ping-collector; ThaiWater keeps alerts current at night
CI / Format & lint (push) Successful in 22s
CI / Test suite (push) Successful in 59s
Docs / Validate documentation (push) Successful in 25s
Rust / fmt, clippy, test, build (push) Successful in 10m2s
Security / Dependency vulnerabilities (push) Successful in 2m11s
Security / Static analysis (push) Successful in 22s
Security / License report (push) Successful in 1m37s
CI / Format & lint (push) Successful in 22s
CI / Test suite (push) Successful in 59s
Docs / Validate documentation (push) Successful in 25s
Rust / fmt, clippy, test, build (push) Successful in 10m2s
Security / Dependency vulnerabilities (push) Successful in 2m11s
Security / Static analysis (push) Successful in 22s
Security / License report (push) Successful in 1m37s
Phase 2b/2c of docs/RUST_REWRITE.md. With FORECAST_IN_COLLECTOR=true (shared .env) ping-collector, after each cycle that brought newer RID or HII readings and at least hourly: loads the last 336 h (RID + HII mirror fill), fetches the rain forecast, runs every exported model (rust/forecast), stores forecast_history and forecast_current, and sends the ntfy notifications. The Python API then serves forecast_current as /forecast (computing in-process only when it is older than FORECAST_MAX_AGE_SECONDS) and neither forecasts nor notifies. ntfy (collector/src/notify.rs): a port of src/notify.py with the same topics, texts, hysteresis, capacity guard and notification_state keys, checked word for word on 751 messages generated by Python (testdata/gen_notify_cases.py). The change: for stations whose ThaiWater twin is the same sensor, a reading newer than RID's drives the level alerts (marked as ThaiWater telemetry), and the feed counts as stale only when neither source has a reading for 3 h. RID alone lagging is reported once after RID_STALE_ALERT_HOURS (18). This ends the nightly "gauge feed stale" / "recovered" pairs caused by RID's late reports. Commands: `forecast [--json] [--save] [--now] [--rain-file]`, `notify --dry-run`. scripts/forecast_parity.py compares Python and the collector on the live database (same moment, same rain); locally 48/48 rows identical. install_collector.sh: models export + that check on every install, --cutover-forecast (refuses on a failed check) and --rollback-forecast. Trees are packed into 24-byte nodes: 70 MB peak with all 15 models (was 140). Forecast values are written to NUMERIC as shortest round-trip text, so they round as psycopg2's do. DB test schemas get a counter (parallel tests collided on Windows' coarse clock).
This commit is contained in:
1 parent
5c4f464383
commit
6ae67dec63
27 files changed
+2656
-118
No files matched your search
+81
-40
@@ -7,19 +7,23 @@
|
||||
use anyhow::{Result, bail};
|
||||
use serde::Deserialize;
|
||||
|
||||
fn thresholds<'de, D: serde::Deserializer<'de>>(d: D) -> Result<Vec<f64>, D::Error> {
|
||||
let raw: Vec<Option<f64>> = Deserialize::deserialize(d)?;
|
||||
Ok(raw
|
||||
.into_iter()
|
||||
.map(|t| t.unwrap_or(f64::INFINITY))
|
||||
.collect())
|
||||
/// One node, packed (24 bytes): `x` is the split threshold of an internal
|
||||
/// node and the output of a leaf.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct Node {
|
||||
x: f64,
|
||||
left: u32,
|
||||
right: u32,
|
||||
feature: u16,
|
||||
leaf: bool,
|
||||
missing_left: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct Tree {
|
||||
/// The exported arrays, as in sklearn's predictor node table.
|
||||
#[derive(Deserialize)]
|
||||
struct RawTree {
|
||||
feature: Vec<usize>,
|
||||
#[serde(deserialize_with = "thresholds")]
|
||||
threshold: Vec<f64>,
|
||||
threshold: Vec<Option<f64>>,
|
||||
missing_left: Vec<bool>,
|
||||
left: Vec<usize>,
|
||||
right: Vec<usize>,
|
||||
@@ -27,44 +31,81 @@ pub struct Tree {
|
||||
value: Vec<f64>,
|
||||
}
|
||||
|
||||
impl Tree {
|
||||
#[inline]
|
||||
fn predict(&self, x: &[f64]) -> f64 {
|
||||
let mut n = 0;
|
||||
while !self.leaf[n] {
|
||||
let v = x[self.feature[n]];
|
||||
let go_left = if v.is_nan() {
|
||||
self.missing_left[n]
|
||||
} else {
|
||||
v <= self.threshold[n]
|
||||
};
|
||||
n = if go_left { self.left[n] } else { self.right[n] };
|
||||
}
|
||||
self.value[n]
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub struct Tree {
|
||||
nodes: Vec<Node>,
|
||||
}
|
||||
|
||||
fn validate(&self, n_features: usize) -> Result<()> {
|
||||
let n = self.value.len();
|
||||
impl<'de> Deserialize<'de> for Tree {
|
||||
fn deserialize<D: serde::Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
|
||||
use serde::de::Error;
|
||||
let r = RawTree::deserialize(d)?;
|
||||
let n = r.value.len();
|
||||
if n == 0
|
||||
|| [
|
||||
self.feature.len(),
|
||||
self.threshold.len(),
|
||||
self.missing_left.len(),
|
||||
self.left.len(),
|
||||
self.right.len(),
|
||||
self.leaf.len(),
|
||||
r.feature.len(),
|
||||
r.threshold.len(),
|
||||
r.missing_left.len(),
|
||||
r.left.len(),
|
||||
r.right.len(),
|
||||
r.leaf.len(),
|
||||
]
|
||||
.iter()
|
||||
.any(|&l| l != n)
|
||||
{
|
||||
bail!("tree arrays have different lengths");
|
||||
return Err(D::Error::custom("tree arrays have different lengths"));
|
||||
}
|
||||
for i in 0..n {
|
||||
if !self.leaf[i]
|
||||
&& (self.feature[i] >= n_features || self.left[i] >= n || self.right[i] >= n)
|
||||
{
|
||||
bail!("tree node {i} points outside the tree or the feature vector");
|
||||
}
|
||||
let nodes = (0..n)
|
||||
.map(|i| {
|
||||
let leaf = r.leaf[i];
|
||||
if !leaf && (r.left[i] >= n || r.right[i] >= n || r.feature[i] > u16::MAX as usize)
|
||||
{
|
||||
return Err(D::Error::custom(format!(
|
||||
"tree node {i} points outside the tree"
|
||||
)));
|
||||
}
|
||||
Ok(Node {
|
||||
// +inf thresholds ("missing vs everything") are exported as null
|
||||
x: if leaf {
|
||||
r.value[i]
|
||||
} else {
|
||||
r.threshold[i].unwrap_or(f64::INFINITY)
|
||||
},
|
||||
left: r.left[i] as u32,
|
||||
right: r.right[i] as u32,
|
||||
feature: r.feature[i] as u16,
|
||||
leaf,
|
||||
missing_left: r.missing_left[i],
|
||||
})
|
||||
})
|
||||
.collect::<Result<_, _>>()?;
|
||||
Ok(Tree { nodes })
|
||||
}
|
||||
}
|
||||
|
||||
impl Tree {
|
||||
#[inline]
|
||||
fn predict(&self, x: &[f64]) -> f64 {
|
||||
let mut node = &self.nodes[0];
|
||||
while !node.leaf {
|
||||
let v = x[node.feature as usize];
|
||||
let go_left = if v.is_nan() {
|
||||
node.missing_left
|
||||
} else {
|
||||
v <= node.x
|
||||
};
|
||||
node = &self.nodes[if go_left { node.left } else { node.right } as usize];
|
||||
}
|
||||
node.x
|
||||
}
|
||||
|
||||
fn validate(&self, n_features: usize) -> Result<()> {
|
||||
if let Some(i) = self
|
||||
.nodes
|
||||
.iter()
|
||||
.position(|n| !n.leaf && n.feature as usize >= n_features)
|
||||
{
|
||||
bail!("tree node {i} reads feature outside the feature vector");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user