fix(forecast): no nightly gaps in the issued model forecasts

The P.1 chart's "24 h peak, as issued" line had a gap every night and a
dip at 21:00. Two causes:

- A run forecasts from the newest reading only. RID publishes 22:00..06:00
  in one batch in the morning (and now and then one hour late), so those
  hours were never forecast from. After each run the collector now
  forecasts the hours of the last 48 h (two weeks on the first run after a
  start) that have a RID reading but no model forecast, each as a run at
  that hour would have: gauges, HII fill and dam state up to that hour.
  Rain is fetched once, so a late hour's rain_fc24 partly holds rain that
  has since fallen (as in training). Only missing (hour, station) pairs are
  written. `ping-collector catch-up-forecasts --hours N` goes further back.
- Overnight each hourly run re-issues the 21:00 reading, after 6 h as the
  heuristic, which overwrote the 21:00 model row. A heuristic row no longer
  replaces a model row in forecast_history.

The dashboard draws only model rows on the issued line.

Checked on a local copy of 18 days of RID data: the catch-up of an hour
matches `ping-collector forecast --now <hour>`; 288 hours took 20 s
(debug build). Tests for both database rules.

rust/README.md

@@ -25,6 +25,9 @@ ping-collector fill-gaps <days|all> # re-fetch RID reports with holes
 ping-collector check-db             # connect, create missing tables, row counts
 ping-collector forecast [--json] [--save] [--now T] [--rain-file F]
                                     # forecast from the database and print it
+ping-collector catch-up-forecasts [--hours N]
+                                    # forecast the hours whose readings came late (RID's
+                                    # overnight batch) as a run then would have; default 336 h
 ping-collector notify --dry-run     # the readings the notifier judges, and what it would send
 ```
 

rust/collector/src/db.rs

@@ -267,6 +267,34 @@ pub async fn earliest_rid_timestamp(pool: &PgPool) -> Result<Option<NaiveDateTim
     )
 }
 
+/// (hour, station) pairs in `[start, end)` where one of `codes` has a RID
+/// level but no model forecast: hours that arrived together with a later
+/// one (RID's overnight batch, a late report), so no run forecast from them.
+pub async fn hours_without_model_forecast(
+    pool: &PgPool,
+    codes: &[String],
+    start: NaiveDateTime,
+    end: NaiveDateTime,
+) -> Result<Vec<(NaiveDateTime, String)>> {
+    sqlx::query_as(
+        "SELECT DISTINCT date_trunc('hour', m.timestamp) AS h, s.station_code::text
+         FROM water_measurements m JOIN stations s ON s.id = m.station_id
+         WHERE m.timestamp >= $1 AND m.timestamp < $2 AND m.water_level IS NOT NULL
+           AND s.station_code = ANY($3)
+           AND NOT EXISTS (
+               SELECT 1 FROM forecast_history f
+                WHERE f.station_code = s.station_code
+                  AND f.as_of = date_trunc('hour', m.timestamp) AND f.source = 'model')
+         ORDER BY 1, 2",
+    )
+    .bind(start)
+    .bind(end)
+    .bind(codes)
+    .fetch_all(pool)
+    .await
+    .context("finding hours without a model forecast")
+}
+
 pub async fn latest_rid_timestamp(pool: &PgPool) -> Result<Option<NaiveDateTime>> {
     Ok(
         sqlx::query_scalar("SELECT MAX(timestamp) FROM water_measurements")
@@ -665,6 +693,11 @@ pub async fn latest_rid_readings(pool: &PgPool) -> Result<Vec<LatestReading>> {
 }
 
 /// `ForecastHistoryStore.save_rows`: one row per (as_of, station, horizon).
+/// Store issued forecasts; a later run's row for the same hour replaces the
+/// earlier one, except that a heuristic row never replaces a model row. The
+/// hourly run re-issues the newest reading's hour while RID is silent
+/// overnight, and after 6 h (`stale_after_h`) it does so as the heuristic:
+/// without the exception the 21:00 model forecast was overwritten each night.
 pub async fn save_forecast_history(
     pool: &PgPool,
     rows: &[ping_forecast::Forecast],
@@ -683,7 +716,8 @@ pub async fn save_forecast_history(
          ON CONFLICT (as_of, station_code, horizon_hours) DO UPDATE SET
             predicted_max_level = EXCLUDED.predicted_max_level, p_warning = EXCLUDED.p_warning,
             p_danger = EXCLUDED.p_danger, current_level = EXCLUDED.current_level,
-            model_version = EXCLUDED.model_version, source = EXCLUDED.source",
+            model_version = EXCLUDED.model_version, source = EXCLUDED.source
+         WHERE forecast_history.source IS DISTINCT FROM 'model' OR EXCLUDED.source = 'model'",
     )
     .bind(rows.iter().map(|r| r.as_of).collect::<Vec<_>>())
     .bind(

rust/collector/src/db_tests.rs

@@ -404,6 +404,109 @@ async fn forecast_outputs_round_trip() {
     db.drop().await;
 }
 
+#[tokio::test]
+async fn heuristic_never_replaces_a_model_forecast() {
+    let Some(db) = TestDb::new().await else {
+        return;
+    };
+    // overnight the run re-issues the 21:00 reading, after 6 h as heuristic
+    let model = forecast_row("P.1", 24, 0.2);
+    let heuristic = ping_forecast::Forecast {
+        p_warning: 0.0,
+        predicted_max_level: 1.23,
+        model_version: "heuristic-v1".into(),
+        source: "heuristic".into(),
+        ..forecast_row("P.1", 24, 0.0)
+    };
+    let stored = async || -> (String, f64) {
+        sqlx::query_as("SELECT source::text, predicted_max_level::float8 FROM forecast_history")
+            .fetch_one(&db.pool)
+            .await
+            .unwrap()
+    };
+    db::save_forecast_history(&db.pool, std::slice::from_ref(&model))
+        .await
+        .unwrap();
+    db::save_forecast_history(&db.pool, std::slice::from_ref(&heuristic))
+        .await
+        .unwrap();
+    assert_eq!(stored().await, ("model".into(), 3.123));
+    // a model row does replace a heuristic one (and a newer model row)
+    sqlx::query("DELETE FROM forecast_history")
+        .execute(&db.pool)
+        .await
+        .unwrap();
+    db::save_forecast_history(&db.pool, &[heuristic])
+        .await
+        .unwrap();
+    assert_eq!(stored().await, ("heuristic".into(), 1.23));
+    db::save_forecast_history(&db.pool, &[model]).await.unwrap();
+    assert_eq!(stored().await, ("model".into(), 3.123));
+    db.drop().await;
+}
+
+#[tokio::test]
+async fn hours_without_a_model_forecast() {
+    let Some(db) = TestDb::new().await else {
+        return;
+    };
+    let stations = Stations::bundled();
+    let r = |h: u32, num: u32| Reading {
+        timestamp: t(2026, 10, 18, h, 0),
+        station_num: num, // 8 = P.1, 1 = P.20
+        water_level: 1.1,
+        discharge: None,
+        discharge_percent: None,
+    };
+    // RID's morning batch: 11..13 arrive at once; only 13:00 (the newest)
+    // and P.20's 11:00 were forecast, and 12:00 only by the heuristic
+    db::save_rid(
+        &db.pool,
+        &[r(11, 8), r(12, 8), r(13, 8), r(11, 1), r(12, 1), r(13, 1)],
+        &stations,
+    )
+    .await
+    .unwrap();
+    let issued = |code: &str, h: u32, source: &str| ping_forecast::Forecast {
+        as_of: t(2026, 10, 18, h, 0),
+        source: source.into(),
+        ..forecast_row(code, 24, 0.1)
+    };
+    db::save_forecast_history(
+        &db.pool,
+        &[
+            issued("P.1", 13, "model"),
+            issued("P.20", 13, "model"),
+            issued("P.20", 11, "model"),
+            issued("P.1", 12, "heuristic"),
+        ],
+    )
+    .await
+    .unwrap();
+    let codes = vec!["P.1".to_string(), "P.20".to_string()];
+    let missing = db::hours_without_model_forecast(
+        &db.pool,
+        &codes,
+        t(2026, 10, 18, 0, 0),
+        t(2026, 10, 18, 14, 0),
+    )
+    .await
+    .unwrap();
+    let at = |h: u32, code: &str| (t(2026, 10, 18, h, 0), code.to_string());
+    assert_eq!(missing, vec![at(11, "P.1"), at(12, "P.1"), at(12, "P.20")]);
+    // stations without a model are not asked about
+    let only_p20 = db::hours_without_model_forecast(
+        &db.pool,
+        &codes[1..],
+        t(2026, 10, 18, 0, 0),
+        t(2026, 10, 18, 14, 0),
+    )
+    .await
+    .unwrap();
+    assert_eq!(only_p20, vec![at(12, "P.20")]);
+    db.drop().await;
+}
+
 #[tokio::test]
 async fn notification_state_round_trip() {
     let Some(db) = TestDb::new().await else {

rust/collector/src/forecasting.rs

@@ -5,7 +5,8 @@
 //! least once per clock hour: load the last 336 h (RID plus the HII mirror
 //! fill), fetch the rain forecast, run every exported model, store the result
 //! (`forecast_history`, and `forecast_current` which the API serves as
-//! /forecast) and push ntfy transitions.
+//! /forecast) and push ntfy transitions. Then forecast the hours whose
+//! readings came late (`catch_up`).
 
 use crate::config::Config;
 use crate::db;
@@ -20,6 +21,7 @@ use ping_forecast::hii::{self, FillInfo};
 use ping_forecast::rain::RainSeries;
 use ping_forecast::{Forecast, ModelSet, ModelStore};
 use sqlx::PgPool;
+use std::collections::{BTreeMap, BTreeSet, HashSet};
 
 /// Everything one forecast run used and produced.
 pub struct Snapshot {
@@ -93,6 +95,87 @@ pub async fn compute(
     })
 }
 
+/// How far back every run looks for hours no run forecast from; the first
+/// run after a start looks two weeks back (the nights before an upgrade).
+pub const CATCH_UP_HOURS: i64 = 48;
+pub const CATCH_UP_START_HOURS: i64 = 14 * 24;
+
+/// What a catch-up did.
+#[derive(Debug, Default)]
+pub struct CatchUp {
+    /// hours computed (also those that produced no model row)
+    pub hours: Vec<NaiveDateTime>,
+    /// forecast_history rows stored
+    pub rows: usize,
+}
+
+/// Model forecasts for the hours whose readings arrived together with a
+/// later hour's. RID publishes 22:00..06:00 in one batch in the morning, so
+/// the hourly run only ever forecast from 06:00 and the night was missing
+/// from the issued history (also single hours RID reported late).
+///
+/// Each hour is computed as a run at that hour would have been: RID, the
+/// HII fill and the dam state up to that hour only. Rain is one fetch now,
+/// so a caught-up hour's `rain_fc24` is partly Open-Meteo's estimate of rain
+/// that has since fallen rather than a forecast (the training condition, not
+/// what a live run had). Only missing (hour, station) pairs are written, and
+/// a stored model row is never replaced. Hours in `skip` were tried before.
+pub async fn catch_up(
+    pool: &PgPool,
+    rain_client: &OpenMeteoClient,
+    models: &ModelSet,
+    now: NaiveDateTime,
+    hours_back: i64,
+    skip: &HashSet<NaiveDateTime>,
+) -> Result<CatchUp> {
+    let codes: Vec<String> = models.keys().cloned().collect();
+    let start = floor_hour(now) - Duration::hours(hours_back);
+    let mut missing: BTreeMap<NaiveDateTime, BTreeSet<String>> = BTreeMap::new();
+    for (hour, code) in db::hours_without_model_forecast(pool, &codes, start, now).await? {
+        if !skip.contains(&hour) {
+            missing.entry(hour).or_default().insert(code);
+        }
+    }
+    if missing.is_empty() {
+        return Ok(CatchUp::default());
+    }
+    // the rain windows (up to 72 h) of the oldest hour, plus a margin
+    let past_days = (hours_back / 24 + 5).min(92) as u32;
+    let rain: RainSeries = rain_client
+        .fetch_forecast(past_days, 2)
+        .await?
+        .into_iter()
+        .map(|r| (r.timestamp, r.catchment_mean.unwrap_or(f64::NAN)))
+        .collect();
+    let mut done = CatchUp::default();
+    for (hour, stations) in missing {
+        let snap = compute(pool, rain_client, models, hour, Some(rain.clone())).await?;
+        let rows: Vec<Forecast> = snap
+            .forecasts
+            .into_iter()
+            .filter(|f| {
+                f.as_of == hour && f.source == "model" && stations.contains(&f.station_code)
+            })
+            .collect();
+        done.rows += db::save_forecast_history(pool, &rows).await?;
+        done.hours.push(hour);
+    }
+    Ok(done)
+}
+
+pub fn describe_catch_up(c: &CatchUp) -> String {
+    match (c.hours.first(), c.hours.last()) {
+        (Some(first), Some(last)) => format!(
+            "{} hours from {} to {}, {} rows stored",
+            c.hours.len(),
+            first.format("%d %H:%M"),
+            last.format("%d %H:%M"),
+            c.rows
+        ),
+        _ => "nothing to catch up".into(),
+    }
+}
+
 pub fn summarize(s: &Snapshot) -> String {
     let model = s.forecasts.iter().filter(|f| f.source == "model").count();
     let p1 = s
@@ -171,6 +254,10 @@ pub struct Forecaster {
     notifier: Option<Notifier>,
     last_inputs: Option<(Option<NaiveDateTime>, Option<NaiveDateTime>)>,
     last_slot: Option<NaiveDateTime>,
+    /// hours `catch_up` computed already (a station whose features cannot
+    /// be built at that hour would otherwise be retried on every run)
+    caught_up: HashSet<NaiveDateTime>,
+    caught_up_since_start: bool,
 }
 
 impl Forecaster {
@@ -202,6 +289,8 @@ impl Forecaster {
             notifier,
             last_inputs: None,
             last_slot: None,
+            caught_up: HashSet::new(),
+            caught_up_since_start: false,
         }
     }
 
@@ -272,6 +361,24 @@ impl Forecaster {
                 Err(e) => tracing::warn!("notify cycle failed: {e:#}"),
             }
         }
+        // after the notifications: they are about now, this is history
+        let window = if self.caught_up_since_start {
+            CATCH_UP_HOURS
+        } else {
+            CATCH_UP_START_HOURS
+        };
+        match catch_up(pool, rain, &models, now, window, &self.caught_up).await {
+            Ok(c) => {
+                if !c.hours.is_empty() {
+                    tracing::info!("forecast catch-up: {}", describe_catch_up(&c));
+                }
+                self.caught_up.extend(c.hours);
+                self.caught_up_since_start = true;
+                let oldest = floor_hour(now) - Duration::hours(CATCH_UP_HOURS);
+                self.caught_up.retain(|h| *h >= oldest);
+            }
+            Err(e) => tracing::warn!("forecast catch-up failed: {e:#}"),
+        }
         Ok(RunInfo {
             at: now,
             rows: snap.forecasts.len(),

rust/collector/src/main.rs

@@ -80,6 +80,14 @@ enum Command {
         #[arg(long)]
         rain_file: Option<PathBuf>,
     },
+    /// Forecast the hours no run forecast from (readings that arrived with a
+    /// later hour's, e.g. RID's overnight batch) as a run at each hour would
+    /// have, and store them; every run does this for the last 48 h
+    CatchUpForecasts {
+        /// How many hours back to look
+        #[arg(long, default_value_t = 336)]
+        hours: i64,
+    },
     /// Show the readings the notifier judges and the ntfy transitions due now
     Notify {
         /// Show what would be published; publish nothing, change no state
@@ -204,6 +212,7 @@ async fn run(cli: Cli) -> Result<()> {
             let rain = rain_file.as_deref().map(read_rain_file).transpose()?;
             collector.forecast(json, save, now, rain).await
         }
+        Command::CatchUpForecasts { hours } => collector.catch_up_forecasts(hours).await,
         Command::Notify { dry_run } => collector.notify(dry_run).await,
         Command::CheckDb => {
             db::ensure_schema(&collector.pool).await?;

rust/collector/src/service.rs

@@ -398,6 +398,34 @@ impl Collector {
         Ok(())
     }
 
+    /// `ping-collector catch-up-forecasts`: `forecasting::catch_up` over
+    /// `hours`, e.g. once after an upgrade for the nights before it.
+    pub async fn catch_up_forecasts(&self, hours: i64) -> Result<()> {
+        if !(1..=24 * 60).contains(&hours) {
+            bail!("--hours must be between 1 and 1440 (Open-Meteo keeps 92 days of past rain)");
+        }
+        let mut store = ModelStore::new(&self.cfg.models_dir);
+        let models = store.refresh()?;
+        if models.is_empty() {
+            bail!(
+                "no exported models in {} (run `python -m src.ml.export`)",
+                self.cfg.models_dir.display()
+            );
+        }
+        db::ensure_schema(&self.pool).await?;
+        let c = forecasting::catch_up(
+            &self.pool,
+            &self.rain,
+            &models,
+            bangkok_now(),
+            hours,
+            &Default::default(),
+        )
+        .await?;
+        eprintln!("forecast catch-up: {}", forecasting::describe_catch_up(&c));
+        Ok(())
+    }
+
     /// `ping-collector notify`: what would be (or, without `dry_run`, is)
     /// published right now against the stored notification state.
     pub async fn notify(&self, dry_run: bool) -> Result<()> {