fix(ml): serve the forecast rain the model was trained with

build_features cut the rain series to the gauge grid before the forward sum,
and the grid ends at the newest reading, so the row every live forecast is
made from had rain_fc24 = NaN. Training and all evaluations had the real
Open-Meteo forecast there. The forward sum now runs over the grid extended by
24 h; earlier rows are unchanged.

Harness (new opt-in variant rise_rain_fc24_missing = rise_rain with
rain_fc24 blanked on test rows, five folds, --from-cache): P.1 and P.103
nearly unaffected, P.21 false alarms 10 -> 5 and MAE 0.232 -> 0.215 with the
fix. Write-up in docs/FLOOD_FORECASTING.md section 5; results in
models/eval_2026-10-18_fc24.json.

Also: HII mirror rows in one hour now resolve to the latest reading via a
stable sort on the raw time (was an unstable sort on the floored hour, i.e.
row-order dependent). Needed for an exact Rust port.

src/ml/data.py

@@ -146,15 +146,22 @@ def _fetch_hii_levels(
 
     with engine.connect() as connection:
         df = pd.read_sql(text(query), connection, params=params)
+    return _prepare_hii_levels(df)
+
+
+def _prepare_hii_levels(df: pd.DataFrame) -> pd.DataFrame:
+    """Hourly mirror rows. Several readings in one hour: the latest wins
+    (stable sort on the raw time, so the choice never depends on row order)."""
     df = df.dropna(subset=["wl_msl"])
     if df.empty:
         return df
-    df["timestamp"] = pd.to_datetime(df["timestamp"]).dt.floor("h")
+    df = df.copy()
+    df["timestamp"] = pd.to_datetime(df["timestamp"])
+    df = df.sort_values("timestamp", kind="stable")
+    df["timestamp"] = df["timestamp"].dt.floor("h")
     df["wl_msl"] = pd.to_numeric(df["wl_msl"], errors="coerce")
     df["discharge"] = pd.to_numeric(df["discharge"], errors="coerce")
-    df = df.sort_values("timestamp").drop_duplicates(
-        subset=["station_code", "timestamp"], keep="last"
-    )
+    df = df.drop_duplicates(subset=["station_code", "timestamp"], keep="last")
     return df
 
 

src/ml/evaluate.py

@@ -72,6 +72,7 @@ class Variant:
         use_dam: bool = False,
         use_fc48: bool = False,
         qsigma: bool = False,
+        fc24_missing_at_serve: bool = False,
     ):
         self.name = name
         self.target = target  # 'abs' or 'rise'
@@ -84,6 +85,10 @@ class Variant:
         # behaviour of the deployed model exactly, since p>=0.5 alerts are
         # sigma-independent) and quantile heads ONLY for a per-row sigma.
         self.qsigma = qsigma
+        # Test rows get rain_fc24 = NaN: what production served until the
+        # 2026-10 fix (the rain series was cut to the gauge grid before the
+        # forward sum, so the newest row never had forecast rain).
+        self.fc24_missing_at_serve = fc24_missing_at_serve
 
     def fit_predict(self, X_tr, y_abs_tr, X_te) -> Tuple[np.ndarray, np.ndarray]:
         if not self.use_rain:
@@ -108,6 +113,9 @@ class Variant:
             X_te = X_te.drop(columns=drop)
         elif "rain_fc48" not in X_tr.columns:
             raise ValueError(f"{self.name} requires the rain series")
+        if self.fc24_missing_at_serve and "rain_fc24" in X_te.columns:
+            X_te = X_te.copy()
+            X_te["rain_fc24"] = np.nan
         level_tr = X_tr["level"]
         level_te = X_te["level"].to_numpy()
         y_tr = (y_abs_tr - level_tr) if self.target == "rise" else y_abs_tr
@@ -164,6 +172,14 @@ VARIANTS: Dict[str, Variant] = {
     "rise_rain_qsigma": Variant(
         "rise_rain_qsigma", target="rise", use_rain=True, qsigma=True
     ),
+    # 2026-10-18: the deployed rise_rain model as it was actually served
+    # before the rain_fc24 serving fix (forecast rain missing at test time).
+    "rise_rain_fc24_missing": Variant(
+        "rise_rain_fc24_missing",
+        target="rise",
+        use_rain=True,
+        fc24_missing_at_serve=True,
+    ),
 }
 
 # Dam variants are opt-in by name: they require dam columns that only exist
@@ -176,6 +192,7 @@ DEFAULT_VARIANTS = [
     if not v.use_dam
     and not v.use_fc48
     and not v.qsigma
+    and not v.fc24_missing_at_serve
     and not (v.quantile and v.use_rain)
 ]
 

src/ml/features.py

@@ -209,6 +209,7 @@ def _hours_since_observed(mask_col: pd.Series) -> pd.Series:
 
 
 RAIN_FEATURES = ("rain_6h", "rain_24h", "rain_72h", "rain_fc24")
+FORWARD_RAIN_H = 24  # rain_fc24 window
 
 DAM_FEATURES = ("dam_storage_pct", "dam_storage_pct_d3", "dam_inflow", "dam_outflow")
 # Stations hydrologically downstream of the Mae Ngat confluence (Ping mainstem
@@ -297,9 +298,25 @@ def build_features(
         cols["rain_6h"] = r.rolling(6, min_periods=1).sum()
         cols["rain_24h"] = r.rolling(24, min_periods=1).sum()
         cols["rain_72h"] = r.rolling(72, min_periods=1).sum()
-        # forward sum over (t, t+24]: shift(-1) starts the window at t+1
+        # forward sum over (t, t+24]: shift(-1) starts the window at t+1.
+        # Computed on the grid extended by the 24 h after its last row: the
+        # last row is the one serving predicts from, and cutting the rain
+        # series to the grid made its forecast rain always NaN (unlike every
+        # training row since 2021). Earlier rows keep their values.
+        future = (
+            pd.date_range(idx[-1], periods=FORWARD_RAIN_H + 1, freq="h")[1:]
+            if len(idx)
+            else idx[:0]
+        )
+        r_ext = rain.reindex(idx.append(future))
         cols["rain_fc24"] = (
-            r.shift(-1).iloc[::-1].rolling(24, min_periods=1).sum().iloc[::-1]
+            r_ext.shift(-1)
+            .iloc[::-1]
+            .rolling(FORWARD_RAIN_H, min_periods=1)
+            .sum()
+            .iloc[::-1]
+            .iloc[: len(idx)]
+            .set_axis(idx)
         )
 
     if dam is not None and station in DAM_STATIONS: