feat: hgb-v3 — Open-Meteo rain features clear the 12h warning gate

The rolling-origin harness (models/eval_rain.json) showed catchment rain
halving flood-year Brier scores, cutting flood-regime MAE 20-40%, and
extending the hard 2024 leads (+6h -> +11h at P.1, +10h -> +19h at
P.103). Ported: train_all loads the catchment-mean series (use_rain /
--no-rain to opt out; without it bundles train as v2), predict fetches
live rain hourly and passes an empty series on failure so rain-trained
bundles serve with NaN features instead of tripping the feature guard,
and the leader worker persists hourly per-point + catchment-mean rows to
a new openmeteo_rain table.

Regenerated backtest: the 2024 record flood now gets a 13-HOUR WARNING
(alert 04:00 vs 17:00 crossing, river at 2.9m at alert time) — the >=12h
acceptance gate PASSES for the first time. Journey on that crossing:
v1 -18h, v2 +6h, v3 +13h. The marginal 2025 double-crest trades its
artifact +46h latch for a calibrated +2h with zero false alarms. P.1
MAE 4.9/7.2/8.7 cm at 6/12/24h. Docs updated throughout.

src/ml/predict.py

@@ -127,6 +127,7 @@ def _model_forecast(
     bundle: dict,
     as_of: pd.Timestamp,
     current_level: float,
+    rain: Optional[pd.Series] = None,
 ) -> List[dict]:
     warn_thr = bundle["thresholds"]["warning"]
     danger_thr = bundle["thresholds"]["danger"]
@@ -145,7 +146,7 @@ def _model_forecast(
         )
         warn_thr, danger_thr = cfg_warn, cfg_danger
 
-    feature_row = features.build_features(grid, station_code).loc[[as_of]]
+    feature_row = features.build_features(grid, station_code, rain=rain).loc[[as_of]]
     expected_columns = bundle["feature_names"]
     missing = [c for c in expected_columns if c not in feature_row.columns]
     if missing:
@@ -229,6 +230,7 @@ def _forecast_station(
     models_dir: Path,
     now: pd.Timestamp,
     horizons: Tuple[int, ...],
+    rain: Optional[pd.Series] = None,
 ) -> List[dict]:
     level_col = (station_code, "water_level")
     if level_col not in grid.observed.columns:
@@ -264,7 +266,9 @@ def _forecast_station(
         )
 
     bundle = _load_bundle(bundle_path)
-    model_results = _model_forecast(station_code, grid, bundle, as_of, current_level)
+    model_results = _model_forecast(
+        station_code, grid, bundle, as_of, current_level, rain=rain
+    )
     if model_results is None:
         return _heuristic_forecast(
             station_code,
@@ -301,6 +305,7 @@ def get_forecasts(
     readings_by_station: Dict[str, List[dict]],
     models_dir: Union[str, Path] = DEFAULT_MODELS_DIR,
     now: Optional[Union[datetime.datetime, str]] = None,
+    rain: Optional[pd.Series] = None,
 ) -> List[dict]:
     """Produce flood forecasts for every station present in `readings_by_station`.
 
@@ -323,7 +328,9 @@ def get_forecasts(
     for station_code in readings_by_station.keys():
         try:
             results.extend(
-                _forecast_station(station_code, grid, models_dir, now, DEFAULT_HORIZONS)
+                _forecast_station(
+                    station_code, grid, models_dir, now, DEFAULT_HORIZONS, rain=rain
+                )
             )
         except Exception as error:
             logger.error(f"Forecast failed for station {station_code}: {error}")
@@ -359,4 +366,14 @@ def get_latest_forecasts(
             f"No recent data for station {missing_station}; omitting from forecasts"
         )
 
-    return get_forecasts(readings_by_station, models_dir=models_dir)
+    # Live rain: trailing days + next-48h forecast. On fetch failure pass an
+    # EMPTY series (not None) so rain-trained bundles still find their columns
+    # (as NaN) and serve model output instead of tripping the feature guard.
+    from .rain import serving_series
+
+    rain = serving_series()
+    if rain is None:
+        logger.warning("live rain unavailable; rain features will be NaN")
+        rain = pd.Series(dtype=float)
+
+    return get_forecasts(readings_by_station, models_dir=models_dir, rain=rain)

src/ml/rain.py

@@ -160,3 +160,60 @@ def serving_series() -> Optional[pd.Series]:
     except Exception as error:
         logger.warning(f"Open-Meteo forecast fetch failed: {error}")
         return None
+
+
+def save_to_db(df: pd.DataFrame, engine, db_type: str) -> int:
+    """Upsert per-point + catchment-mean hourly rain into openmeteo_rain.
+
+    Called by the leader worker's hourly precompute with the live forecast
+    frame, so the DB accumulates both what fell (past rows are the model
+    analysis) and what was forecast (future rows, overwritten as they become
+    past). The ML training path reads Open-Meteo's own archive, not this
+    table — this is for dashboards, SQL analysis, and source independence.
+    """
+    if df is None or df.empty:
+        return 0
+    from sqlalchemy import text
+
+    point_cols = [p[0] for p in CATCHMENT_POINTS]
+    ddl_cols = ", ".join(f"{c} NUMERIC(6,2)" for c in point_cols)
+    ddl = (
+        "CREATE TABLE IF NOT EXISTS openmeteo_rain ("
+        "timestamp TIMESTAMP PRIMARY KEY, "
+        f"{ddl_cols}, catchment_mean NUMERIC(6,2), "
+        "created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP)"
+    )
+    cols = ["timestamp"] + point_cols + ["catchment_mean"]
+    placeholders = ", ".join(f":{c}" for c in cols)
+    updates = ", ".join(
+        f"{c} = "
+        + (f"VALUES({c})" if db_type == "mysql" else f"EXCLUDED.{c}")
+        for c in cols[1:]
+    )
+    if db_type == "mysql":
+        sql = (
+            f"INSERT INTO openmeteo_rain ({', '.join(cols)}) VALUES ({placeholders}) "
+            f"ON DUPLICATE KEY UPDATE {updates}"
+        )
+    else:
+        sql = (
+            f"INSERT INTO openmeteo_rain ({', '.join(cols)}) VALUES ({placeholders}) "
+            f"ON CONFLICT (timestamp) DO UPDATE SET {updates}"
+        )
+    mean = df.mean(axis=1)
+    params = [
+        {
+            "timestamp": ts.to_pydatetime(),
+            **{c: (None if pd.isna(row[c]) else float(row[c])) for c in point_cols},
+            "catchment_mean": None if pd.isna(mean.loc[ts]) else float(mean.loc[ts]),
+        }
+        for ts, row in df.iterrows()
+    ]
+    try:
+        with engine.begin() as conn:
+            conn.execute(text(ddl))
+            conn.execute(text(sql), params)
+        return len(params)
+    except Exception as error:
+        logger.error(f"openmeteo_rain save failed: {error}")
+        return 0

src/ml/train.py

@@ -203,9 +203,10 @@ def train_station(
     split_train_end: str = SPLIT_B_TRAIN_END,
     split_test_start: str = SPLIT_B_TEST_START,
     split_test_end: str = SPLIT_B_TEST_END,
+    rain: Optional[pd.Series] = None,
 ) -> Tuple[Optional[dict], dict]:
     """Train every head for one station. Returns (bundle_or_None, station_metrics)."""
-    X, Y, meta = features.build_matrix(df_long, station, horizons)
+    X, Y, meta = features.build_matrix(df_long, station, horizons, rain=rain)
     if meta["n_rows"] < MIN_ROWS_TO_TRAIN:
         return None, {
             "status": "failed",
@@ -411,10 +412,12 @@ def train_station(
                 ] = f"only {n_pos} positives in train span (< {MIN_POSITIVES_FOR_CLASSIFIER})"
                 final_heads[head_key] = None
 
+    # v3 = rise target + Open-Meteo rain features; v2 = rise target only
+    version_prefix = "hgb-v3" if "rain_24h" in feature_names else "hgb-v2"
     bundle = {
         "station_code": station,
-        "model_version": f"hgb-v2+{_git_short_sha()}",
-        # v2: regression heads predict the RISE over the current level; the
+        "model_version": f"{version_prefix}+{_git_short_sha()}",
+        # v2+: regression heads predict the RISE over the current level; the
         # serving side must add the level back. Old v1 bundles lack this key.
         "regression_target": "rise",
         "trained_at": datetime.datetime.now().isoformat(),
@@ -439,11 +442,28 @@ def train_all(
     models_dir: Path = Path("models"),
     skip_eval: bool = False,
     hgb_overrides: Optional[dict] = None,
+    use_rain: bool = True,
 ) -> dict:
     """Train and save every requested station's models. Returns the metrics.json payload."""
     models_dir = Path(models_dir)
     models_dir.mkdir(parents=True, exist_ok=True)
-    model_version = f"hgb-v2+{_git_short_sha()}"
+
+    # Catchment rain (Open-Meteo archive, 2021+). Optional: without it the
+    # models train as v2 (no rain columns) and still serve correctly.
+    rain_series = None
+    if use_rain:
+        try:
+            from . import rain as rain_mod
+
+            rain_series = rain_mod.catchment_mean(rain_mod.load_history())
+        except Exception as error:
+            logger.warning(f"rain history unavailable, training without it: {error}")
+    if rain_series is not None:
+        logger.info(
+            f"rain series: {rain_series.index.min()} .. {rain_series.index.max()}"
+        )
+    version_prefix = "hgb-v3" if rain_series is not None else "hgb-v2"
+    model_version = f"{version_prefix}+{_git_short_sha()}"
 
     station_results: Dict[str, dict] = {}
     for station in stations:
@@ -459,6 +479,7 @@ def train_all(
                 horizons,
                 skip_eval=skip_eval,
                 hgb_overrides=hgb_overrides,
+                rain=rain_series,
             )
             if bundle is None:
                 logger.warning(f"{station}: failed ({station_metrics.get('reason')})")
@@ -516,6 +537,11 @@ def main(argv: Optional[List[str]] = None) -> None:
     parser.add_argument(
         "--end", default=None, help="ISO date; latest measurement to load"
     )
+    parser.add_argument(
+        "--no-rain",
+        action="store_true",
+        help="train without the Open-Meteo rain features (v2-style bundles)",
+    )
     args = parser.parse_args(argv)
 
     if args.stations == "all":
@@ -539,7 +565,11 @@ def main(argv: Optional[List[str]] = None) -> None:
     )
 
     metrics_payload = train_all(
-        df_long, stations, models_dir=Path(args.models_dir), skip_eval=args.skip_eval
+        df_long,
+        stations,
+        models_dir=Path(args.models_dir),
+        skip_eval=args.skip_eval,
+        use_rain=not args.no_rain,
     )
     trained = sum(
         1 for s in metrics_payload["stations"].values() if s["status"] == "trained"

src/web_api.py

@@ -264,6 +264,27 @@ app.add_middleware(
 )
 
 
+async def _persist_rain():
+    """Save the latest Open-Meteo rain frame into openmeteo_rain (leader only)."""
+    store = app_state.get("forecast_store")  # reuse its SQL engine
+    if not store:
+        return
+    try:
+        from .ml import rain as rain_mod
+
+        def fetch_and_save():
+            frame = rain_mod.fetch_forecast()
+            if not store.engine and not store.connect():
+                return 0
+            return rain_mod.save_to_db(frame, store.engine, store.db_type)
+
+        saved = await asyncio.to_thread(fetch_and_save)
+        if saved:
+            logger.info(f"openmeteo_rain: {saved} hourly rows upserted")
+    except Exception as e:
+        logger.warning(f"rain persistence failed: {e}")
+
+
 async def _precompute_forecasts():
     """Refresh the forecast cache and persist the issued forecasts (leader only)."""
     try:
@@ -351,6 +372,10 @@ async def background_scraping_task():
                         except Exception as e:
                             logger.error(f"HII collection failed: {e}")
 
+                    # Persist the Open-Meteo catchment rain (observed tail +
+                    # 48h forecast) so the DB carries the weather context too
+                    await _persist_rain()
+
                     # Precompute forecasts on fresh data: primes the response
                     # cache (user requests never pay for inference) and records
                     # what the model predicted for later predicted-vs-actual