feat: refuse silent v3->v2 downgrade; monthly retrain timer with staged promote

train_all() now raises RainUnavailableError when use_rain=True and the
Open-Meteo history cannot be loaded, instead of logging a warning and
writing gauge-only (v2) bundles over the deployed v3 set -- which is what
the 2026-09-01 server retrain did unnoticed. --no-rain remains the explicit
way to get v2. CLI exits 2 with a one-line error. Three tests cover the
guard, the opt-out, and the v3 happy path.

scripts/retrain.sh trains into models/.staging, refuses to promote unless
metrics.json shows hgb-v3+ and >=14 trained stations, then renames bundles
into place (previous generation kept in models/.previous). No API restart:
predict.py reloads by mtime on the hourly precompute.

water-monitor-retrain.{service,timer}: 1st of each month 03:30, Persistent,
OMP_NUM_THREADS=4, Nice=15, same sandbox as the API unit. install.sh now
does `uv sync` into .venv (one env rule; removes a stale venv/) and enables
the timer. water-monitor.service in the repo matched neither the deployed
unit nor the uv env; it now does (run.py --web-api, .venv, EnvironmentFile).

src/ml/train.py

@@ -45,6 +45,14 @@ MIN_SIGMA = 0.15
 MIN_ROWS_TO_TRAIN = 200
 MIN_ROWS_FOR_HEAD = 50
 
+
+class RainUnavailableError(RuntimeError):
+    """Raised when a rain-enabled training run cannot obtain the rain series.
+
+    Training would otherwise fall through to gauge-only (v2) bundles and
+    overwrite the deployed v3 artifacts without anyone noticing.
+    """
+
 HGB_PARAMS = {
     "max_iter": 300,
     "learning_rate": 0.06,
@@ -456,8 +464,13 @@ def train_all(
     models_dir = Path(models_dir)
     models_dir.mkdir(parents=True, exist_ok=True)
 
-    # Catchment rain (Open-Meteo archive, 2021+). Optional: without it the
-    # models train as v2 (no rain columns) and still serve correctly.
+    # Catchment rain (Open-Meteo archive, 2021+). A rain-less run produces v2
+    # bundles that serve fine but have measurably less flood lead (the 2024
+    # record flood: 13 h early with rain vs 18 h late without). The 2026-09-01
+    # server retrain hit exactly that -- the archive fetch failed on a checkout
+    # with no models/cache/ and the run quietly wrote v2 over v3. So the
+    # downgrade is now an error unless the caller opts out with use_rain=False
+    # (the --no-rain flag), which is the only way to get v2 deliberately.
     rain_series = None
     if use_rain:
         try:
@@ -465,7 +478,19 @@ def train_all(
 
             rain_series = rain_mod.catchment_mean(rain_mod.load_history())
         except Exception as error:
-            logger.warning(f"rain history unavailable, training without it: {error}")
+            raise RainUnavailableError(
+                f"rain history unavailable ({error}); refusing to silently "
+                "downgrade to v2 bundles -- fix Open-Meteo access or restore "
+                "models/cache/rain_openmeteo.csv.gz, or pass --no-rain to "
+                "train gauge-only bundles on purpose"
+            ) from error
+        if rain_series is None:
+            raise RainUnavailableError(
+                "rain history unavailable (Open-Meteo archive unreachable and "
+                "no models/cache/rain_openmeteo.csv.gz); refusing to silently "
+                "downgrade to v2 bundles -- fix access, restore the cache file, "
+                "or pass --no-rain to train gauge-only bundles on purpose"
+            )
     if rain_series is not None:
         logger.info(
             f"rain series: {rain_series.index.min()} .. {rain_series.index.max()}"
@@ -584,7 +609,9 @@ def main(argv: Optional[List[str]] = None) -> None:
     parser.add_argument(
         "--no-rain",
         action="store_true",
-        help="train without the Open-Meteo rain features (v2-style bundles)",
+        help="DELIBERATELY train without the Open-Meteo rain features "
+        "(v2-style bundles). Without this flag a missing rain series aborts "
+        "the run instead of quietly downgrading the deployed model",
     )
     parser.add_argument(
         "--dam",
@@ -627,9 +654,21 @@ def main(argv: Optional[List[str]] = None) -> None:
         1 for s in metrics_payload["stations"].values() if s["status"] == "trained"
     )
     logger.info(
-        f"Done: {trained}/{len(stations)} stations trained. metrics.json written to {args.models_dir}"
+        f"Done: {trained}/{len(stations)} stations trained "
+        f"({metrics_payload['model_version']}). "
+        f"metrics.json written to {args.models_dir}"
     )
 
 
+def cli() -> int:
+    """Console entry: RainUnavailableError becomes a one-line error, exit 2."""
+    try:
+        main()
+    except RainUnavailableError as error:
+        logger.error(str(error))
+        return 2
+    return 0
+
+
 if __name__ == "__main__":
-    main()
+    raise SystemExit(cli())