fix: backtest/review findings in the flood-ML package

From the adversarial review and threshold backtest (swarm verification):

- predict.py: when a bundle's trained thresholds differ from the current
  config (deploy before retrain), skip its stale classifier heads and
  derive p_warning/p_danger from the regression + sigma against the
  CURRENT thresholds - the dashboard can no longer show contradictory
  old-threshold classifier output next to new-threshold stages
- features.py: decouple the low-coverage regression-label rescue from
  the warning threshold (now the station's own p97.5 level); the old
  coupling silently dropped 34% of P.5's regression training rows and
  cost +46% MAE when its threshold rose
- features.py: P.82 danger 3.80 -> 3.75 (3.80 was above the station's
  8-year maximum of 3.78, so danger could never train or fire)
- data.py / predict.py: anchor models/cache paths to the repo root; the
  relative paths silently returned zero rows when run from another CWD
- annotate P.4A thresholds as low-confidence (11 supporting readings)

47 tests pass. Retrain required for the label-rescue and P.82 changes
to reach the classifier heads.

src/ml/train.py

@@ -20,8 +20,16 @@ import joblib
 import numpy as np
 import pandas as pd
 import sklearn
-from sklearn.ensemble import HistGradientBoostingClassifier, HistGradientBoostingRegressor
-from sklearn.metrics import average_precision_score, brier_score_loss, mean_absolute_error, mean_squared_error
+from sklearn.ensemble import (
+    HistGradientBoostingClassifier,
+    HistGradientBoostingRegressor,
+)
+from sklearn.metrics import (
+    average_precision_score,
+    brier_score_loss,
+    mean_absolute_error,
+    mean_squared_error,
+)
 
 from . import features
 from .data import DEFAULT_API_URL, load_measurements, resolve_db_url
@@ -51,7 +59,11 @@ HGB_PARAMS = {
 def _git_short_sha() -> str:
     try:
         result = subprocess.run(
-            ["git", "rev-parse", "--short", "HEAD"], capture_output=True, text=True, timeout=5, check=True
+            ["git", "rev-parse", "--short", "HEAD"],
+            capture_output=True,
+            text=True,
+            timeout=5,
+            check=True,
         )
         sha = result.stdout.strip()
         return sha or "nogit"
@@ -64,12 +76,20 @@ def _make_regressor(overrides: Optional[dict] = None) -> HistGradientBoostingReg
     return HistGradientBoostingRegressor(loss="squared_error", **params)
 
 
-def _make_classifier(overrides: Optional[dict] = None) -> HistGradientBoostingClassifier:
+def _make_classifier(
+    overrides: Optional[dict] = None,
+) -> HistGradientBoostingClassifier:
     params = {**HGB_PARAMS, **(overrides or {})}
     return HistGradientBoostingClassifier(**params)
 
 
-def _safe_fit(estimator, X: pd.DataFrame, y: pd.Series, head_key: str, skipped_heads: Dict[str, str]):
+def _safe_fit(
+    estimator,
+    X: pd.DataFrame,
+    y: pd.Series,
+    head_key: str,
+    skipped_heads: Dict[str, str],
+):
     """Fit an estimator, converting any failure (e.g. HistGradientBoosting's binning
     step rejecting an all-NaN/constant feature column) into a recorded skip rather
     than a station-killing exception."""
@@ -82,7 +102,9 @@ def _safe_fit(estimator, X: pd.DataFrame, y: pd.Series, head_key: str, skipped_h
         return None
 
 
-def _recall_at_far(y_true: np.ndarray, y_score: np.ndarray, target_far: float) -> Optional[float]:
+def _recall_at_far(
+    y_true: np.ndarray, y_score: np.ndarray, target_far: float
+) -> Optional[float]:
     """Recall at the score threshold whose false-positive rate over true negatives is <= target_far."""
     y_true = np.asarray(y_true)
     y_score = np.asarray(y_score)
@@ -98,7 +120,9 @@ def _recall_at_far(y_true: np.ndarray, y_score: np.ndarray, target_far: float) -
     return tp / n_pos
 
 
-def _p_warning_series(head, reg, X: pd.DataFrame, threshold: float, sigma: float) -> pd.Series:
+def _p_warning_series(
+    head, reg, X: pd.DataFrame, threshold: float, sigma: float
+) -> pd.Series:
     """Model score if a classifier head exists, else the sigmoid-derived fallback probability."""
     if head is not None:
         return pd.Series(head.predict_proba(X)[:, 1], index=X.index)
@@ -117,12 +141,24 @@ def _find_events(observed_level: pd.Series, warn_thr: float) -> List[dict]:
             start = t
         elif not is_above and start is not None:
             window = observed_level.loc[start:prev_t]
-            events.append({"crossed_warn_at": start, "peak_time": window.idxmax(), "peak_level": float(window.max())})
+            events.append(
+                {
+                    "crossed_warn_at": start,
+                    "peak_time": window.idxmax(),
+                    "peak_level": float(window.max()),
+                }
+            )
             start = None
         prev_t = t
     if start is not None:
         window = observed_level.loc[start:]
-        events.append({"crossed_warn_at": start, "peak_time": window.idxmax(), "peak_level": float(window.max())})
+        events.append(
+            {
+                "crossed_warn_at": start,
+                "peak_time": window.idxmax(),
+                "peak_level": float(window.max()),
+            }
+        )
     return events
 
 
@@ -142,9 +178,13 @@ def _events_with_lead_time(
     events = _find_events(observed_level_test, warn_thr)
     for event in events:
         first_alert_at = _first_alert_at(p_warning_test, event["crossed_warn_at"])
-        event["first_alert_at"] = first_alert_at.isoformat() if first_alert_at is not None else None
+        event["first_alert_at"] = (
+            first_alert_at.isoformat() if first_alert_at is not None else None
+        )
         if first_alert_at is not None:
-            lead_hours = (event["crossed_warn_at"] - first_alert_at).total_seconds() / 3600.0
+            lead_hours = (
+                event["crossed_warn_at"] - first_alert_at
+            ).total_seconds() / 3600.0
         else:
             lead_hours = None
         event["lead_hours"] = lead_hours
@@ -166,7 +206,10 @@ def train_station(
     """Train every head for one station. Returns (bundle_or_None, station_metrics)."""
     X, Y, meta = features.build_matrix(df_long, station, horizons)
     if meta["n_rows"] < MIN_ROWS_TO_TRAIN:
-        return None, {"status": "failed", "reason": f"only {meta['n_rows']} usable rows (< {MIN_ROWS_TO_TRAIN})"}
+        return None, {
+            "status": "failed",
+            "reason": f"only {meta['n_rows']} usable rows (< {MIN_ROWS_TO_TRAIN})",
+        }
 
     warn_thr, danger_thr = features.get_thresholds(station)
     feature_names = list(X.columns)
@@ -176,7 +219,9 @@ def train_station(
         test_mask = pd.Series(False, index=X.index)
     else:
         train_mask = X.index <= pd.Timestamp(split_train_end)
-        test_mask = (X.index >= pd.Timestamp(split_test_start)) & (X.index <= pd.Timestamp(split_test_end))
+        test_mask = (X.index >= pd.Timestamp(split_test_start)) & (
+            X.index <= pd.Timestamp(split_test_end)
+        )
     X_train, Y_train = X.loc[train_mask], Y.loc[train_mask]
     X_test, Y_test = X.loc[test_mask], Y.loc[test_mask]
     eval_X, eval_Y = (X, Y) if skip_eval else (X_train, Y_train)
@@ -188,7 +233,11 @@ def train_station(
     observed_grid = features.make_hourly_grid(df_long).observed
 
     for h in horizons:
-        max_col, warn_col, danger_col = f"max_level_{h}", f"exceed_warn_{h}", f"exceed_danger_{h}"
+        max_col, warn_col, danger_col = (
+            f"max_level_{h}",
+            f"exceed_warn_{h}",
+            f"exceed_danger_{h}",
+        )
         horizon_metrics: dict = {}
 
         # --- regression head (max level) ---
@@ -215,19 +264,28 @@ def train_station(
                 sigma_h = max(float(np.std(residuals)), MIN_SIGMA)
                 horizon_metrics["n_test"] = int(test_labeled.sum())
                 horizon_metrics["mae"] = float(mean_absolute_error(y_true, y_pred))
-                horizon_metrics["rmse"] = float(np.sqrt(mean_squared_error(y_true, y_pred)))
+                horizon_metrics["rmse"] = float(
+                    np.sqrt(mean_squared_error(y_true, y_pred))
+                )
                 above_2m = y_true >= 2.0
                 horizon_metrics["mae_above_2m"] = (
-                    float(mean_absolute_error(y_true[above_2m], y_pred[above_2m])) if above_2m.any() else None
+                    float(mean_absolute_error(y_true[above_2m], y_pred[above_2m]))
+                    if above_2m.any()
+                    else None
                 )
         sigma[h] = sigma_h
         horizon_metrics["sigma"] = sigma_h
 
         # --- classification heads (warn / danger) ---
         p_warning_test = None
-        for label_name, col, thr in (("warn", warn_col, warn_thr), ("danger", danger_col, danger_thr)):
+        for label_name, col, thr in (
+            ("warn", warn_col, warn_thr),
+            ("danger", danger_col, danger_thr),
+        ):
             train_labeled = eval_Y[col].notna()
-            n_pos = int(eval_Y.loc[train_labeled, col].sum()) if train_labeled.any() else 0
+            n_pos = (
+                int(eval_Y.loc[train_labeled, col].sum()) if train_labeled.any() else 0
+            )
             head_key = f"{label_name}_{h}"
             clf = None
             if n_pos >= MIN_POSITIVES_FOR_CLASSIFIER:
@@ -239,21 +297,37 @@ def train_station(
                     skipped_heads,
                 )
             else:
-                skipped_heads[head_key] = f"only {n_pos} positives in train span (< {MIN_POSITIVES_FOR_CLASSIFIER})"
+                skipped_heads[
+                    head_key
+                ] = f"only {n_pos} positives in train span (< {MIN_POSITIVES_FOR_CLASSIFIER})"
             heads[head_key] = clf
 
             if not skip_eval:
                 test_labeled = Y_test[col].notna()
                 horizon_metrics[f"base_rate_{label_name}"] = (
-                    float(Y_test.loc[test_labeled, col].mean()) if test_labeled.any() else None
+                    float(Y_test.loc[test_labeled, col].mean())
+                    if test_labeled.any()
+                    else None
                 )
-                if clf is not None and test_labeled.sum() > 0 and Y_test.loc[test_labeled, col].nunique() > 1:
+                if (
+                    clf is not None
+                    and test_labeled.sum() > 0
+                    and Y_test.loc[test_labeled, col].nunique() > 1
+                ):
                     y_true = Y_test.loc[test_labeled, col]
                     y_score = clf.predict_proba(X_test.loc[test_labeled])[:, 1]
-                    horizon_metrics[f"pr_auc_{label_name}"] = float(average_precision_score(y_true, y_score))
-                    horizon_metrics[f"brier_{label_name}"] = float(brier_score_loss(y_true, y_score))
-                    horizon_metrics[f"recall_{label_name}_at_far1pct"] = _recall_at_far(y_true, y_score, 0.01)
-                    horizon_metrics[f"recall_{label_name}_at_far5pct"] = _recall_at_far(y_true, y_score, 0.05)
+                    horizon_metrics[f"pr_auc_{label_name}"] = float(
+                        average_precision_score(y_true, y_score)
+                    )
+                    horizon_metrics[f"brier_{label_name}"] = float(
+                        brier_score_loss(y_true, y_score)
+                    )
+                    horizon_metrics[f"recall_{label_name}_at_far1pct"] = _recall_at_far(
+                        y_true, y_score, 0.01
+                    )
+                    horizon_metrics[f"recall_{label_name}_at_far5pct"] = _recall_at_far(
+                        y_true, y_score, 0.05
+                    )
                 else:
                     horizon_metrics[f"pr_auc_{label_name}"] = None
                     horizon_metrics[f"brier_{label_name}"] = None
@@ -269,8 +343,12 @@ def train_station(
         if not skip_eval and reg is not None and p_warning_test is not None:
             observed_test_level = observed_grid.get((station, "water_level"))
             if observed_test_level is not None:
-                observed_test_level = observed_test_level.loc[observed_test_level.index.isin(X_test.index)]
-                per_horizon[h]["events"] = _events_with_lead_time(observed_test_level, warn_thr, p_warning_test)
+                observed_test_level = observed_test_level.loc[
+                    observed_test_level.index.isin(X_test.index)
+                ]
+                per_horizon[h]["events"] = _events_with_lead_time(
+                    observed_test_level, warn_thr, p_warning_test
+                )
 
     # --- full refit on the ENTIRE record for the deployed artifact ---
     # This may include/exclude different heads than the eval-phase gate above (the
@@ -278,11 +356,21 @@ def train_station(
     # skipped_heads must reflect what actually ends up in the saved bundle.
     final_heads: Dict[str, object] = {}
     for h in horizons:
-        max_col, warn_col, danger_col = f"max_level_{h}", f"exceed_warn_{h}", f"exceed_danger_{h}"
+        max_col, warn_col, danger_col = (
+            f"max_level_{h}",
+            f"exceed_warn_{h}",
+            f"exceed_danger_{h}",
+        )
         head_key = f"max_{h}"
         labeled = Y[max_col].notna()
         if labeled.sum() >= MIN_ROWS_FOR_HEAD:
-            reg = _safe_fit(_make_regressor(hgb_overrides), X.loc[labeled], Y.loc[labeled, max_col], head_key, skipped_heads)
+            reg = _safe_fit(
+                _make_regressor(hgb_overrides),
+                X.loc[labeled],
+                Y.loc[labeled, max_col],
+                head_key,
+                skipped_heads,
+            )
             final_heads[head_key] = reg
             if reg is not None:
                 skipped_heads.pop(head_key, None)
@@ -296,13 +384,19 @@ def train_station(
             n_pos = int(Y.loc[train_labeled, col].sum()) if train_labeled.any() else 0
             if n_pos >= MIN_POSITIVES_FOR_CLASSIFIER:
                 clf = _safe_fit(
-                    _make_classifier(hgb_overrides), X.loc[train_labeled], Y.loc[train_labeled, col], head_key, skipped_heads
+                    _make_classifier(hgb_overrides),
+                    X.loc[train_labeled],
+                    Y.loc[train_labeled, col],
+                    head_key,
+                    skipped_heads,
                 )
                 final_heads[head_key] = clf
                 if clf is not None:
                     skipped_heads.pop(head_key, None)
             else:
-                skipped_heads[head_key] = f"only {n_pos} positives in train span (< {MIN_POSITIVES_FOR_CLASSIFIER})"
+                skipped_heads[
+                    head_key
+                ] = f"only {n_pos} positives in train span (< {MIN_POSITIVES_FOR_CLASSIFIER})"
                 final_heads[head_key] = None
 
     bundle = {
@@ -345,7 +439,11 @@ def train_all(
             continue
         try:
             bundle, station_metrics = train_station(
-                df_long, station, horizons, skip_eval=skip_eval, hgb_overrides=hgb_overrides
+                df_long,
+                station,
+                horizons,
+                skip_eval=skip_eval,
+                hgb_overrides=hgb_overrides,
             )
             if bundle is None:
                 logger.warning(f"{station}: failed ({station_metrics.get('reason')})")
@@ -377,16 +475,32 @@ def train_all(
 
 
 def main(argv: Optional[List[str]] = None) -> None:
-    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
+    logging.basicConfig(
+        level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s"
+    )
 
-    parser = argparse.ArgumentParser(description="Train Ping River flood forecast models")
-    parser.add_argument("--stations", default="all", help="'all' or a comma-separated list of station codes")
+    parser = argparse.ArgumentParser(
+        description="Train Ping River flood forecast models"
+    )
+    parser.add_argument(
+        "--stations",
+        default="all",
+        help="'all' or a comma-separated list of station codes",
+    )
     parser.add_argument("--models-dir", default="models")
     parser.add_argument("--db-url", default=None)
     parser.add_argument("--api-url", default=DEFAULT_API_URL)
-    parser.add_argument("--skip-eval", action="store_true", help="Refit-only fast path; skip Split B evaluation")
-    parser.add_argument("--start", default=None, help="ISO date; earliest measurement to load")
-    parser.add_argument("--end", default=None, help="ISO date; latest measurement to load")
+    parser.add_argument(
+        "--skip-eval",
+        action="store_true",
+        help="Refit-only fast path; skip Split B evaluation",
+    )
+    parser.add_argument(
+        "--start", default=None, help="ISO date; earliest measurement to load"
+    )
+    parser.add_argument(
+        "--end", default=None, help="ISO date; latest measurement to load"
+    )
     args = parser.parse_args(argv)
 
     if args.stations == "all":
@@ -399,15 +513,25 @@ def main(argv: Optional[List[str]] = None) -> None:
 
     logger.info(f"Loading measurements for {len(stations)} stations...")
     df_long = load_measurements(
-        db_url=resolve_db_url(args.db_url), stations=None, start=start, end=end, api_url=args.api_url
+        db_url=resolve_db_url(args.db_url),
+        stations=None,
+        start=start,
+        end=end,
+        api_url=args.api_url,
+    )
+    logger.info(
+        f"Loaded {len(df_long)} rows spanning {df_long['timestamp'].min()} .. {df_long['timestamp'].max()}"
     )
-    logger.info(f"Loaded {len(df_long)} rows spanning {df_long['timestamp'].min()} .. {df_long['timestamp'].max()}")
 
     metrics_payload = train_all(
         df_long, stations, models_dir=Path(args.models_dir), skip_eval=args.skip_eval
     )
-    trained = sum(1 for s in metrics_payload["stations"].values() if s["status"] == "trained")
-    logger.info(f"Done: {trained}/{len(stations)} stations trained. metrics.json written to {args.models_dir}")
+    trained = sum(
+        1 for s in metrics_payload["stations"].values() if s["status"] == "trained"
+    )
+    logger.info(
+        f"Done: {trained}/{len(stations)} stations trained. metrics.json written to {args.models_dir}"
+    )
 
 
 if __name__ == "__main__":