fix: backtest/review findings in the flood-ML package
CI/CD Pipeline - Northern Thailand Ping River Monitor / Test Suite (3.11) (push) Failing after 23s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Test Suite (3.12) (push) Failing after 26s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Build Docker Image (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Integration Test with Services (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Deploy to Staging (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Deploy to Production (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Performance Test (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Code Quality (push) Successful in 13s
Documentation / Validate Documentation (push) Failing after 8s
Documentation / Generate API Documentation (push) Successful in 14s
Documentation / Build Sphinx Documentation (push) Successful in 18s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Cleanup (push) Successful in 1s
Documentation / Documentation Summary (push) Successful in 3s

From the adversarial review and threshold backtest (swarm verification):

- predict.py: when a bundle's trained thresholds differ from the current
  config (deploy before retrain), skip its stale classifier heads and
  derive p_warning/p_danger from the regression + sigma against the
  CURRENT thresholds - the dashboard can no longer show contradictory
  old-threshold classifier output next to new-threshold stages
- features.py: decouple the low-coverage regression-label rescue from
  the warning threshold (now the station's own p97.5 level); the old
  coupling silently dropped 34% of P.5's regression training rows and
  cost +46% MAE when its threshold rose
- features.py: P.82 danger 3.80 -> 3.75 (3.80 was above the station's
  8-year maximum of 3.78, so danger could never train or fire)
- data.py / predict.py: anchor models/cache paths to the repo root; the
  relative paths silently returned zero rows when run from another CWD
- annotate P.4A thresholds as low-confidence (11 supporting readings)

47 tests pass. Retrain required for the label-rescue and P.82 changes
to reach the classifier heads.
This commit is contained in:
2026-08-10 15:57:01 +07:00
parent 9cac9c4d2a
commit ecd34177bb
4 changed files with 312 additions and 76 deletions
+72 -15
View File
@@ -20,6 +20,9 @@ from . import features
logger = logging.getLogger(__name__)
# Anchored to the repo root so the API finds trained bundles regardless of CWD.
DEFAULT_MODELS_DIR = Path(__file__).resolve().parents[2] / "models"
DEFAULT_HORIZONS: Tuple[int, ...] = (6, 12, 24)
STALE_AFTER_H = 6.0
HEURISTIC_SIGMA = 0.3
@@ -60,7 +63,9 @@ def _readings_to_long_df(readings_by_station: Dict[str, List[dict]]) -> pd.DataF
}
)
if not rows:
return pd.DataFrame(columns=["timestamp", "station_code", "water_level", "discharge"])
return pd.DataFrame(
columns=["timestamp", "station_code", "water_level", "discharge"]
)
df = pd.DataFrame(rows)
return df.dropna(subset=["timestamp"])
@@ -90,8 +95,12 @@ def _heuristic_forecast(
results = []
for horizon_h in horizons:
predicted_max = max(current_level + rate * horizon_h * 0.7, current_level)
p_warning = _clip_probability(_sigmoid_probability(predicted_max, warn_thr, HEURISTIC_SIGMA))
p_danger = _clip_probability(_sigmoid_probability(predicted_max, danger_thr, HEURISTIC_SIGMA))
p_warning = _clip_probability(
_sigmoid_probability(predicted_max, warn_thr, HEURISTIC_SIGMA)
)
p_danger = _clip_probability(
_sigmoid_probability(predicted_max, danger_thr, HEURISTIC_SIGMA)
)
p_danger = min(p_danger, p_warning)
results.append(
{
@@ -121,12 +130,28 @@ def _model_forecast(
) -> List[dict]:
warn_thr = bundle["thresholds"]["warning"]
danger_thr = bundle["thresholds"]["danger"]
# If thresholds changed since this bundle was trained, its classifier heads
# answer the OLD question (labels for the old levels) while stages/config use
# the new ones — a silent contradiction on the dashboard. Until a retrain,
# answer the current question consistently: use the regression + sigma against
# the configured thresholds and skip the stale heads.
cfg_warn, cfg_danger = features.get_thresholds(station_code)
thresholds_stale = (cfg_warn, cfg_danger) != (warn_thr, danger_thr)
if thresholds_stale:
logger.warning(
f"{station_code}: bundle thresholds ({warn_thr}, {danger_thr}) differ from "
f"configured ({cfg_warn}, {cfg_danger}); using regression-derived probabilities "
"until the model is retrained"
)
warn_thr, danger_thr = cfg_warn, cfg_danger
feature_row = features.build_features(grid, station_code).loc[[as_of]]
expected_columns = bundle["feature_names"]
missing = [c for c in expected_columns if c not in feature_row.columns]
if missing:
logger.error(f"Feature mismatch for {station_code} (missing {missing}); falling back to heuristic")
logger.error(
f"Feature mismatch for {station_code} (missing {missing}); falling back to heuristic"
)
return None
feature_row = feature_row[expected_columns]
@@ -139,13 +164,17 @@ def _model_forecast(
predicted_max = max(float(reg.predict(feature_row)[0]), current_level)
sigma_h = bundle["sigma"].get(horizon_h, HEURISTIC_SIGMA)
warn_head = bundle["heads"].get(f"warn_{horizon_h}")
warn_head = (
None if thresholds_stale else bundle["heads"].get(f"warn_{horizon_h}")
)
if warn_head is not None:
p_warning = float(warn_head.predict_proba(feature_row)[0][1])
else:
p_warning = _sigmoid_probability(predicted_max, warn_thr, sigma_h)
danger_head = bundle["heads"].get(f"danger_{horizon_h}")
danger_head = (
None if thresholds_stale else bundle["heads"].get(f"danger_{horizon_h}")
)
if danger_head is not None:
p_danger = float(danger_head.predict_proba(feature_row)[0][1])
else:
@@ -209,19 +238,35 @@ def _forecast_station(
warn_thr, danger_thr = features.get_thresholds(station_code)
t_minus_3 = as_of - pd.Timedelta(hours=3)
level_t_minus_3 = float(observed_level.loc[t_minus_3]) if t_minus_3 in observed_level.index else None
level_t_minus_3 = (
float(observed_level.loc[t_minus_3])
if t_minus_3 in observed_level.index
else None
)
bundle_path = models_dir / f"flood_{station_code}.joblib"
if not bundle_path.exists() or staleness_h > STALE_AFTER_H:
return _heuristic_forecast(
station_code, as_of, current_level, level_t_minus_3, warn_thr, danger_thr, horizons
station_code,
as_of,
current_level,
level_t_minus_3,
warn_thr,
danger_thr,
horizons,
)
bundle = _load_bundle(bundle_path)
model_results = _model_forecast(station_code, grid, bundle, as_of, current_level)
if model_results is None:
return _heuristic_forecast(
station_code, as_of, current_level, level_t_minus_3, warn_thr, danger_thr, horizons
station_code,
as_of,
current_level,
level_t_minus_3,
warn_thr,
danger_thr,
horizons,
)
# Per-horizon heads that were skipped at train time (e.g. too few positives) still
@@ -233,7 +278,13 @@ def _forecast_station(
else:
filled.extend(
_heuristic_forecast(
station_code, as_of, current_level, level_t_minus_3, warn_thr, danger_thr, (horizon_h,)
station_code,
as_of,
current_level,
level_t_minus_3,
warn_thr,
danger_thr,
(horizon_h,),
)
)
return filled
@@ -241,7 +292,7 @@ def _forecast_station(
def get_forecasts(
readings_by_station: Dict[str, List[dict]],
models_dir: Union[str, Path] = "models",
models_dir: Union[str, Path] = DEFAULT_MODELS_DIR,
now: Optional[Union[datetime.datetime, str]] = None,
) -> List[dict]:
"""Produce flood forecasts for every station present in `readings_by_station`.
@@ -264,7 +315,9 @@ def get_forecasts(
results: List[dict] = []
for station_code in readings_by_station.keys():
try:
results.extend(_forecast_station(station_code, grid, models_dir, now, DEFAULT_HORIZONS))
results.extend(
_forecast_station(station_code, grid, models_dir, now, DEFAULT_HORIZONS)
)
except Exception as error:
logger.error(f"Forecast failed for station {station_code}: {error}")
return results
@@ -272,7 +325,7 @@ def get_forecasts(
def get_latest_forecasts(
db_url: Optional[str] = None,
models_dir: Union[str, Path] = "models",
models_dir: Union[str, Path] = DEFAULT_MODELS_DIR,
hours: int = 336,
) -> List[dict]:
"""Convenience wrapper for web_api: load the latest window from the DB/API and forecast.
@@ -289,10 +342,14 @@ def get_latest_forecasts(
readings_by_station: Dict[str, List[dict]] = {}
if not df_long.empty:
for station_code, group in df_long.groupby("station_code"):
readings_by_station[station_code] = group[["timestamp", "water_level", "discharge"]].to_dict("records")
readings_by_station[station_code] = group[
["timestamp", "water_level", "discharge"]
].to_dict("records")
expected_stations = set(features.UPSTREAM_LEADS.keys())
for missing_station in expected_stations - set(readings_by_station.keys()):
logger.warning(f"No recent data for station {missing_station}; omitting from forecasts")
logger.warning(
f"No recent data for station {missing_station}; omitting from forecasts"
)
return get_forecasts(readings_by_station, models_dir=models_dir)