fix: backtest/review findings in the flood-ML package
CI/CD Pipeline - Northern Thailand Ping River Monitor / Test Suite (3.11) (push) Failing after 23s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Test Suite (3.12) (push) Failing after 26s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Build Docker Image (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Integration Test with Services (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Deploy to Staging (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Deploy to Production (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Performance Test (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Code Quality (push) Successful in 13s
Documentation / Validate Documentation (push) Failing after 8s
Documentation / Generate API Documentation (push) Successful in 14s
Documentation / Build Sphinx Documentation (push) Successful in 18s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Cleanup (push) Successful in 1s
Documentation / Documentation Summary (push) Successful in 3s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Test Suite (3.11) (push) Failing after 23s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Test Suite (3.12) (push) Failing after 26s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Build Docker Image (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Integration Test with Services (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Deploy to Staging (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Deploy to Production (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Performance Test (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Code Quality (push) Successful in 13s
Documentation / Validate Documentation (push) Failing after 8s
Documentation / Generate API Documentation (push) Successful in 14s
Documentation / Build Sphinx Documentation (push) Successful in 18s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Cleanup (push) Successful in 1s
Documentation / Documentation Summary (push) Successful in 3s
From the adversarial review and threshold backtest (swarm verification): - predict.py: when a bundle's trained thresholds differ from the current config (deploy before retrain), skip its stale classifier heads and derive p_warning/p_danger from the regression + sigma against the CURRENT thresholds - the dashboard can no longer show contradictory old-threshold classifier output next to new-threshold stages - features.py: decouple the low-coverage regression-label rescue from the warning threshold (now the station's own p97.5 level); the old coupling silently dropped 34% of P.5's regression training rows and cost +46% MAE when its threshold rose - features.py: P.82 danger 3.80 -> 3.75 (3.80 was above the station's 8-year maximum of 3.78, so danger could never train or fire) - data.py / predict.py: anchor models/cache paths to the repo root; the relative paths silently returned zero rows when run from another CWD - annotate P.4A thresholds as low-confidence (11 supporting readings) 47 tests pass. Retrain required for the label-rescue and P.82 changes to reach the classifier heads.
This commit is contained in:
+72
-15
@@ -20,6 +20,9 @@ from . import features
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Anchored to the repo root so the API finds trained bundles regardless of CWD.
|
||||
DEFAULT_MODELS_DIR = Path(__file__).resolve().parents[2] / "models"
|
||||
|
||||
DEFAULT_HORIZONS: Tuple[int, ...] = (6, 12, 24)
|
||||
STALE_AFTER_H = 6.0
|
||||
HEURISTIC_SIGMA = 0.3
|
||||
@@ -60,7 +63,9 @@ def _readings_to_long_df(readings_by_station: Dict[str, List[dict]]) -> pd.DataF
|
||||
}
|
||||
)
|
||||
if not rows:
|
||||
return pd.DataFrame(columns=["timestamp", "station_code", "water_level", "discharge"])
|
||||
return pd.DataFrame(
|
||||
columns=["timestamp", "station_code", "water_level", "discharge"]
|
||||
)
|
||||
df = pd.DataFrame(rows)
|
||||
return df.dropna(subset=["timestamp"])
|
||||
|
||||
@@ -90,8 +95,12 @@ def _heuristic_forecast(
|
||||
results = []
|
||||
for horizon_h in horizons:
|
||||
predicted_max = max(current_level + rate * horizon_h * 0.7, current_level)
|
||||
p_warning = _clip_probability(_sigmoid_probability(predicted_max, warn_thr, HEURISTIC_SIGMA))
|
||||
p_danger = _clip_probability(_sigmoid_probability(predicted_max, danger_thr, HEURISTIC_SIGMA))
|
||||
p_warning = _clip_probability(
|
||||
_sigmoid_probability(predicted_max, warn_thr, HEURISTIC_SIGMA)
|
||||
)
|
||||
p_danger = _clip_probability(
|
||||
_sigmoid_probability(predicted_max, danger_thr, HEURISTIC_SIGMA)
|
||||
)
|
||||
p_danger = min(p_danger, p_warning)
|
||||
results.append(
|
||||
{
|
||||
@@ -121,12 +130,28 @@ def _model_forecast(
|
||||
) -> List[dict]:
|
||||
warn_thr = bundle["thresholds"]["warning"]
|
||||
danger_thr = bundle["thresholds"]["danger"]
|
||||
# If thresholds changed since this bundle was trained, its classifier heads
|
||||
# answer the OLD question (labels for the old levels) while stages/config use
|
||||
# the new ones — a silent contradiction on the dashboard. Until a retrain,
|
||||
# answer the current question consistently: use the regression + sigma against
|
||||
# the configured thresholds and skip the stale heads.
|
||||
cfg_warn, cfg_danger = features.get_thresholds(station_code)
|
||||
thresholds_stale = (cfg_warn, cfg_danger) != (warn_thr, danger_thr)
|
||||
if thresholds_stale:
|
||||
logger.warning(
|
||||
f"{station_code}: bundle thresholds ({warn_thr}, {danger_thr}) differ from "
|
||||
f"configured ({cfg_warn}, {cfg_danger}); using regression-derived probabilities "
|
||||
"until the model is retrained"
|
||||
)
|
||||
warn_thr, danger_thr = cfg_warn, cfg_danger
|
||||
|
||||
feature_row = features.build_features(grid, station_code).loc[[as_of]]
|
||||
expected_columns = bundle["feature_names"]
|
||||
missing = [c for c in expected_columns if c not in feature_row.columns]
|
||||
if missing:
|
||||
logger.error(f"Feature mismatch for {station_code} (missing {missing}); falling back to heuristic")
|
||||
logger.error(
|
||||
f"Feature mismatch for {station_code} (missing {missing}); falling back to heuristic"
|
||||
)
|
||||
return None
|
||||
feature_row = feature_row[expected_columns]
|
||||
|
||||
@@ -139,13 +164,17 @@ def _model_forecast(
|
||||
predicted_max = max(float(reg.predict(feature_row)[0]), current_level)
|
||||
sigma_h = bundle["sigma"].get(horizon_h, HEURISTIC_SIGMA)
|
||||
|
||||
warn_head = bundle["heads"].get(f"warn_{horizon_h}")
|
||||
warn_head = (
|
||||
None if thresholds_stale else bundle["heads"].get(f"warn_{horizon_h}")
|
||||
)
|
||||
if warn_head is not None:
|
||||
p_warning = float(warn_head.predict_proba(feature_row)[0][1])
|
||||
else:
|
||||
p_warning = _sigmoid_probability(predicted_max, warn_thr, sigma_h)
|
||||
|
||||
danger_head = bundle["heads"].get(f"danger_{horizon_h}")
|
||||
danger_head = (
|
||||
None if thresholds_stale else bundle["heads"].get(f"danger_{horizon_h}")
|
||||
)
|
||||
if danger_head is not None:
|
||||
p_danger = float(danger_head.predict_proba(feature_row)[0][1])
|
||||
else:
|
||||
@@ -209,19 +238,35 @@ def _forecast_station(
|
||||
|
||||
warn_thr, danger_thr = features.get_thresholds(station_code)
|
||||
t_minus_3 = as_of - pd.Timedelta(hours=3)
|
||||
level_t_minus_3 = float(observed_level.loc[t_minus_3]) if t_minus_3 in observed_level.index else None
|
||||
level_t_minus_3 = (
|
||||
float(observed_level.loc[t_minus_3])
|
||||
if t_minus_3 in observed_level.index
|
||||
else None
|
||||
)
|
||||
|
||||
bundle_path = models_dir / f"flood_{station_code}.joblib"
|
||||
if not bundle_path.exists() or staleness_h > STALE_AFTER_H:
|
||||
return _heuristic_forecast(
|
||||
station_code, as_of, current_level, level_t_minus_3, warn_thr, danger_thr, horizons
|
||||
station_code,
|
||||
as_of,
|
||||
current_level,
|
||||
level_t_minus_3,
|
||||
warn_thr,
|
||||
danger_thr,
|
||||
horizons,
|
||||
)
|
||||
|
||||
bundle = _load_bundle(bundle_path)
|
||||
model_results = _model_forecast(station_code, grid, bundle, as_of, current_level)
|
||||
if model_results is None:
|
||||
return _heuristic_forecast(
|
||||
station_code, as_of, current_level, level_t_minus_3, warn_thr, danger_thr, horizons
|
||||
station_code,
|
||||
as_of,
|
||||
current_level,
|
||||
level_t_minus_3,
|
||||
warn_thr,
|
||||
danger_thr,
|
||||
horizons,
|
||||
)
|
||||
|
||||
# Per-horizon heads that were skipped at train time (e.g. too few positives) still
|
||||
@@ -233,7 +278,13 @@ def _forecast_station(
|
||||
else:
|
||||
filled.extend(
|
||||
_heuristic_forecast(
|
||||
station_code, as_of, current_level, level_t_minus_3, warn_thr, danger_thr, (horizon_h,)
|
||||
station_code,
|
||||
as_of,
|
||||
current_level,
|
||||
level_t_minus_3,
|
||||
warn_thr,
|
||||
danger_thr,
|
||||
(horizon_h,),
|
||||
)
|
||||
)
|
||||
return filled
|
||||
@@ -241,7 +292,7 @@ def _forecast_station(
|
||||
|
||||
def get_forecasts(
|
||||
readings_by_station: Dict[str, List[dict]],
|
||||
models_dir: Union[str, Path] = "models",
|
||||
models_dir: Union[str, Path] = DEFAULT_MODELS_DIR,
|
||||
now: Optional[Union[datetime.datetime, str]] = None,
|
||||
) -> List[dict]:
|
||||
"""Produce flood forecasts for every station present in `readings_by_station`.
|
||||
@@ -264,7 +315,9 @@ def get_forecasts(
|
||||
results: List[dict] = []
|
||||
for station_code in readings_by_station.keys():
|
||||
try:
|
||||
results.extend(_forecast_station(station_code, grid, models_dir, now, DEFAULT_HORIZONS))
|
||||
results.extend(
|
||||
_forecast_station(station_code, grid, models_dir, now, DEFAULT_HORIZONS)
|
||||
)
|
||||
except Exception as error:
|
||||
logger.error(f"Forecast failed for station {station_code}: {error}")
|
||||
return results
|
||||
@@ -272,7 +325,7 @@ def get_forecasts(
|
||||
|
||||
def get_latest_forecasts(
|
||||
db_url: Optional[str] = None,
|
||||
models_dir: Union[str, Path] = "models",
|
||||
models_dir: Union[str, Path] = DEFAULT_MODELS_DIR,
|
||||
hours: int = 336,
|
||||
) -> List[dict]:
|
||||
"""Convenience wrapper for web_api: load the latest window from the DB/API and forecast.
|
||||
@@ -289,10 +342,14 @@ def get_latest_forecasts(
|
||||
readings_by_station: Dict[str, List[dict]] = {}
|
||||
if not df_long.empty:
|
||||
for station_code, group in df_long.groupby("station_code"):
|
||||
readings_by_station[station_code] = group[["timestamp", "water_level", "discharge"]].to_dict("records")
|
||||
readings_by_station[station_code] = group[
|
||||
["timestamp", "water_level", "discharge"]
|
||||
].to_dict("records")
|
||||
|
||||
expected_stations = set(features.UPSTREAM_LEADS.keys())
|
||||
for missing_station in expected_stations - set(readings_by_station.keys()):
|
||||
logger.warning(f"No recent data for station {missing_station}; omitting from forecasts")
|
||||
logger.warning(
|
||||
f"No recent data for station {missing_station}; omitting from forecasts"
|
||||
)
|
||||
|
||||
return get_forecasts(readings_by_station, models_dir=models_dir)
|
||||
|
||||
Reference in New Issue
Block a user