feat: hgb-v3 — Open-Meteo rain features clear the 12h warning gate
CI/CD Pipeline - Northern Thailand Ping River Monitor / Deploy to Production (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Performance Test (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Code Quality (push) Successful in 13s
Documentation / Validate Documentation (push) Failing after 7s
Documentation / Generate API Documentation (push) Successful in 8s
Documentation / Build Sphinx Documentation (push) Successful in 17s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Cleanup (push) Successful in 0s
Documentation / Documentation Summary (push) Successful in 2s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Test Suite (3.11) (push) Failing after 24s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Build Docker Image (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Integration Test with Services (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Deploy to Staging (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Deploy to Production (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Performance Test (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Code Quality (push) Successful in 13s
Documentation / Validate Documentation (push) Failing after 7s
Documentation / Generate API Documentation (push) Successful in 8s
Documentation / Build Sphinx Documentation (push) Successful in 17s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Cleanup (push) Successful in 0s
Documentation / Documentation Summary (push) Successful in 2s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Test Suite (3.11) (push) Failing after 24s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Build Docker Image (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Integration Test with Services (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Deploy to Staging (push) Skipped
The rolling-origin harness (models/eval_rain.json) showed catchment rain halving flood-year Brier scores, cutting flood-regime MAE 20-40%, and extending the hard 2024 leads (+6h -> +11h at P.1, +10h -> +19h at P.103). Ported: train_all loads the catchment-mean series (use_rain / --no-rain to opt out; without it bundles train as v2), predict fetches live rain hourly and passes an empty series on failure so rain-trained bundles serve with NaN features instead of tripping the feature guard, and the leader worker persists hourly per-point + catchment-mean rows to a new openmeteo_rain table. Regenerated backtest: the 2024 record flood now gets a 13-HOUR WARNING (alert 04:00 vs 17:00 crossing, river at 2.9m at alert time) — the >=12h acceptance gate PASSES for the first time. Journey on that crossing: v1 -18h, v2 +6h, v3 +13h. The marginal 2025 double-crest trades its artifact +46h latch for a calibrated +2h with zero false alarms. P.1 MAE 4.9/7.2/8.7 cm at 6/12/24h. Docs updated throughout.
This commit is contained in:
+35
-5
@@ -203,9 +203,10 @@ def train_station(
|
||||
split_train_end: str = SPLIT_B_TRAIN_END,
|
||||
split_test_start: str = SPLIT_B_TEST_START,
|
||||
split_test_end: str = SPLIT_B_TEST_END,
|
||||
rain: Optional[pd.Series] = None,
|
||||
) -> Tuple[Optional[dict], dict]:
|
||||
"""Train every head for one station. Returns (bundle_or_None, station_metrics)."""
|
||||
X, Y, meta = features.build_matrix(df_long, station, horizons)
|
||||
X, Y, meta = features.build_matrix(df_long, station, horizons, rain=rain)
|
||||
if meta["n_rows"] < MIN_ROWS_TO_TRAIN:
|
||||
return None, {
|
||||
"status": "failed",
|
||||
@@ -411,10 +412,12 @@ def train_station(
|
||||
] = f"only {n_pos} positives in train span (< {MIN_POSITIVES_FOR_CLASSIFIER})"
|
||||
final_heads[head_key] = None
|
||||
|
||||
# v3 = rise target + Open-Meteo rain features; v2 = rise target only
|
||||
version_prefix = "hgb-v3" if "rain_24h" in feature_names else "hgb-v2"
|
||||
bundle = {
|
||||
"station_code": station,
|
||||
"model_version": f"hgb-v2+{_git_short_sha()}",
|
||||
# v2: regression heads predict the RISE over the current level; the
|
||||
"model_version": f"{version_prefix}+{_git_short_sha()}",
|
||||
# v2+: regression heads predict the RISE over the current level; the
|
||||
# serving side must add the level back. Old v1 bundles lack this key.
|
||||
"regression_target": "rise",
|
||||
"trained_at": datetime.datetime.now().isoformat(),
|
||||
@@ -439,11 +442,28 @@ def train_all(
|
||||
models_dir: Path = Path("models"),
|
||||
skip_eval: bool = False,
|
||||
hgb_overrides: Optional[dict] = None,
|
||||
use_rain: bool = True,
|
||||
) -> dict:
|
||||
"""Train and save every requested station's models. Returns the metrics.json payload."""
|
||||
models_dir = Path(models_dir)
|
||||
models_dir.mkdir(parents=True, exist_ok=True)
|
||||
model_version = f"hgb-v2+{_git_short_sha()}"
|
||||
|
||||
# Catchment rain (Open-Meteo archive, 2021+). Optional: without it the
|
||||
# models train as v2 (no rain columns) and still serve correctly.
|
||||
rain_series = None
|
||||
if use_rain:
|
||||
try:
|
||||
from . import rain as rain_mod
|
||||
|
||||
rain_series = rain_mod.catchment_mean(rain_mod.load_history())
|
||||
except Exception as error:
|
||||
logger.warning(f"rain history unavailable, training without it: {error}")
|
||||
if rain_series is not None:
|
||||
logger.info(
|
||||
f"rain series: {rain_series.index.min()} .. {rain_series.index.max()}"
|
||||
)
|
||||
version_prefix = "hgb-v3" if rain_series is not None else "hgb-v2"
|
||||
model_version = f"{version_prefix}+{_git_short_sha()}"
|
||||
|
||||
station_results: Dict[str, dict] = {}
|
||||
for station in stations:
|
||||
@@ -459,6 +479,7 @@ def train_all(
|
||||
horizons,
|
||||
skip_eval=skip_eval,
|
||||
hgb_overrides=hgb_overrides,
|
||||
rain=rain_series,
|
||||
)
|
||||
if bundle is None:
|
||||
logger.warning(f"{station}: failed ({station_metrics.get('reason')})")
|
||||
@@ -516,6 +537,11 @@ def main(argv: Optional[List[str]] = None) -> None:
|
||||
parser.add_argument(
|
||||
"--end", default=None, help="ISO date; latest measurement to load"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-rain",
|
||||
action="store_true",
|
||||
help="train without the Open-Meteo rain features (v2-style bundles)",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
if args.stations == "all":
|
||||
@@ -539,7 +565,11 @@ def main(argv: Optional[List[str]] = None) -> None:
|
||||
)
|
||||
|
||||
metrics_payload = train_all(
|
||||
df_long, stations, models_dir=Path(args.models_dir), skip_eval=args.skip_eval
|
||||
df_long,
|
||||
stations,
|
||||
models_dir=Path(args.models_dir),
|
||||
skip_eval=args.skip_eval,
|
||||
use_rain=not args.no_rain,
|
||||
)
|
||||
trained = sum(
|
||||
1 for s in metrics_payload["stations"].values() if s["status"] == "trained"
|
||||
|
||||
Reference in New Issue
Block a user