B4L/Northern-Thailand-Ping-River-Monitor
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 15 Wiki Activity

eval: quantile heads and fc48 on top of hgb-v3 (rejected/deferred); HII gauge-rain aggregate
CI/CD Pipeline - Northern Thailand Ping River Monitor / Test Suite (3.11) (push) Failing after 41s
Details
CI/CD Pipeline - Northern Thailand Ping River Monitor / Build Docker Image (push) Skipped
Details
CI/CD Pipeline - Northern Thailand Ping River Monitor / Integration Test with Services (push) Skipped
Details
CI/CD Pipeline - Northern Thailand Ping River Monitor / Deploy to Staging (push) Skipped
Details
CI/CD Pipeline - Northern Thailand Ping River Monitor / Deploy to Production (push) Skipped
Details
CI/CD Pipeline - Northern Thailand Ping River Monitor / Performance Test (push) Skipped
Details
CI/CD Pipeline - Northern Thailand Ping River Monitor / Code Quality (push) Successful in 17s
Details
Documentation / Validate Documentation (push) Failing after 16s
Details
Documentation / Generate API Documentation (push) Successful in 11s
Details
Documentation / Build Sphinx Documentation (push) Successful in 18s
Details
CI/CD Pipeline - Northern Thailand Ping River Monitor / Cleanup (push) Successful in 1s
Details
Documentation / Documentation Summary (push) Successful in 3s
Details

Browse Source 
Rolling-origin harness gains rise_rain_quantile, rise_rain_quantile_uw,
rise_rain_qsigma (L2 point + quantile sigma) and rise_rain_fc48, all
opt-in, plus --from-cache for reproducible offline reruns. Results in
models/eval_2026-09-12*.json, write-up in docs/FLOOD_FORECASTING.md:

- quantile point prediction: better MAE, worse first-alert lead at 5 of
  11 events (P.103 2022-08-14 +6h -> +1h) -> rejected
- quantile sigma only: Brier within noise (0.0031 -> 0.0029) -> not worth 3x heads
- rain_fc48: neutral everywhere except 2024-10-03 P.1 (+21h -> +72h), n=1
  -> deferred to after the 2026 season

src/ml/hii_rain.py: catchment-mean hourly rain from the ~130 HII gauges in
the upper-Ping box and a 24h-sum comparison against Open-Meteo. Not a
training feature (table exists only since 2026-08-11, no archive); exposed
at GET /api/hii/rainfall/catchment so the two sources' agreement is on
record by the time a fold can test it.

data._read_cache now skips non-station files in models/cache/ (the shared
dir also holds rain_openmeteo / dam_* caches, which crashed the reader).
scripts/summarize_eval.py prints per-variant lead/peak-error tables.
This commit is contained in:
grabowski
2026-09-11 21:55:37 +02:00
parent 764764e07e
commit d621aa9ce7
10 changed files with 1607 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
models/eval_2026-09-12_qsigma.json
+439
View File
@@ -0,0 +1,439 @@
[
{
"station": "P.1",
"warn_thr": 3.7,
"folds": [
{
"year": 2021,
"n_train": 20024,
"n_test": 4392,
"events": [],
"variants": {
"rise_rain": {
"mae": 0.07375926701460789,
"mae_above_2p5": null,
"brier_warn": 0.0,
"events": [],
"false_alarm_episodes": 0
},
"rise_rain_qsigma": {
"mae": 0.07375926701460789,
"mae_above_2p5": null,
"brier_warn": 7.852802408474157e-14,
"events": [],
"false_alarm_episodes": 0
}
}
},
{
"year": 2022,
"n_train": 28784,
"n_test": 4392,
"events": [
{
"crossing": "2022-10-02T19:00:00",
"peak_ts": "2022-10-03T15:00:00",
"peak_level": 4.65
}
],
"variants": {
"rise_rain": {
"mae": 0.08452847354970178,
"mae_above_2p5": 0.25082252888260664,
"brier_warn": 0.004300908725927739,
"events": [
{
"crossing": "2022-10-02T19:00:00",
"lead_h": 5.0,
"peak_level": 4.65,
"peak_pred_24h_before": 3.8173954245046406
}
],
"false_alarm_episodes": 0
},
"rise_rain_qsigma": {
"mae": 0.08452847354970178,
"mae_above_2p5": 0.25082252888260664,
"brier_warn": 0.0039815091186836665,
"events": [
{
"crossing": "2022-10-02T19:00:00",
"lead_h": 5.0,
"peak_level": 4.65,
"peak_pred_24h_before": 3.8173954245046406
}
],
"false_alarm_episodes": 0
}
}
},
{
"year": 2023,
"n_train": 37539,
"n_test": 4392,
"events": [],
"variants": {
"rise_rain": {
"mae": 0.07696637416933236,
"mae_above_2p5": 0.10643655855133666,
"brier_warn": 1.919860722404625e-15,
"events": [],
"false_alarm_episodes": 0
},
"rise_rain_qsigma": {
"mae": 0.07696637416933236,
"mae_above_2p5": 0.10643655855133666,
"brier_warn": 4.0844243581898366e-08,
"events": [],
"false_alarm_episodes": 0
}
}
},
{
"year": 2024,
"n_train": 46323,
"n_test": 4392,
"events": [
{
"crossing": "2024-09-24T17:00:00",
"peak_ts": "2024-09-26T02:00:00",
"peak_level": 4.93
},
{
"crossing": "2024-10-03T09:00:00",
"peak_ts": "2024-10-05T12:00:00",
"peak_level": 5.3
}
],
"variants": {
"rise_rain": {
"mae": 0.0890019157543212,
"mae_above_2p5": 0.2093245927883516,
"brier_warn": 0.005826169840715695,
"events": [
{
"crossing": "2024-09-24T17:00:00",
"lead_h": 10.0,
"peak_level": 4.93,
"peak_pred_24h_before": 4.817519939833057
},
{
"crossing": "2024-10-03T09:00:00",
"lead_h": 21.0,
"peak_level": 5.3,
"peak_pred_24h_before": 5.546588884631041
}
],
"false_alarm_episodes": 0
},
"rise_rain_qsigma": {
"mae": 0.0890019157543212,
"mae_above_2p5": 0.2093245927883516,
"brier_warn": 0.005475787026695418,
"events": [
{
"crossing": "2024-09-24T17:00:00",
"lead_h": 10.0,
"peak_level": 4.93,
"peak_pred_24h_before": 4.817519939833057
},
{
"crossing": "2024-10-03T09:00:00",
"lead_h": 21.0,
"peak_level": 5.3,
"peak_pred_24h_before": 5.546588884631041
}
],
"false_alarm_episodes": 0
}
}
},
{
"year": 2025,
"n_train": 55083,
"n_test": 4392,
"events": [
{
"crossing": "2025-09-27T18:00:00",
"peak_ts": "2025-09-27T22:00:00",
"peak_level": 3.93
}
],
"variants": {
"rise_rain": {
"mae": 0.11202835461695825,
"mae_above_2p5": 0.2456782496767171,
"brier_warn": 0.005178356039745929,
"events": [
{
"crossing": "2025-09-27T18:00:00",
"lead_h": 2.0,
"peak_level": 3.93,
"peak_pred_24h_before": 3.23
}
],
"false_alarm_episodes": 0
},
"rise_rain_qsigma": {
"mae": 0.11202835461695825,
"mae_above_2p5": 0.2456782496767171,
"brier_warn": 0.00491346243876589,
"events": [
{
"crossing": "2025-09-27T18:00:00",
"lead_h": 2.0,
"peak_level": 3.93,
"peak_pred_24h_before": 3.23
}
],
"false_alarm_episodes": 0
}
}
}
]
},
{
"station": "P.103",
"warn_thr": 5.95,
"folds": [
{
"year": 2021,
"n_train": 20009,
"n_test": 4392,
"events": [],
"variants": {
"rise_rain": {
"mae": 0.14438683486790602,
"mae_above_2p5": null,
"brier_warn": 0.0,
"events": [],
"false_alarm_episodes": 0
},
"rise_rain_qsigma": {
"mae": 0.14438683486790602,
"mae_above_2p5": null,
"brier_warn": 3.5349670949872053e-13,
"events": [],
"false_alarm_episodes": 0
}
}
},
{
"year": 2022,
"n_train": 28769,
"n_test": 4392,
"events": [
{
"crossing": "2022-08-14T04:00:00",
"peak_ts": "2022-08-14T08:00:00",
"peak_level": 6.09
},
{
"crossing": "2022-10-02T16:00:00",
"peak_ts": "2022-10-03T16:00:00",
"peak_level": 7.54
}
],
"variants": {
"rise_rain": {
"mae": 0.14392334606606189,
"mae_above_2p5": 0.41075382386412596,
"brier_warn": 0.00764679988213082,
"events": [
{
"crossing": "2022-08-14T04:00:00",
"lead_h": 6.0,
"peak_level": 6.09,
"peak_pred_24h_before": 4.825270553204425
},
{
"crossing": "2022-10-02T16:00:00",
"lead_h": 9.0,
"peak_level": 7.54,
"peak_pred_24h_before": 7.194064117976157
}
],
"false_alarm_episodes": 0
},
"rise_rain_qsigma": {
"mae": 0.14392334606606189,
"mae_above_2p5": 0.41075382386412596,
"brier_warn": 0.006975493312169236,
"events": [
{
"crossing": "2022-08-14T04:00:00",
"lead_h": 6.0,
"peak_level": 6.09,
"peak_pred_24h_before": 4.825270553204425
},
{
"crossing": "2022-10-02T16:00:00",
"lead_h": 9.0,
"peak_level": 7.54,
"peak_pred_24h_before": 7.194064117976157
}
],
"false_alarm_episodes": 0
}
}
},
{
"year": 2023,
"n_train": 37524,
"n_test": 4392,
"events": [],
"variants": {
"rise_rain": {
"mae": 0.16195301512514512,
"mae_above_2p5": null,
"brier_warn": 3.0208441147560167e-07,
"events": [],
"false_alarm_episodes": 0
},
"rise_rain_qsigma": {
"mae": 0.16195301512514512,
"mae_above_2p5": null,
"brier_warn": 3.2176039819636275e-05,
"events": [],
"false_alarm_episodes": 0
}
}
},
{
"year": 2024,
"n_train": 46308,
"n_test": 4058,
"events": [
{
"crossing": "2024-09-24T10:00:00",
"peak_ts": "2024-09-26T00:00:00",
"peak_level": 8.27
},
{
"crossing": "2024-09-30T03:00:00",
"peak_ts": "2024-09-30T06:00:00",
"peak_level": 5.99
},
{
"crossing": "2024-10-03T06:00:00",
"peak_ts": "2024-10-05T07:00:00",
"peak_level": 9.93
}
],
"variants": {
"rise_rain": {
"mae": 0.13901842325128816,
"mae_above_2p5": 0.2887416310966684,
"brier_warn": 0.0072040857753137046,
"events": [
{
"crossing": "2024-09-24T10:00:00",
"lead_h": 19.0,
"peak_level": 8.27,
"peak_pred_24h_before": 8.212734363860193
},
{
"crossing": "2024-09-30T03:00:00",
"lead_h": 9.0,
"peak_level": 5.99,
"peak_pred_24h_before": 5.47565489914091
},
{
"crossing": "2024-10-03T06:00:00",
"lead_h": 55.0,
"peak_level": 9.93,
"peak_pred_24h_before": 8.780278464915938
}
],
"false_alarm_episodes": 0
},
"rise_rain_qsigma": {
"mae": 0.13901842325128816,
"mae_above_2p5": 0.2887416310966684,
"brier_warn": 0.006910847607098417,
"events": [
{
"crossing": "2024-09-24T10:00:00",
"lead_h": 19.0,
"peak_level": 8.27,
"peak_pred_24h_before": 8.212734363860193
},
{
"crossing": "2024-09-30T03:00:00",
"lead_h": 9.0,
"peak_level": 5.99,
"peak_pred_24h_before": 5.47565489914091
},
{
"crossing": "2024-10-03T06:00:00",
"lead_h": 55.0,
"peak_level": 9.93,
"peak_pred_24h_before": 8.780278464915938
}
],
"false_alarm_episodes": 0
}
}
},
{
"year": 2025,
"n_train": 54577,
"n_test": 4392,
"events": [
{
"crossing": "2025-09-26T06:00:00",
"peak_ts": "2025-09-27T21:00:00",
"peak_level": 6.64
},
{
"crossing": "2025-10-03T06:00:00",
"peak_ts": "2025-10-03T12:00:00",
"peak_level": 6.14
}
],
"variants": {
"rise_rain": {
"mae": 0.17206685418009837,
"mae_above_2p5": 0.3840520085986099,
"brier_warn": 0.015824852891785323,
"events": [
{
"crossing": "2025-09-26T06:00:00",
"lead_h": 6.0,
"peak_level": 6.64,
"peak_pred_24h_before": 5.73043665401637
},
{
"crossing": "2025-10-03T06:00:00",
"lead_h": 8.0,
"peak_level": 6.14,
"peak_pred_24h_before": 5.609919760117381
}
],
"false_alarm_episodes": 2
},
"rise_rain_qsigma": {
"mae": 0.17206685418009837,
"mae_above_2p5": 0.3840520085986099,
"brier_warn": 0.015889933586185904,
"events": [
{
"crossing": "2025-09-26T06:00:00",
"lead_h": 6.0,
"peak_level": 6.64,
"peak_pred_24h_before": 5.73043665401637
},
{
"crossing": "2025-10-03T06:00:00",
"lead_h": 8.0,
"peak_level": 6.14,
"peak_pred_24h_before": 5.609919760117381
}
],
"false_alarm_episodes": 2
}
}
}
]
}
]