feat: "Is the model getting better?" - live verification per model version
CI / Test suite (push) Successful in 22s
CI / Format & lint (push) Successful in 16s
Docs / Validate documentation (push) Successful in 10s
Security / Dependency vulnerabilities (push) Successful in 1m34s
Security / Static analysis (push) Successful in 10s
Security / License report (push) Successful in 13s

src/ml/skill.py joins forecast_history (what each deployed version
predicted for the 24 h peak, hourly) to water_measurements (what the
river did) and reports per version: verified hours, peak MAE, bias, the
persistence baseline (peak = current level), skill = 1 - MAE/persistence,
and the same MAE restricted to observed peaks >= 2 m. Only forecasts
whose window has elapsed with >= 75 % of hours observed count; a
version needs 24 verified hours before it is compared.

GET /api/forecast/skill?station_code=P.1&horizon=24 returns it (SWR
cached, 15 min). The dashboard's forecast card gains a panel with a
one-line verdict (current vs previous version), the per-version table,
and a caveat that quiet weeks measure quiet-river accuracy only: the
model is judged on flood-onset lead, which the backtests cover. EN + TH.

On today's production data: hgb-v3+28b62e5 (369 h, Aug 13 - Sep 1)
MAE 15.2 cm, skill -0.05; hgb-v2+f6570ac (224 h, Sep 1 - 11) MAE
12.3 cm, skill 0.36 - the "worse" v2 model scores better on a quieter
fortnight, which is exactly why the panel shows the >= 2 m column and
the caveat. Tests: 3, sqlite, synthetic.

scripts/dev_proxy.py: DEV_PROXY_LOCAL lets a not-yet-deployed endpoint be
answered from a local JSON file while everything else goes to prod.
This commit is contained in:
2026-09-11 23:44:44 +02:00
parent 2e19974fad
commit 7b31d4d0dd
5 changed files with 433 additions and 7 deletions
+170
View File
@@ -0,0 +1,170 @@
"""Live forecast skill: what the deployed model said versus what the river did.
Every hour the precompute stores the issued 24 h forecast (forecast_history);
water_measurements holds what actually happened. Joining the two gives a
verification that needs no retraining and answers the question the dashboard
is asked most: "is the model getting better?" — per model version, on the
hours that version was actually serving.
Metrics per version and horizon:
n verified forecasts (issued, and the horizon has since elapsed)
mae |predicted_max - observed_max| over the horizon window, metres
bias mean(predicted - observed): >0 over-predicts the peak
persistence MAE of the trivial "peak = current level" forecast on the
same rows; a model is only useful if it beats this
skill 1 - mae/persistence (0 = no better than persistence, 1 = perfect)
above_2m same MAE restricted to rows where the observed peak >= 2 m,
i.e. the flood-relevant regime
Only the P.1 gauge is verified by default: it is the one the city threshold
is keyed to, and one station keeps the query cheap enough to run on request.
"""
import datetime
import logging
from typing import Dict, List, Optional
logger = logging.getLogger(__name__)
DEFAULT_STATION = "P.1"
DEFAULT_HORIZON = 24
MIN_VERIFIED = 24 # fewer than a day of verified hours is not a number
def _sql_for(db_type: str) -> str:
"""Join each issued forecast to the observed max over (as_of, as_of + h]."""
if db_type == "postgresql":
window_end = "f.as_of + (f.horizon_hours || ' hours')::interval"
elif db_type == "mysql":
window_end = "DATE_ADD(f.as_of, INTERVAL f.horizon_hours HOUR)"
else: # sqlite
window_end = "datetime(f.as_of, '+' || f.horizon_hours || ' hours')"
return f"""
SELECT f.as_of, f.model_version, f.predicted_max_level, f.current_level,
(SELECT MAX(m.water_level) FROM water_measurements m
JOIN stations s ON s.id = m.station_id
WHERE s.station_code = f.station_code
AND m.timestamp > f.as_of AND m.timestamp <= {window_end}) AS observed_max,
(SELECT COUNT(m.water_level) FROM water_measurements m
JOIN stations s ON s.id = m.station_id
WHERE s.station_code = f.station_code
AND m.timestamp > f.as_of AND m.timestamp <= {window_end}) AS observed_n
FROM forecast_history f
WHERE f.station_code = :code AND f.horizon_hours = :horizon
AND f.source = 'model' AND f.predicted_max_level IS NOT NULL
AND f.as_of <= :verifiable_before
ORDER BY f.as_of
"""
def compute_skill(
engine,
db_type: str,
station_code: str = DEFAULT_STATION,
horizon_hours: int = DEFAULT_HORIZON,
now: Optional[datetime.datetime] = None,
) -> Dict:
"""Per-model-version verification of issued forecasts against observations."""
from sqlalchemy import text
now = now or datetime.datetime.now()
verifiable_before = now - datetime.timedelta(hours=horizon_hours)
with engine.connect() as conn:
rows = [
dict(r._mapping)
for r in conn.execute(
text(_sql_for(db_type)),
{
"code": station_code,
"horizon": horizon_hours,
"verifiable_before": verifiable_before,
},
)
]
def _ts(value):
# sqlite hands back strings; postgres/mysql give datetimes
if isinstance(value, datetime.datetime):
return value
return datetime.datetime.fromisoformat(str(value).replace(" ", "T"))
by_version: Dict[str, List[dict]] = {}
for r in rows:
r["as_of"] = _ts(r["as_of"])
# need most of the window observed, or the "max" is not the peak
if r["observed_max"] is None or (r["observed_n"] or 0) < horizon_hours * 0.75:
continue
by_version.setdefault(r["model_version"] or "unknown", []).append(r)
versions = []
for version, vrows in by_version.items():
pred = [float(r["predicted_max_level"]) for r in vrows]
obs = [float(r["observed_max"]) for r in vrows]
cur = [
float(r["current_level"]) if r["current_level"] is not None else None
for r in vrows
]
err = [p - o for p, o in zip(pred, obs)]
mae = sum(abs(e) for e in err) / len(err)
bias = sum(err) / len(err)
pers_rows = [(c, o) for c, o in zip(cur, obs) if c is not None]
persistence = (
sum(abs(c - o) for c, o in pers_rows) / len(pers_rows)
if pers_rows
else None
)
high = [(p, o) for p, o in zip(pred, obs) if o >= 2.0]
versions.append(
{
"model_version": version,
"first_issued": min(r["as_of"] for r in vrows).isoformat(),
"last_issued": max(r["as_of"] for r in vrows).isoformat(),
"n": len(vrows),
"mae_m": round(mae, 3),
"bias_m": round(bias, 3),
"persistence_mae_m": (
None if persistence is None else round(persistence, 3)
),
"skill": (
None if not persistence else round(1.0 - mae / persistence, 3)
),
"above_2m_n": len(high),
"above_2m_mae_m": (
round(sum(abs(p - o) for p, o in high) / len(high), 3)
if high
else None
),
"enough_data": len(vrows) >= MIN_VERIFIED,
}
)
versions.sort(key=lambda v: v["first_issued"])
# Headline: current version vs the previous one that had enough data
current = versions[-1] if versions else None
previous = (
next((v for v in reversed(versions[:-1]) if v["enough_data"]), None)
if versions
else None
)
trend = None
if current and previous and current["enough_data"]:
trend = {
"previous_version": previous["model_version"],
"mae_delta_m": round(current["mae_m"] - previous["mae_m"], 3),
"skill_delta": (
None
if current["skill"] is None or previous["skill"] is None
else round(current["skill"] - previous["skill"], 3)
),
"better": current["mae_m"] < previous["mae_m"],
}
return {
"station_code": station_code,
"horizon_hours": horizon_hours,
"verified_until": verifiable_before.isoformat(),
"min_verified": MIN_VERIFIED,
"versions": versions,
"current": current,
"trend": trend,
}
+110
View File
@@ -155,6 +155,18 @@
.stat-value { margin-top: 9px; font-size: 1.65rem; font-weight: 800; letter-spacing: -.04em; white-space: nowrap; }
.stat-note { color: var(--muted); margin-top: 3px; font-size: .77rem; }
.stat.stale { border-color: var(--red); background: rgba(204,75,55,.08); }
.skill-panel { border: 1px solid var(--border); border-radius: 12px; padding: 12px 14px; margin-top: 14px; background: var(--surface-3); }
.skill-head { display: flex; justify-content: space-between; align-items: baseline; gap: 12px; flex-wrap: wrap; }
.skill-headline { margin: 8px 0 10px; font-weight: 700; font-size: .9rem; }
.skill-headline.better { color: var(--green); }
.skill-headline.worse { color: var(--amber); }
.skill-table-wrap { overflow-x: auto; }
.skill-table { border-collapse: collapse; font-size: .76rem; width: 100%; min-width: 560px; }
.skill-table th { text-align: left; color: var(--muted); font-weight: 700; font-size: .66rem; text-transform: uppercase; letter-spacing: .06em; padding: 4px 8px; border-bottom: 1px solid var(--border); }
.skill-table td { padding: 5px 8px; border-bottom: 1px solid var(--border); white-space: nowrap; }
.skill-table tr.current td { font-weight: 700; }
.skill-table td.num { text-align: right; font-variant-numeric: tabular-nums; }
.skill-table td.dim { color: var(--muted); }
.stat.stale .stat-value, .stat.stale .stat-note { color: var(--red); }
.workspace { display: grid; grid-template-columns: minmax(0, 1fr) 330px; gap: 14px; min-height: 640px; }
.map-card, .side-card { background: var(--card); border: 1px solid var(--border); border-radius: 19px; box-shadow: var(--shadow); overflow: hidden; }
@@ -417,6 +429,16 @@
<div class="p1-peak" style="margin-top:7px" data-i18n="outlook.explainer">Chance the river reaches each official inundation stage within 24 h — city flooding begins at stage 1 (3.70 m); each stage floods additional districts.</div>
</div>
<button type="button" class="zones-button" id="forecast-expand" style="display:none;margin-top:12px">Show all station forecasts ▾</button>
<div class="skill-panel" id="skill-panel" style="display:none">
<div class="skill-head">
<strong data-i18n="skill.title">Is the model getting better?</strong>
<span class="subtitle" id="skill-sub"></span>
</div>
<div class="skill-headline" id="skill-headline"></div>
<p class="subtitle skill-caveat" id="skill-caveat" style="margin:-4px 0 10px"></p>
<div class="skill-table-wrap"><table class="skill-table" id="skill-table"></table></div>
<p class="subtitle" style="margin:8px 0 0" data-i18n="skill.explain">Every hour the deployed model's 24 h peak forecast for P.1 is stored; once those 24 hours have passed it is compared with what the river actually did. "Skill" is how much better the model was than assuming the level stays where it is (0 = no better, 1 = perfect). Versions retrained on more data appear as new rows, so improvement, or its absence, is visible here rather than claimed.</p>
</div>
<div class="forecast-grid" id="forecast-grid" style="display:none"></div>
</section>
@@ -556,6 +578,26 @@
'forecast.chip.peak': (lvl) => ` · peak ~${lvl} m`,
'forecast.chip.heuristic': ' · heuristic fallback',
'forecast.expand': (n) => `Show all ${n} station forecasts ▾`,
'skill.title': 'Is the model getting better?',
'skill.sub': (n, since) => `${n} verified 24 h forecasts for P.1 since ${since}`,
'skill.explain': 'Every hour the deployed model\'s 24 h peak forecast for P.1 is stored; once those 24 hours have passed it is compared with what the river actually did. "Skill" is how much better the model was than assuming the level stays where it is (0 = no better, 1 = perfect). Versions retrained on more data appear as new rows, so improvement, or its absence, is visible here rather than claimed.',
'skill.better': (v, prev, d) => `Current model ${v} is more accurate than ${prev}: peak error ${d} cm lower on the hours it has served.`,
'skill.worse': (v, prev, d) => `Current model ${v} has a higher peak error than ${prev} so far (+${d} cm).`,
'skill.caveat.quiet': 'All verified hours so far were below 2 m: this measures quiet-river accuracy only. The model is built and judged for flood onset (lead time before 3.70 m), which no quiet week can test — see the backtests in the documentation.',
'skill.caveat.regime': 'Versions served different weeks; the ≥ 2 m column compares them on the hours that matter.',
'skill.single': (v) => `Only ${v} has enough verified hours yet; the next retrain adds a row to compare.`,
'skill.young': (v, n, min) => `${v} has ${n} verified hours; a comparison needs ${min}.`,
'skill.none': 'No verified forecasts yet — the first appear 24 h after a model starts serving.',
'skill.col.version': 'Model',
'skill.col.period': 'Served',
'skill.col.n': 'Hours',
'skill.col.mae': 'Peak error',
'skill.col.bias': 'Bias',
'skill.col.pers': 'Persistence',
'skill.col.skill': 'Skill',
'skill.col.high': '≥ 2 m error',
'skill.cm': (v) => `${v} cm`,
'skill.na': '—',
'forecast.collapse': 'Hide station forecasts ▴',
'outlook.title': 'Chiang Mai city flood outlook · P.1 Nawarat Bridge',
'outlook.explainer': 'Chance the river reaches each official inundation stage within 24 h — city flooding begins at stage 1 (3.70 m); each stage floods additional districts.',
@@ -732,6 +774,26 @@
'forecast.chip.peak': (lvl) => ` · ระดับสูงสุดประมาณ ${lvl} ม.`,
'forecast.chip.heuristic': ' · ใช้การประมาณอย่างง่าย',
'forecast.expand': (n) => `แสดงพยากรณ์ทั้ง ${n} สถานี ▾`,
'skill.title': 'โมเดลแม่นยำขึ้นหรือไม่?',
'skill.sub': (n, since) => `พยากรณ์ 24 ชม. ของ P.1 ที่ตรวจสอบแล้ว ${n} ครั้ง ตั้งแต่ ${since}`,
'skill.explain': 'ทุกชั่วโมงระบบบันทึกค่าพยากรณ์ระดับน้ำสูงสุดใน 24 ชม. ของ P.1 ไว้ เมื่อครบ 24 ชม. จึงนำมาเทียบกับระดับน้ำจริง "ทักษะ" คือโมเดลดีกว่าการสมมติว่าระดับน้ำคงที่มากเพียงใด (0 = ไม่ดีกว่า, 1 = สมบูรณ์แบบ) โมเดลที่ฝึกใหม่ด้วยข้อมูลมากขึ้นจะปรากฏเป็นแถวใหม่ จึงเห็นได้ว่าดีขึ้นจริงหรือไม่',
'skill.better': (v, prev, d) => `โมเดลปัจจุบัน ${v} แม่นยำกว่า ${prev}: ค่าคลาดเคลื่อนต่ำกว่า ${d} ซม. ในช่วงที่ให้บริการ`,
'skill.worse': (v, prev, d) => `โมเดลปัจจุบัน ${v} มีค่าคลาดเคลื่อนสูงกว่า ${prev} (+${d} ซม.)`,
'skill.caveat.quiet': 'ชั่วโมงที่ตรวจสอบทั้งหมดอยู่ต่ำกว่า 2 ม.: วัดได้เพียงความแม่นยำช่วงน้ำปกติ โมเดลถูกสร้างและประเมินสำหรับช่วงน้ำเริ่มท่วม (เวลาเตือนล่วงหน้าก่อน 3.70 ม.) ซึ่งสัปดาห์ปกติทดสอบไม่ได้ — ดูผลทดสอบย้อนหลังในเอกสาร',
'skill.caveat.regime': 'แต่ละเวอร์ชันให้บริการคนละช่วงเวลา คอลัมน์ ≥ 2 ม. เปรียบเทียบเฉพาะชั่วโมงที่สำคัญ',
'skill.single': (v) => `มีเพียง ${v} ที่มีข้อมูลตรวจสอบเพียงพอ การฝึกครั้งถัดไปจะเพิ่มแถวให้เปรียบเทียบ`,
'skill.young': (v, n, min) => `${v} มีข้อมูลตรวจสอบ ${n} ชั่วโมง ต้องการอย่างน้อย ${min} เพื่อเปรียบเทียบ`,
'skill.none': 'ยังไม่มีพยากรณ์ที่ตรวจสอบได้ — จะเริ่มมี 24 ชม. หลังโมเดลเริ่มทำงาน',
'skill.col.version': 'โมเดล',
'skill.col.period': 'ช่วงเวลา',
'skill.col.n': 'ชั่วโมง',
'skill.col.mae': 'คลาดเคลื่อน',
'skill.col.bias': 'อคติ',
'skill.col.pers': 'ระดับคงที่',
'skill.col.skill': 'ทักษะ',
'skill.col.high': 'คลาดเคลื่อน ≥ 2 ม.',
'skill.cm': (v) => `${v} ซม.`,
'skill.na': '—',
'forecast.collapse': 'ซ่อนพยากรณ์รายสถานี ▴',
'outlook.title': 'แนวโน้มน้ำท่วมเมืองเชียงใหม่ · P.1 สะพานนวรัฐ',
'outlook.explainer': 'โอกาสที่ระดับน้ำจะถึงแต่ละระดับการท่วมตามประกาศทางการภายใน 24 ชม. — น้ำเริ่มท่วมเมืองที่ระดับ 1 (3.70 ม.) และแต่ละระดับจะท่วมพื้นที่เพิ่มขึ้น',
@@ -1989,11 +2051,59 @@
? t('forecast.collapse')
: t('forecast.expand', stations.length);
card.style.display = 'block';
loadSkill(); // non-blocking; panel stays hidden until there is verified data
} catch (error) {
card.style.display = 'none';
}
}
async function loadSkill() {
const panel = $('skill-panel');
try {
const response = await fetch('/api/forecast/skill?station_code=P.1&horizon=24');
if (!response.ok) throw new Error(`HTTP ${response.status}`);
const data = await response.json();
const versions = (data.versions || []).filter((v) => v.n > 0);
if (!versions.length) { panel.style.display = 'none'; return; }
const cm = (m) => m == null ? t('skill.na') : t('skill.cm', (m * 100).toFixed(1));
const fmtDay = (v) => parseTs(v).toLocaleDateString(loc(), { timeZone: TZ, day: 'numeric', month: 'short' });
const total = versions.reduce((a, v) => a + v.n, 0);
$('skill-sub').textContent = t('skill.sub', total.toLocaleString(loc()), fmtDay(versions[0].first_issued));
const head = $('skill-headline');
head.className = 'skill-headline';
const cur = data.current;
if (data.trend && cur) {
const delta = Math.abs(data.trend.mae_delta_m * 100).toFixed(1);
head.textContent = data.trend.better
? t('skill.better', cur.model_version, data.trend.previous_version, delta)
: t('skill.worse', cur.model_version, data.trend.previous_version, delta);
head.classList.add(data.trend.better ? 'better' : 'worse');
} else if (cur && cur.enough_data) {
head.textContent = t('skill.single', cur.model_version);
} else if (cur) {
head.textContent = t('skill.young', cur.model_version, cur.n, data.min_verified);
} else head.textContent = t('skill.none');
const anyHigh = versions.some((v) => v.above_2m_n > 0);
const compared = versions.filter((v) => v.enough_data).length > 1;
$('skill-caveat').textContent = !anyHigh ? t('skill.caveat.quiet') : compared ? t('skill.caveat.regime') : '';
const cols = ['version', 'period', 'n', 'mae', 'bias', 'pers', 'skill', 'high'];
const rows = versions.map((v) => `<tr class="${v === cur ? 'current' : ''}${v.enough_data ? '' : ' young'}">`
+ `<td>${escapeHtml(v.model_version)}</td>`
+ `<td class="dim">${fmtDay(v.first_issued)} ${fmtDay(v.last_issued)}</td>`
+ `<td class="num">${v.n.toLocaleString(loc())}</td>`
+ `<td class="num">${cm(v.mae_m)}</td>`
+ `<td class="num">${v.bias_m == null ? t('skill.na') : (v.bias_m >= 0 ? '+' : '') + (v.bias_m * 100).toFixed(1)}</td>`
+ `<td class="num dim">${cm(v.persistence_mae_m)}</td>`
+ `<td class="num">${v.skill == null ? t('skill.na') : v.skill.toFixed(2)}</td>`
+ `<td class="num">${v.above_2m_n ? `${cm(v.above_2m_mae_m)} <span class="dim">(${v.above_2m_n})</span>` : t('skill.na')}</td>`
+ '</tr>').join('');
$('skill-table').innerHTML = `<thead><tr>${cols.map((c) => `<th>${escapeHtml(t('skill.col.' + c))}</th>`).join('')}</tr></thead><tbody>${rows}</tbody>`;
panel.style.display = 'block';
} catch (error) {
panel.style.display = 'none';
}
}
async function loadDbStats() {
const strip = $('db-stats');
try {
+53 -7
View File
@@ -864,7 +864,7 @@ def _hii_rows(sql: str, params: Dict[str, Any]) -> List[Dict[str, Any]]:
# flood, slightly stale readings with a visible timestamp beat an error page.
HII_CACHE: Dict[str, Any] = {}
HII_CACHE_LOCK = Lock()
_HII_COMPUTE_LOCKS = {"rain": Lock(), "waterlevel": Lock()}
_HII_COMPUTE_LOCKS = {"rain": Lock(), "waterlevel": Lock(), "skill": Lock()}
LATEST_CACHE: Dict[str, Any] = {}
LATEST_CACHE_LOCK = Lock()
_LATEST_COMPUTE_LOCK = Lock()
@@ -1230,6 +1230,56 @@ async def get_forecast_history(
return await asyncio.to_thread(store.fetch, station_code, start_dt, end_dt, horizon)
@app.get("/api/forecast/skill")
async def get_forecast_skill(
response: Response,
station_code: str = Query("P.1"),
horizon: int = Query(24, ge=1, le=48),
):
"""Is the model getting better? Issued forecasts verified against what the
river then did, per model version, with a persistence baseline.
Read from forecast_history (what each deployed version predicted, hourly)
joined to water_measurements; no retraining involved. Cached like the HII
feeds because the join is a few hundred correlated subqueries.
"""
increment_counter("api_requests", labels={"endpoint": "forecast_skill"})
store = app_state.get("forecast_store")
if not store:
return {
"station_code": station_code,
"horizon_hours": horizon,
"versions": [],
"current": None,
"trend": None,
}
def compute():
from .ml import skill
if not store.engine and not store.connect():
return {
"station_code": station_code,
"horizon_hours": horizon,
"versions": [],
"current": None,
"trend": None,
}
return skill.compute_skill(store.engine, store.db_type, station_code, horizon)
payload, stale = await _cached_swr(
HII_CACHE,
HII_CACHE_LOCK,
_HII_COMPUTE_LOCKS["skill"],
f"skill:{station_code}:{horizon}",
max(Config.HII_CACHE_TTL_SECONDS, 900),
compute,
)
if stale:
response.headers["X-Data-Stale"] = "true"
return payload
@app.get("/measurements/latest", response_model=List[MeasurementResponse])
async def get_latest_measurements(response: Response, limit: int = 100):
"""Get latest measurements from all stations"""
@@ -1313,9 +1363,7 @@ async def get_database_stats():
from sqlalchemy import text
with engine.connect() as conn:
return conn.execute(
text(
"""
return conn.execute(text("""
SELECT (SELECT COUNT(*) FROM hii_rainfall) AS rain_n,
(SELECT COUNT(*) FROM hii_waterlevel) AS wl_n,
(SELECT COUNT(*) FROM hii_rain_stations) AS rain_s,
@@ -1324,9 +1372,7 @@ async def get_database_stats():
(SELECT MAX(timestamp) FROM hii_rainfall) AS rain_hi,
(SELECT MIN(timestamp) FROM hii_waterlevel) AS wl_lo,
(SELECT MAX(timestamp) FROM hii_waterlevel) AS wl_hi
"""
)
).one()
""")).one()
def compute():
# Heavy: full-table counts and coverage over ~1.7M rows. Runs at most