diff --git a/scripts/dev_proxy.py b/scripts/dev_proxy.py
index 6407816..fb6f18f 100644
--- a/scripts/dev_proxy.py
+++ b/scripts/dev_proxy.py
@@ -3,6 +3,7 @@ live server, so browser-side changes can be checked against real data
before deploy. Usage: python scripts/dev_proxy.py [port]"""
import http.server
+import os
import sys
import urllib.request
from pathlib import Path
@@ -17,6 +18,12 @@ class Handler(http.server.BaseHTTPRequestHandler):
body = (STATIC / "dashboard.html").read_bytes()
self._send(200, "text/html; charset=utf-8", body)
return
+ # Local overrides for endpoints not yet deployed: DEV_PROXY_LOCAL=/api/x=file.json,...
+ for pair in filter(None, os.environ.get("DEV_PROXY_LOCAL", "").split(",")):
+ prefix, file = pair.split("=", 1)
+ if self.path.split("?")[0] == prefix:
+ self._send(200, "application/json", Path(file).read_bytes())
+ return
if self.path.startswith("/static/"):
f = STATIC / self.path[len("/static/"):].split("?")[0]
if f.is_file():
diff --git a/src/ml/skill.py b/src/ml/skill.py
new file mode 100644
index 0000000..e692001
--- /dev/null
+++ b/src/ml/skill.py
@@ -0,0 +1,170 @@
+"""Live forecast skill: what the deployed model said versus what the river did.
+
+Every hour the precompute stores the issued 24 h forecast (forecast_history);
+water_measurements holds what actually happened. Joining the two gives a
+verification that needs no retraining and answers the question the dashboard
+is asked most: "is the model getting better?" — per model version, on the
+hours that version was actually serving.
+
+Metrics per version and horizon:
+ n verified forecasts (issued, and the horizon has since elapsed)
+ mae |predicted_max - observed_max| over the horizon window, metres
+ bias mean(predicted - observed): >0 over-predicts the peak
+ persistence MAE of the trivial "peak = current level" forecast on the
+ same rows; a model is only useful if it beats this
+ skill 1 - mae/persistence (0 = no better than persistence, 1 = perfect)
+ above_2m same MAE restricted to rows where the observed peak >= 2 m,
+ i.e. the flood-relevant regime
+
+Only the P.1 gauge is verified by default: it is the one the city threshold
+is keyed to, and one station keeps the query cheap enough to run on request.
+"""
+
+import datetime
+import logging
+from typing import Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_STATION = "P.1"
+DEFAULT_HORIZON = 24
+MIN_VERIFIED = 24 # fewer than a day of verified hours is not a number
+
+
+def _sql_for(db_type: str) -> str:
+ """Join each issued forecast to the observed max over (as_of, as_of + h]."""
+ if db_type == "postgresql":
+ window_end = "f.as_of + (f.horizon_hours || ' hours')::interval"
+ elif db_type == "mysql":
+ window_end = "DATE_ADD(f.as_of, INTERVAL f.horizon_hours HOUR)"
+ else: # sqlite
+ window_end = "datetime(f.as_of, '+' || f.horizon_hours || ' hours')"
+ return f"""
+ SELECT f.as_of, f.model_version, f.predicted_max_level, f.current_level,
+ (SELECT MAX(m.water_level) FROM water_measurements m
+ JOIN stations s ON s.id = m.station_id
+ WHERE s.station_code = f.station_code
+ AND m.timestamp > f.as_of AND m.timestamp <= {window_end}) AS observed_max,
+ (SELECT COUNT(m.water_level) FROM water_measurements m
+ JOIN stations s ON s.id = m.station_id
+ WHERE s.station_code = f.station_code
+ AND m.timestamp > f.as_of AND m.timestamp <= {window_end}) AS observed_n
+ FROM forecast_history f
+ WHERE f.station_code = :code AND f.horizon_hours = :horizon
+ AND f.source = 'model' AND f.predicted_max_level IS NOT NULL
+ AND f.as_of <= :verifiable_before
+ ORDER BY f.as_of
+ """
+
+
+def compute_skill(
+ engine,
+ db_type: str,
+ station_code: str = DEFAULT_STATION,
+ horizon_hours: int = DEFAULT_HORIZON,
+ now: Optional[datetime.datetime] = None,
+) -> Dict:
+ """Per-model-version verification of issued forecasts against observations."""
+ from sqlalchemy import text
+
+ now = now or datetime.datetime.now()
+ verifiable_before = now - datetime.timedelta(hours=horizon_hours)
+ with engine.connect() as conn:
+ rows = [
+ dict(r._mapping)
+ for r in conn.execute(
+ text(_sql_for(db_type)),
+ {
+ "code": station_code,
+ "horizon": horizon_hours,
+ "verifiable_before": verifiable_before,
+ },
+ )
+ ]
+
+ def _ts(value):
+ # sqlite hands back strings; postgres/mysql give datetimes
+ if isinstance(value, datetime.datetime):
+ return value
+ return datetime.datetime.fromisoformat(str(value).replace(" ", "T"))
+
+ by_version: Dict[str, List[dict]] = {}
+ for r in rows:
+ r["as_of"] = _ts(r["as_of"])
+ # need most of the window observed, or the "max" is not the peak
+ if r["observed_max"] is None or (r["observed_n"] or 0) < horizon_hours * 0.75:
+ continue
+ by_version.setdefault(r["model_version"] or "unknown", []).append(r)
+
+ versions = []
+ for version, vrows in by_version.items():
+ pred = [float(r["predicted_max_level"]) for r in vrows]
+ obs = [float(r["observed_max"]) for r in vrows]
+ cur = [
+ float(r["current_level"]) if r["current_level"] is not None else None
+ for r in vrows
+ ]
+ err = [p - o for p, o in zip(pred, obs)]
+ mae = sum(abs(e) for e in err) / len(err)
+ bias = sum(err) / len(err)
+ pers_rows = [(c, o) for c, o in zip(cur, obs) if c is not None]
+ persistence = (
+ sum(abs(c - o) for c, o in pers_rows) / len(pers_rows)
+ if pers_rows
+ else None
+ )
+ high = [(p, o) for p, o in zip(pred, obs) if o >= 2.0]
+ versions.append(
+ {
+ "model_version": version,
+ "first_issued": min(r["as_of"] for r in vrows).isoformat(),
+ "last_issued": max(r["as_of"] for r in vrows).isoformat(),
+ "n": len(vrows),
+ "mae_m": round(mae, 3),
+ "bias_m": round(bias, 3),
+ "persistence_mae_m": (
+ None if persistence is None else round(persistence, 3)
+ ),
+ "skill": (
+ None if not persistence else round(1.0 - mae / persistence, 3)
+ ),
+ "above_2m_n": len(high),
+ "above_2m_mae_m": (
+ round(sum(abs(p - o) for p, o in high) / len(high), 3)
+ if high
+ else None
+ ),
+ "enough_data": len(vrows) >= MIN_VERIFIED,
+ }
+ )
+ versions.sort(key=lambda v: v["first_issued"])
+
+ # Headline: current version vs the previous one that had enough data
+ current = versions[-1] if versions else None
+ previous = (
+ next((v for v in reversed(versions[:-1]) if v["enough_data"]), None)
+ if versions
+ else None
+ )
+ trend = None
+ if current and previous and current["enough_data"]:
+ trend = {
+ "previous_version": previous["model_version"],
+ "mae_delta_m": round(current["mae_m"] - previous["mae_m"], 3),
+ "skill_delta": (
+ None
+ if current["skill"] is None or previous["skill"] is None
+ else round(current["skill"] - previous["skill"], 3)
+ ),
+ "better": current["mae_m"] < previous["mae_m"],
+ }
+
+ return {
+ "station_code": station_code,
+ "horizon_hours": horizon_hours,
+ "verified_until": verifiable_before.isoformat(),
+ "min_verified": MIN_VERIFIED,
+ "versions": versions,
+ "current": current,
+ "trend": trend,
+ }
diff --git a/src/static/dashboard.html b/src/static/dashboard.html
index 3c07e1e..2074a83 100644
--- a/src/static/dashboard.html
+++ b/src/static/dashboard.html
@@ -155,6 +155,18 @@
.stat-value { margin-top: 9px; font-size: 1.65rem; font-weight: 800; letter-spacing: -.04em; white-space: nowrap; }
.stat-note { color: var(--muted); margin-top: 3px; font-size: .77rem; }
.stat.stale { border-color: var(--red); background: rgba(204,75,55,.08); }
+ .skill-panel { border: 1px solid var(--border); border-radius: 12px; padding: 12px 14px; margin-top: 14px; background: var(--surface-3); }
+ .skill-head { display: flex; justify-content: space-between; align-items: baseline; gap: 12px; flex-wrap: wrap; }
+ .skill-headline { margin: 8px 0 10px; font-weight: 700; font-size: .9rem; }
+ .skill-headline.better { color: var(--green); }
+ .skill-headline.worse { color: var(--amber); }
+ .skill-table-wrap { overflow-x: auto; }
+ .skill-table { border-collapse: collapse; font-size: .76rem; width: 100%; min-width: 560px; }
+ .skill-table th { text-align: left; color: var(--muted); font-weight: 700; font-size: .66rem; text-transform: uppercase; letter-spacing: .06em; padding: 4px 8px; border-bottom: 1px solid var(--border); }
+ .skill-table td { padding: 5px 8px; border-bottom: 1px solid var(--border); white-space: nowrap; }
+ .skill-table tr.current td { font-weight: 700; }
+ .skill-table td.num { text-align: right; font-variant-numeric: tabular-nums; }
+ .skill-table td.dim { color: var(--muted); }
.stat.stale .stat-value, .stat.stale .stat-note { color: var(--red); }
.workspace { display: grid; grid-template-columns: minmax(0, 1fr) 330px; gap: 14px; min-height: 640px; }
.map-card, .side-card { background: var(--card); border: 1px solid var(--border); border-radius: 19px; box-shadow: var(--shadow); overflow: hidden; }
@@ -417,6 +429,16 @@
Chance the river reaches each official inundation stage within 24 h — city flooding begins at stage 1 (3.70 m); each stage floods additional districts.
+
+
+ Is the model getting better?
+
+
+
+
+
+
Every hour the deployed model's 24 h peak forecast for P.1 is stored; once those 24 hours have passed it is compared with what the river actually did. "Skill" is how much better the model was than assuming the level stays where it is (0 = no better, 1 = perfect). Versions retrained on more data appear as new rows, so improvement, or its absence, is visible here rather than claimed.
+
@@ -556,6 +578,26 @@
'forecast.chip.peak': (lvl) => ` · peak ~${lvl} m`,
'forecast.chip.heuristic': ' · heuristic fallback',
'forecast.expand': (n) => `Show all ${n} station forecasts ▾`,
+ 'skill.title': 'Is the model getting better?',
+ 'skill.sub': (n, since) => `${n} verified 24 h forecasts for P.1 since ${since}`,
+ 'skill.explain': 'Every hour the deployed model\'s 24 h peak forecast for P.1 is stored; once those 24 hours have passed it is compared with what the river actually did. "Skill" is how much better the model was than assuming the level stays where it is (0 = no better, 1 = perfect). Versions retrained on more data appear as new rows, so improvement, or its absence, is visible here rather than claimed.',
+ 'skill.better': (v, prev, d) => `Current model ${v} is more accurate than ${prev}: peak error ${d} cm lower on the hours it has served.`,
+ 'skill.worse': (v, prev, d) => `Current model ${v} has a higher peak error than ${prev} so far (+${d} cm).`,
+ 'skill.caveat.quiet': 'All verified hours so far were below 2 m: this measures quiet-river accuracy only. The model is built and judged for flood onset (lead time before 3.70 m), which no quiet week can test — see the backtests in the documentation.',
+ 'skill.caveat.regime': 'Versions served different weeks; the ≥ 2 m column compares them on the hours that matter.',
+ 'skill.single': (v) => `Only ${v} has enough verified hours yet; the next retrain adds a row to compare.`,
+ 'skill.young': (v, n, min) => `${v} has ${n} verified hours; a comparison needs ${min}.`,
+ 'skill.none': 'No verified forecasts yet — the first appear 24 h after a model starts serving.',
+ 'skill.col.version': 'Model',
+ 'skill.col.period': 'Served',
+ 'skill.col.n': 'Hours',
+ 'skill.col.mae': 'Peak error',
+ 'skill.col.bias': 'Bias',
+ 'skill.col.pers': 'Persistence',
+ 'skill.col.skill': 'Skill',
+ 'skill.col.high': '≥ 2 m error',
+ 'skill.cm': (v) => `${v} cm`,
+ 'skill.na': '—',
'forecast.collapse': 'Hide station forecasts ▴',
'outlook.title': 'Chiang Mai city flood outlook · P.1 Nawarat Bridge',
'outlook.explainer': 'Chance the river reaches each official inundation stage within 24 h — city flooding begins at stage 1 (3.70 m); each stage floods additional districts.',
@@ -732,6 +774,26 @@
'forecast.chip.peak': (lvl) => ` · ระดับสูงสุดประมาณ ${lvl} ม.`,
'forecast.chip.heuristic': ' · ใช้การประมาณอย่างง่าย',
'forecast.expand': (n) => `แสดงพยากรณ์ทั้ง ${n} สถานี ▾`,
+ 'skill.title': 'โมเดลแม่นยำขึ้นหรือไม่?',
+ 'skill.sub': (n, since) => `พยากรณ์ 24 ชม. ของ P.1 ที่ตรวจสอบแล้ว ${n} ครั้ง ตั้งแต่ ${since}`,
+ 'skill.explain': 'ทุกชั่วโมงระบบบันทึกค่าพยากรณ์ระดับน้ำสูงสุดใน 24 ชม. ของ P.1 ไว้ เมื่อครบ 24 ชม. จึงนำมาเทียบกับระดับน้ำจริง "ทักษะ" คือโมเดลดีกว่าการสมมติว่าระดับน้ำคงที่มากเพียงใด (0 = ไม่ดีกว่า, 1 = สมบูรณ์แบบ) โมเดลที่ฝึกใหม่ด้วยข้อมูลมากขึ้นจะปรากฏเป็นแถวใหม่ จึงเห็นได้ว่าดีขึ้นจริงหรือไม่',
+ 'skill.better': (v, prev, d) => `โมเดลปัจจุบัน ${v} แม่นยำกว่า ${prev}: ค่าคลาดเคลื่อนต่ำกว่า ${d} ซม. ในช่วงที่ให้บริการ`,
+ 'skill.worse': (v, prev, d) => `โมเดลปัจจุบัน ${v} มีค่าคลาดเคลื่อนสูงกว่า ${prev} (+${d} ซม.)`,
+ 'skill.caveat.quiet': 'ชั่วโมงที่ตรวจสอบทั้งหมดอยู่ต่ำกว่า 2 ม.: วัดได้เพียงความแม่นยำช่วงน้ำปกติ โมเดลถูกสร้างและประเมินสำหรับช่วงน้ำเริ่มท่วม (เวลาเตือนล่วงหน้าก่อน 3.70 ม.) ซึ่งสัปดาห์ปกติทดสอบไม่ได้ — ดูผลทดสอบย้อนหลังในเอกสาร',
+ 'skill.caveat.regime': 'แต่ละเวอร์ชันให้บริการคนละช่วงเวลา คอลัมน์ ≥ 2 ม. เปรียบเทียบเฉพาะชั่วโมงที่สำคัญ',
+ 'skill.single': (v) => `มีเพียง ${v} ที่มีข้อมูลตรวจสอบเพียงพอ การฝึกครั้งถัดไปจะเพิ่มแถวให้เปรียบเทียบ`,
+ 'skill.young': (v, n, min) => `${v} มีข้อมูลตรวจสอบ ${n} ชั่วโมง ต้องการอย่างน้อย ${min} เพื่อเปรียบเทียบ`,
+ 'skill.none': 'ยังไม่มีพยากรณ์ที่ตรวจสอบได้ — จะเริ่มมี 24 ชม. หลังโมเดลเริ่มทำงาน',
+ 'skill.col.version': 'โมเดล',
+ 'skill.col.period': 'ช่วงเวลา',
+ 'skill.col.n': 'ชั่วโมง',
+ 'skill.col.mae': 'คลาดเคลื่อน',
+ 'skill.col.bias': 'อคติ',
+ 'skill.col.pers': 'ระดับคงที่',
+ 'skill.col.skill': 'ทักษะ',
+ 'skill.col.high': 'คลาดเคลื่อน ≥ 2 ม.',
+ 'skill.cm': (v) => `${v} ซม.`,
+ 'skill.na': '—',
'forecast.collapse': 'ซ่อนพยากรณ์รายสถานี ▴',
'outlook.title': 'แนวโน้มน้ำท่วมเมืองเชียงใหม่ · P.1 สะพานนวรัฐ',
'outlook.explainer': 'โอกาสที่ระดับน้ำจะถึงแต่ละระดับการท่วมตามประกาศทางการภายใน 24 ชม. — น้ำเริ่มท่วมเมืองที่ระดับ 1 (3.70 ม.) และแต่ละระดับจะท่วมพื้นที่เพิ่มขึ้น',
@@ -1989,11 +2051,59 @@
? t('forecast.collapse')
: t('forecast.expand', stations.length);
card.style.display = 'block';
+ loadSkill(); // non-blocking; panel stays hidden until there is verified data
} catch (error) {
card.style.display = 'none';
}
}
+ async function loadSkill() {
+ const panel = $('skill-panel');
+ try {
+ const response = await fetch('/api/forecast/skill?station_code=P.1&horizon=24');
+ if (!response.ok) throw new Error(`HTTP ${response.status}`);
+ const data = await response.json();
+ const versions = (data.versions || []).filter((v) => v.n > 0);
+ if (!versions.length) { panel.style.display = 'none'; return; }
+ const cm = (m) => m == null ? t('skill.na') : t('skill.cm', (m * 100).toFixed(1));
+ const fmtDay = (v) => parseTs(v).toLocaleDateString(loc(), { timeZone: TZ, day: 'numeric', month: 'short' });
+ const total = versions.reduce((a, v) => a + v.n, 0);
+ $('skill-sub').textContent = t('skill.sub', total.toLocaleString(loc()), fmtDay(versions[0].first_issued));
+ const head = $('skill-headline');
+ head.className = 'skill-headline';
+ const cur = data.current;
+ if (data.trend && cur) {
+ const delta = Math.abs(data.trend.mae_delta_m * 100).toFixed(1);
+ head.textContent = data.trend.better
+ ? t('skill.better', cur.model_version, data.trend.previous_version, delta)
+ : t('skill.worse', cur.model_version, data.trend.previous_version, delta);
+ head.classList.add(data.trend.better ? 'better' : 'worse');
+ } else if (cur && cur.enough_data) {
+ head.textContent = t('skill.single', cur.model_version);
+ } else if (cur) {
+ head.textContent = t('skill.young', cur.model_version, cur.n, data.min_verified);
+ } else head.textContent = t('skill.none');
+ const anyHigh = versions.some((v) => v.above_2m_n > 0);
+ const compared = versions.filter((v) => v.enough_data).length > 1;
+ $('skill-caveat').textContent = !anyHigh ? t('skill.caveat.quiet') : compared ? t('skill.caveat.regime') : '';
+ const cols = ['version', 'period', 'n', 'mae', 'bias', 'pers', 'skill', 'high'];
+ const rows = versions.map((v) => `