feat: "Is the model getting better?" - live verification per model version

src/ml/skill.py joins forecast_history (what each deployed version
predicted for the 24 h peak, hourly) to water_measurements (what the
river did) and reports per version: verified hours, peak MAE, bias, the
persistence baseline (peak = current level), skill = 1 - MAE/persistence,
and the same MAE restricted to observed peaks >= 2 m. Only forecasts
whose window has elapsed with >= 75 % of hours observed count; a
version needs 24 verified hours before it is compared.

GET /api/forecast/skill?station_code=P.1&horizon=24 returns it (SWR
cached, 15 min). The dashboard's forecast card gains a panel with a
one-line verdict (current vs previous version), the per-version table,
and a caveat that quiet weeks measure quiet-river accuracy only: the
model is judged on flood-onset lead, which the backtests cover. EN + TH.

On today's production data: hgb-v3+28b62e5 (369 h, Aug 13 - Sep 1)
MAE 15.2 cm, skill -0.05; hgb-v2+f6570ac (224 h, Sep 1 - 11) MAE
12.3 cm, skill 0.36 - the "worse" v2 model scores better on a quieter
fortnight, which is exactly why the panel shows the >= 2 m column and
the caveat. Tests: 3, sqlite, synthetic.

scripts/dev_proxy.py: DEV_PROXY_LOCAL lets a not-yet-deployed endpoint be
answered from a local JSON file while everything else goes to prod.

scripts/dev_proxy.py

@@ -3,6 +3,7 @@ live server, so browser-side changes can be checked against real data
 before deploy. Usage: python scripts/dev_proxy.py [port]"""
 
 import http.server
+import os
 import sys
 import urllib.request
 from pathlib import Path
@@ -17,6 +18,12 @@ class Handler(http.server.BaseHTTPRequestHandler):
             body = (STATIC / "dashboard.html").read_bytes()
             self._send(200, "text/html; charset=utf-8", body)
             return
+        # Local overrides for endpoints not yet deployed: DEV_PROXY_LOCAL=/api/x=file.json,...
+        for pair in filter(None, os.environ.get("DEV_PROXY_LOCAL", "").split(",")):
+            prefix, file = pair.split("=", 1)
+            if self.path.split("?")[0] == prefix:
+                self._send(200, "application/json", Path(file).read_bytes())
+                return
         if self.path.startswith("/static/"):
             f = STATIC / self.path[len("/static/"):].split("?")[0]
             if f.is_file():

src/ml/skill.py

@@ -0,0 +1,170 @@
+"""Live forecast skill: what the deployed model said versus what the river did.
+
+Every hour the precompute stores the issued 24 h forecast (forecast_history);
+water_measurements holds what actually happened. Joining the two gives a
+verification that needs no retraining and answers the question the dashboard
+is asked most: "is the model getting better?" — per model version, on the
+hours that version was actually serving.
+
+Metrics per version and horizon:
+  n           verified forecasts (issued, and the horizon has since elapsed)
+  mae         |predicted_max - observed_max| over the horizon window, metres
+  bias        mean(predicted - observed): >0 over-predicts the peak
+  persistence MAE of the trivial "peak = current level" forecast on the
+              same rows; a model is only useful if it beats this
+  skill       1 - mae/persistence  (0 = no better than persistence, 1 = perfect)
+  above_2m    same MAE restricted to rows where the observed peak >= 2 m,
+              i.e. the flood-relevant regime
+
+Only the P.1 gauge is verified by default: it is the one the city threshold
+is keyed to, and one station keeps the query cheap enough to run on request.
+"""
+
+import datetime
+import logging
+from typing import Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_STATION = "P.1"
+DEFAULT_HORIZON = 24
+MIN_VERIFIED = 24  # fewer than a day of verified hours is not a number
+
+
+def _sql_for(db_type: str) -> str:
+    """Join each issued forecast to the observed max over (as_of, as_of + h]."""
+    if db_type == "postgresql":
+        window_end = "f.as_of + (f.horizon_hours || ' hours')::interval"
+    elif db_type == "mysql":
+        window_end = "DATE_ADD(f.as_of, INTERVAL f.horizon_hours HOUR)"
+    else:  # sqlite
+        window_end = "datetime(f.as_of, '+' || f.horizon_hours || ' hours')"
+    return f"""
+        SELECT f.as_of, f.model_version, f.predicted_max_level, f.current_level,
+               (SELECT MAX(m.water_level) FROM water_measurements m
+                 JOIN stations s ON s.id = m.station_id
+                WHERE s.station_code = f.station_code
+                  AND m.timestamp > f.as_of AND m.timestamp <= {window_end}) AS observed_max,
+               (SELECT COUNT(m.water_level) FROM water_measurements m
+                 JOIN stations s ON s.id = m.station_id
+                WHERE s.station_code = f.station_code
+                  AND m.timestamp > f.as_of AND m.timestamp <= {window_end}) AS observed_n
+        FROM forecast_history f
+        WHERE f.station_code = :code AND f.horizon_hours = :horizon
+          AND f.source = 'model' AND f.predicted_max_level IS NOT NULL
+          AND f.as_of <= :verifiable_before
+        ORDER BY f.as_of
+    """
+
+
+def compute_skill(
+    engine,
+    db_type: str,
+    station_code: str = DEFAULT_STATION,
+    horizon_hours: int = DEFAULT_HORIZON,
+    now: Optional[datetime.datetime] = None,
+) -> Dict:
+    """Per-model-version verification of issued forecasts against observations."""
+    from sqlalchemy import text
+
+    now = now or datetime.datetime.now()
+    verifiable_before = now - datetime.timedelta(hours=horizon_hours)
+    with engine.connect() as conn:
+        rows = [
+            dict(r._mapping)
+            for r in conn.execute(
+                text(_sql_for(db_type)),
+                {
+                    "code": station_code,
+                    "horizon": horizon_hours,
+                    "verifiable_before": verifiable_before,
+                },
+            )
+        ]
+
+    def _ts(value):
+        # sqlite hands back strings; postgres/mysql give datetimes
+        if isinstance(value, datetime.datetime):
+            return value
+        return datetime.datetime.fromisoformat(str(value).replace(" ", "T"))
+
+    by_version: Dict[str, List[dict]] = {}
+    for r in rows:
+        r["as_of"] = _ts(r["as_of"])
+        # need most of the window observed, or the "max" is not the peak
+        if r["observed_max"] is None or (r["observed_n"] or 0) < horizon_hours * 0.75:
+            continue
+        by_version.setdefault(r["model_version"] or "unknown", []).append(r)
+
+    versions = []
+    for version, vrows in by_version.items():
+        pred = [float(r["predicted_max_level"]) for r in vrows]
+        obs = [float(r["observed_max"]) for r in vrows]
+        cur = [
+            float(r["current_level"]) if r["current_level"] is not None else None
+            for r in vrows
+        ]
+        err = [p - o for p, o in zip(pred, obs)]
+        mae = sum(abs(e) for e in err) / len(err)
+        bias = sum(err) / len(err)
+        pers_rows = [(c, o) for c, o in zip(cur, obs) if c is not None]
+        persistence = (
+            sum(abs(c - o) for c, o in pers_rows) / len(pers_rows)
+            if pers_rows
+            else None
+        )
+        high = [(p, o) for p, o in zip(pred, obs) if o >= 2.0]
+        versions.append(
+            {
+                "model_version": version,
+                "first_issued": min(r["as_of"] for r in vrows).isoformat(),
+                "last_issued": max(r["as_of"] for r in vrows).isoformat(),
+                "n": len(vrows),
+                "mae_m": round(mae, 3),
+                "bias_m": round(bias, 3),
+                "persistence_mae_m": (
+                    None if persistence is None else round(persistence, 3)
+                ),
+                "skill": (
+                    None if not persistence else round(1.0 - mae / persistence, 3)
+                ),
+                "above_2m_n": len(high),
+                "above_2m_mae_m": (
+                    round(sum(abs(p - o) for p, o in high) / len(high), 3)
+                    if high
+                    else None
+                ),
+                "enough_data": len(vrows) >= MIN_VERIFIED,
+            }
+        )
+    versions.sort(key=lambda v: v["first_issued"])
+
+    # Headline: current version vs the previous one that had enough data
+    current = versions[-1] if versions else None
+    previous = (
+        next((v for v in reversed(versions[:-1]) if v["enough_data"]), None)
+        if versions
+        else None
+    )
+    trend = None
+    if current and previous and current["enough_data"]:
+        trend = {
+            "previous_version": previous["model_version"],
+            "mae_delta_m": round(current["mae_m"] - previous["mae_m"], 3),
+            "skill_delta": (
+                None
+                if current["skill"] is None or previous["skill"] is None
+                else round(current["skill"] - previous["skill"], 3)
+            ),
+            "better": current["mae_m"] < previous["mae_m"],
+        }
+
+    return {
+        "station_code": station_code,
+        "horizon_hours": horizon_hours,
+        "verified_until": verifiable_before.isoformat(),
+        "min_verified": MIN_VERIFIED,
+        "versions": versions,
+        "current": current,
+        "trend": trend,
+    }

src/static/dashboard.html

@@ -155,6 +155,18 @@
         .stat-value { margin-top: 9px; font-size: 1.65rem; font-weight: 800; letter-spacing: -.04em; white-space: nowrap; }
         .stat-note { color: var(--muted); margin-top: 3px; font-size: .77rem; }
         .stat.stale { border-color: var(--red); background: rgba(204,75,55,.08); }
+        .skill-panel { border: 1px solid var(--border); border-radius: 12px; padding: 12px 14px; margin-top: 14px; background: var(--surface-3); }
+        .skill-head { display: flex; justify-content: space-between; align-items: baseline; gap: 12px; flex-wrap: wrap; }
+        .skill-headline { margin: 8px 0 10px; font-weight: 700; font-size: .9rem; }
+        .skill-headline.better { color: var(--green); }
+        .skill-headline.worse { color: var(--amber); }
+        .skill-table-wrap { overflow-x: auto; }
+        .skill-table { border-collapse: collapse; font-size: .76rem; width: 100%; min-width: 560px; }
+        .skill-table th { text-align: left; color: var(--muted); font-weight: 700; font-size: .66rem; text-transform: uppercase; letter-spacing: .06em; padding: 4px 8px; border-bottom: 1px solid var(--border); }
+        .skill-table td { padding: 5px 8px; border-bottom: 1px solid var(--border); white-space: nowrap; }
+        .skill-table tr.current td { font-weight: 700; }
+        .skill-table td.num { text-align: right; font-variant-numeric: tabular-nums; }
+        .skill-table td.dim { color: var(--muted); }
         .stat.stale .stat-value, .stat.stale .stat-note { color: var(--red); }
         .workspace { display: grid; grid-template-columns: minmax(0, 1fr) 330px; gap: 14px; min-height: 640px; }
         .map-card, .side-card { background: var(--card); border: 1px solid var(--border); border-radius: 19px; box-shadow: var(--shadow); overflow: hidden; }
@@ -417,6 +429,16 @@
             <div class="p1-peak" style="margin-top:7px" data-i18n="outlook.explainer">Chance the river reaches each official inundation stage within 24 h — city flooding begins at stage 1 (3.70 m); each stage floods additional districts.</div>
         </div>
         <button type="button" class="zones-button" id="forecast-expand" style="display:none;margin-top:12px">Show all station forecasts ▾</button>
+        <div class="skill-panel" id="skill-panel" style="display:none">
+            <div class="skill-head">
+                <strong data-i18n="skill.title">Is the model getting better?</strong>
+                <span class="subtitle" id="skill-sub"></span>
+            </div>
+            <div class="skill-headline" id="skill-headline"></div>
+            <p class="subtitle skill-caveat" id="skill-caveat" style="margin:-4px 0 10px"></p>
+            <div class="skill-table-wrap"><table class="skill-table" id="skill-table"></table></div>
+            <p class="subtitle" style="margin:8px 0 0" data-i18n="skill.explain">Every hour the deployed model's 24 h peak forecast for P.1 is stored; once those 24 hours have passed it is compared with what the river actually did. "Skill" is how much better the model was than assuming the level stays where it is (0 = no better, 1 = perfect). Versions retrained on more data appear as new rows, so improvement, or its absence, is visible here rather than claimed.</p>
+        </div>
         <div class="forecast-grid" id="forecast-grid" style="display:none"></div>
     </section>
 
@@ -556,6 +578,26 @@
             'forecast.chip.peak': (lvl) => ` · peak ~${lvl} m`,
             'forecast.chip.heuristic': ' · heuristic fallback',
             'forecast.expand': (n) => `Show all ${n} station forecasts ▾`,
+            'skill.title': 'Is the model getting better?',
+            'skill.sub': (n, since) => `${n} verified 24 h forecasts for P.1 since ${since}`,
+            'skill.explain': 'Every hour the deployed model\'s 24 h peak forecast for P.1 is stored; once those 24 hours have passed it is compared with what the river actually did. "Skill" is how much better the model was than assuming the level stays where it is (0 = no better, 1 = perfect). Versions retrained on more data appear as new rows, so improvement, or its absence, is visible here rather than claimed.',
+            'skill.better': (v, prev, d) => `Current model ${v} is more accurate than ${prev}: peak error ${d} cm lower on the hours it has served.`,
+            'skill.worse': (v, prev, d) => `Current model ${v} has a higher peak error than ${prev} so far (+${d} cm).`,
+            'skill.caveat.quiet': 'All verified hours so far were below 2 m: this measures quiet-river accuracy only. The model is built and judged for flood onset (lead time before 3.70 m), which no quiet week can test — see the backtests in the documentation.',
+            'skill.caveat.regime': 'Versions served different weeks; the ≥ 2 m column compares them on the hours that matter.',
+            'skill.single': (v) => `Only ${v} has enough verified hours yet; the next retrain adds a row to compare.`,
+            'skill.young': (v, n, min) => `${v} has ${n} verified hours; a comparison needs ${min}.`,
+            'skill.none': 'No verified forecasts yet — the first appear 24 h after a model starts serving.',
+            'skill.col.version': 'Model',
+            'skill.col.period': 'Served',
+            'skill.col.n': 'Hours',
+            'skill.col.mae': 'Peak error',
+            'skill.col.bias': 'Bias',
+            'skill.col.pers': 'Persistence',
+            'skill.col.skill': 'Skill',
+            'skill.col.high': '≥ 2 m error',
+            'skill.cm': (v) => `${v} cm`,
+            'skill.na': '—',
             'forecast.collapse': 'Hide station forecasts ▴',
             'outlook.title': 'Chiang Mai city flood outlook · P.1 Nawarat Bridge',
             'outlook.explainer': 'Chance the river reaches each official inundation stage within 24 h — city flooding begins at stage 1 (3.70 m); each stage floods additional districts.',
@@ -732,6 +774,26 @@
             'forecast.chip.peak': (lvl) => ` · ระดับสูงสุดประมาณ ${lvl} ม.`,
             'forecast.chip.heuristic': ' · ใช้การประมาณอย่างง่าย',
             'forecast.expand': (n) => `แสดงพยากรณ์ทั้ง ${n} สถานี ▾`,
+            'skill.title': 'โมเดลแม่นยำขึ้นหรือไม่?',
+            'skill.sub': (n, since) => `พยากรณ์ 24 ชม. ของ P.1 ที่ตรวจสอบแล้ว ${n} ครั้ง ตั้งแต่ ${since}`,
+            'skill.explain': 'ทุกชั่วโมงระบบบันทึกค่าพยากรณ์ระดับน้ำสูงสุดใน 24 ชม. ของ P.1 ไว้ เมื่อครบ 24 ชม. จึงนำมาเทียบกับระดับน้ำจริง "ทักษะ" คือโมเดลดีกว่าการสมมติว่าระดับน้ำคงที่มากเพียงใด (0 = ไม่ดีกว่า, 1 = สมบูรณ์แบบ) โมเดลที่ฝึกใหม่ด้วยข้อมูลมากขึ้นจะปรากฏเป็นแถวใหม่ จึงเห็นได้ว่าดีขึ้นจริงหรือไม่',
+            'skill.better': (v, prev, d) => `โมเดลปัจจุบัน ${v} แม่นยำกว่า ${prev}: ค่าคลาดเคลื่อนต่ำกว่า ${d} ซม. ในช่วงที่ให้บริการ`,
+            'skill.worse': (v, prev, d) => `โมเดลปัจจุบัน ${v} มีค่าคลาดเคลื่อนสูงกว่า ${prev} (+${d} ซม.)`,
+            'skill.caveat.quiet': 'ชั่วโมงที่ตรวจสอบทั้งหมดอยู่ต่ำกว่า 2 ม.: วัดได้เพียงความแม่นยำช่วงน้ำปกติ โมเดลถูกสร้างและประเมินสำหรับช่วงน้ำเริ่มท่วม (เวลาเตือนล่วงหน้าก่อน 3.70 ม.) ซึ่งสัปดาห์ปกติทดสอบไม่ได้ — ดูผลทดสอบย้อนหลังในเอกสาร',
+            'skill.caveat.regime': 'แต่ละเวอร์ชันให้บริการคนละช่วงเวลา คอลัมน์ ≥ 2 ม. เปรียบเทียบเฉพาะชั่วโมงที่สำคัญ',
+            'skill.single': (v) => `มีเพียง ${v} ที่มีข้อมูลตรวจสอบเพียงพอ การฝึกครั้งถัดไปจะเพิ่มแถวให้เปรียบเทียบ`,
+            'skill.young': (v, n, min) => `${v} มีข้อมูลตรวจสอบ ${n} ชั่วโมง ต้องการอย่างน้อย ${min} เพื่อเปรียบเทียบ`,
+            'skill.none': 'ยังไม่มีพยากรณ์ที่ตรวจสอบได้ — จะเริ่มมี 24 ชม. หลังโมเดลเริ่มทำงาน',
+            'skill.col.version': 'โมเดล',
+            'skill.col.period': 'ช่วงเวลา',
+            'skill.col.n': 'ชั่วโมง',
+            'skill.col.mae': 'คลาดเคลื่อน',
+            'skill.col.bias': 'อคติ',
+            'skill.col.pers': 'ระดับคงที่',
+            'skill.col.skill': 'ทักษะ',
+            'skill.col.high': 'คลาดเคลื่อน ≥ 2 ม.',
+            'skill.cm': (v) => `${v} ซม.`,
+            'skill.na': '—',
             'forecast.collapse': 'ซ่อนพยากรณ์รายสถานี ▴',
             'outlook.title': 'แนวโน้มน้ำท่วมเมืองเชียงใหม่ · P.1 สะพานนวรัฐ',
             'outlook.explainer': 'โอกาสที่ระดับน้ำจะถึงแต่ละระดับการท่วมตามประกาศทางการภายใน 24 ชม. — น้ำเริ่มท่วมเมืองที่ระดับ 1 (3.70 ม.) และแต่ละระดับจะท่วมพื้นที่เพิ่มขึ้น',
@@ -1989,11 +2051,59 @@
                 ? t('forecast.collapse')
                 : t('forecast.expand', stations.length);
             card.style.display = 'block';
+            loadSkill(); // non-blocking; panel stays hidden until there is verified data
         } catch (error) {
             card.style.display = 'none';
         }
     }
 
+    async function loadSkill() {
+        const panel = $('skill-panel');
+        try {
+            const response = await fetch('/api/forecast/skill?station_code=P.1&horizon=24');
+            if (!response.ok) throw new Error(`HTTP ${response.status}`);
+            const data = await response.json();
+            const versions = (data.versions || []).filter((v) => v.n > 0);
+            if (!versions.length) { panel.style.display = 'none'; return; }
+            const cm = (m) => m == null ? t('skill.na') : t('skill.cm', (m * 100).toFixed(1));
+            const fmtDay = (v) => parseTs(v).toLocaleDateString(loc(), { timeZone: TZ, day: 'numeric', month: 'short' });
+            const total = versions.reduce((a, v) => a + v.n, 0);
+            $('skill-sub').textContent = t('skill.sub', total.toLocaleString(loc()), fmtDay(versions[0].first_issued));
+            const head = $('skill-headline');
+            head.className = 'skill-headline';
+            const cur = data.current;
+            if (data.trend && cur) {
+                const delta = Math.abs(data.trend.mae_delta_m * 100).toFixed(1);
+                head.textContent = data.trend.better
+                    ? t('skill.better', cur.model_version, data.trend.previous_version, delta)
+                    : t('skill.worse', cur.model_version, data.trend.previous_version, delta);
+                head.classList.add(data.trend.better ? 'better' : 'worse');
+            } else if (cur && cur.enough_data) {
+                head.textContent = t('skill.single', cur.model_version);
+            } else if (cur) {
+                head.textContent = t('skill.young', cur.model_version, cur.n, data.min_verified);
+            } else head.textContent = t('skill.none');
+            const anyHigh = versions.some((v) => v.above_2m_n > 0);
+            const compared = versions.filter((v) => v.enough_data).length > 1;
+            $('skill-caveat').textContent = !anyHigh ? t('skill.caveat.quiet') : compared ? t('skill.caveat.regime') : '';
+            const cols = ['version', 'period', 'n', 'mae', 'bias', 'pers', 'skill', 'high'];
+            const rows = versions.map((v) => `<tr class="${v === cur ? 'current' : ''}${v.enough_data ? '' : ' young'}">`
+                + `<td>${escapeHtml(v.model_version)}</td>`
+                + `<td class="dim">${fmtDay(v.first_issued)} – ${fmtDay(v.last_issued)}</td>`
+                + `<td class="num">${v.n.toLocaleString(loc())}</td>`
+                + `<td class="num">${cm(v.mae_m)}</td>`
+                + `<td class="num">${v.bias_m == null ? t('skill.na') : (v.bias_m >= 0 ? '+' : '') + (v.bias_m * 100).toFixed(1)}</td>`
+                + `<td class="num dim">${cm(v.persistence_mae_m)}</td>`
+                + `<td class="num">${v.skill == null ? t('skill.na') : v.skill.toFixed(2)}</td>`
+                + `<td class="num">${v.above_2m_n ? `${cm(v.above_2m_mae_m)} <span class="dim">(${v.above_2m_n})</span>` : t('skill.na')}</td>`
+                + '</tr>').join('');
+            $('skill-table').innerHTML = `<thead><tr>${cols.map((c) => `<th>${escapeHtml(t('skill.col.' + c))}</th>`).join('')}</tr></thead><tbody>${rows}</tbody>`;
+            panel.style.display = 'block';
+        } catch (error) {
+            panel.style.display = 'none';
+        }
+    }
+
     async function loadDbStats() {
         const strip = $('db-stats');
         try {

src/web_api.py

@@ -864,7 +864,7 @@ def _hii_rows(sql: str, params: Dict[str, Any]) -> List[Dict[str, Any]]:
 # flood, slightly stale readings with a visible timestamp beat an error page.
 HII_CACHE: Dict[str, Any] = {}
 HII_CACHE_LOCK = Lock()
-_HII_COMPUTE_LOCKS = {"rain": Lock(), "waterlevel": Lock()}
+_HII_COMPUTE_LOCKS = {"rain": Lock(), "waterlevel": Lock(), "skill": Lock()}
 LATEST_CACHE: Dict[str, Any] = {}
 LATEST_CACHE_LOCK = Lock()
 _LATEST_COMPUTE_LOCK = Lock()
@@ -1230,6 +1230,56 @@ async def get_forecast_history(
     return await asyncio.to_thread(store.fetch, station_code, start_dt, end_dt, horizon)
 
 
+@app.get("/api/forecast/skill")
+async def get_forecast_skill(
+    response: Response,
+    station_code: str = Query("P.1"),
+    horizon: int = Query(24, ge=1, le=48),
+):
+    """Is the model getting better? Issued forecasts verified against what the
+    river then did, per model version, with a persistence baseline.
+
+    Read from forecast_history (what each deployed version predicted, hourly)
+    joined to water_measurements; no retraining involved. Cached like the HII
+    feeds because the join is a few hundred correlated subqueries.
+    """
+    increment_counter("api_requests", labels={"endpoint": "forecast_skill"})
+    store = app_state.get("forecast_store")
+    if not store:
+        return {
+            "station_code": station_code,
+            "horizon_hours": horizon,
+            "versions": [],
+            "current": None,
+            "trend": None,
+        }
+
+    def compute():
+        from .ml import skill
+
+        if not store.engine and not store.connect():
+            return {
+                "station_code": station_code,
+                "horizon_hours": horizon,
+                "versions": [],
+                "current": None,
+                "trend": None,
+            }
+        return skill.compute_skill(store.engine, store.db_type, station_code, horizon)
+
+    payload, stale = await _cached_swr(
+        HII_CACHE,
+        HII_CACHE_LOCK,
+        _HII_COMPUTE_LOCKS["skill"],
+        f"skill:{station_code}:{horizon}",
+        max(Config.HII_CACHE_TTL_SECONDS, 900),
+        compute,
+    )
+    if stale:
+        response.headers["X-Data-Stale"] = "true"
+    return payload
+
+
 @app.get("/measurements/latest", response_model=List[MeasurementResponse])
 async def get_latest_measurements(response: Response, limit: int = 100):
     """Get latest measurements from all stations"""
@@ -1313,9 +1363,7 @@ async def get_database_stats():
         from sqlalchemy import text
 
         with engine.connect() as conn:
-            return conn.execute(
+            return conn.execute(text("""
-                text(
-                    """
                     SELECT (SELECT COUNT(*) FROM hii_rainfall) AS rain_n,
                            (SELECT COUNT(*) FROM hii_waterlevel) AS wl_n,
                            (SELECT COUNT(*) FROM hii_rain_stations) AS rain_s,
@@ -1324,9 +1372,7 @@ async def get_database_stats():
                            (SELECT MAX(timestamp) FROM hii_rainfall) AS rain_hi,
                            (SELECT MIN(timestamp) FROM hii_waterlevel) AS wl_lo,
                            (SELECT MAX(timestamp) FROM hii_waterlevel) AS wl_hi
-                    """
+                    """)).one()
-                )
-            ).one()
 
     def compute():
         # Heavy: full-table counts and coverage over ~1.7M rows. Runs at most

tests/test_forecast_skill.py

@@ -0,0 +1,93 @@
+"""Forecast skill verification: issued forecasts vs observed peaks (sqlite)."""
+
+import datetime
+
+import pytest
+from sqlalchemy import create_engine, text
+
+from src.ml import skill
+
+
+@pytest.fixture
+def engine(tmp_path):
+    eng = create_engine(f"sqlite:///{tmp_path / 'skill.db'}")
+    with eng.begin() as c:
+        c.execute(text("CREATE TABLE stations (id INTEGER PRIMARY KEY, station_code TEXT)"))
+        c.execute(text("INSERT INTO stations VALUES (1, 'P.1')"))
+        c.execute(
+            text(
+                "CREATE TABLE water_measurements (timestamp DATETIME, station_id INTEGER, water_level REAL)"
+            )
+        )
+        c.execute(
+            text(
+                "CREATE TABLE forecast_history (as_of TIMESTAMP, station_code TEXT, horizon_hours INTEGER, "
+                "predicted_max_level REAL, p_warning REAL, p_danger REAL, current_level REAL, "
+                "model_version TEXT, source TEXT)"
+            )
+        )
+    return eng
+
+
+def _fill(engine, start, hours, level_fn, forecasts):
+    """hours of hourly observations from `start`, plus (as_of_offset_h, version, pred) rows."""
+    with engine.begin() as c:
+        for h in range(hours):
+            ts = start + datetime.timedelta(hours=h)
+            c.execute(
+                text("INSERT INTO water_measurements VALUES (:t, 1, :l)"),
+                {"t": ts, "l": level_fn(h)},
+            )
+        for off, version, pred in forecasts:
+            ts = start + datetime.timedelta(hours=off)
+            c.execute(
+                text(
+                    "INSERT INTO forecast_history VALUES (:t, 'P.1', 24, :p, 0, 0, :cur, :v, 'model')"
+                ),
+                {"t": ts, "p": pred, "cur": level_fn(off), "v": version},
+            )
+
+
+def test_skill_per_version_and_trend(engine):
+    start = datetime.datetime(2026, 8, 1)
+    # river: flat 1.5 m, with a bump to 2.4 m around hour 100
+    level = lambda h: 2.4 if 96 <= h <= 104 else 1.5
+    forecasts = []
+    # old version: always predicts 1.5 (persistence-like, misses the bump)
+    for off in range(0, 60):
+        forecasts.append((off, "hgb-v2+aaaaaaa", 1.5))
+    # new version: predicts 1.5 normally and 2.3 ahead of the bump
+    for off in range(60, 200):
+        pred = 2.3 if 72 <= off <= 104 else 1.5
+        forecasts.append((off, "hgb-v3+bbbbbbb", pred))
+    _fill(engine, start, 260, level, forecasts)
+
+    out = skill.compute_skill(engine, "sqlite", "P.1", 24, now=start + datetime.timedelta(hours=300))
+    assert [v["model_version"] for v in out["versions"]] == ["hgb-v2+aaaaaaa", "hgb-v3+bbbbbbb"]
+    old, new = out["versions"]
+    assert old["n"] == 60 and old["enough_data"]
+    assert new["n"] == 140 and new["enough_data"]
+    # the old version issued only on flat hours: perfect there, no bump rows
+    assert old["mae_m"] == 0.0 and old["above_2m_n"] == 0
+    # the new version saw the bump: nonzero MAE but positive skill vs persistence
+    assert new["above_2m_n"] > 0
+    assert new["skill"] is not None and new["skill"] > 0
+    assert out["current"]["model_version"] == "hgb-v3+bbbbbbb"
+    assert out["trend"]["previous_version"] == "hgb-v2+aaaaaaa"
+    assert out["trend"]["better"] is False  # honest: old had an easier period
+
+
+def test_skill_requires_full_window(engine):
+    start = datetime.datetime(2026, 8, 1)
+    # forecasts issued at the very end have no observed window yet
+    _fill(engine, start, 30, lambda h: 1.5, [(o, "hgb-v3+ccccccc", 1.5) for o in range(0, 30)])
+    out = skill.compute_skill(engine, "sqlite", "P.1", 24, now=start + datetime.timedelta(hours=30))
+    # only as_of <= now-24h AND with >= 18 observed hours in the window count
+    assert out["versions"] and out["versions"][0]["n"] == 7  # as_of 0..6 h: <= now-24h with >= 18 observed hours
+    assert out["versions"][0]["enough_data"] is False
+    assert out["trend"] is None
+
+
+def test_skill_empty(engine):
+    out = skill.compute_skill(engine, "sqlite", "P.1", 24)
+    assert out["versions"] == [] and out["current"] is None and out["trend"] is None