feat: "Is the model getting better?" - live verification per model version

src/ml/skill.py joins forecast_history (what each deployed version
predicted for the 24 h peak, hourly) to water_measurements (what the
river did) and reports per version: verified hours, peak MAE, bias, the
persistence baseline (peak = current level), skill = 1 - MAE/persistence,
and the same MAE restricted to observed peaks >= 2 m. Only forecasts
whose window has elapsed with >= 75 % of hours observed count; a
version needs 24 verified hours before it is compared.

GET /api/forecast/skill?station_code=P.1&horizon=24 returns it (SWR
cached, 15 min). The dashboard's forecast card gains a panel with a
one-line verdict (current vs previous version), the per-version table,
and a caveat that quiet weeks measure quiet-river accuracy only: the
model is judged on flood-onset lead, which the backtests cover. EN + TH.

On today's production data: hgb-v3+28b62e5 (369 h, Aug 13 - Sep 1)
MAE 15.2 cm, skill -0.05; hgb-v2+f6570ac (224 h, Sep 1 - 11) MAE
12.3 cm, skill 0.36 - the "worse" v2 model scores better on a quieter
fortnight, which is exactly why the panel shows the >= 2 m column and
the caveat. Tests: 3, sqlite, synthetic.

scripts/dev_proxy.py: DEV_PROXY_LOCAL lets a not-yet-deployed endpoint be
answered from a local JSON file while everything else goes to prod.

src/ml/skill.py

@@ -0,0 +1,170 @@
+"""Live forecast skill: what the deployed model said versus what the river did.
+
+Every hour the precompute stores the issued 24 h forecast (forecast_history);
+water_measurements holds what actually happened. Joining the two gives a
+verification that needs no retraining and answers the question the dashboard
+is asked most: "is the model getting better?" — per model version, on the
+hours that version was actually serving.
+
+Metrics per version and horizon:
+  n           verified forecasts (issued, and the horizon has since elapsed)
+  mae         |predicted_max - observed_max| over the horizon window, metres
+  bias        mean(predicted - observed): >0 over-predicts the peak
+  persistence MAE of the trivial "peak = current level" forecast on the
+              same rows; a model is only useful if it beats this
+  skill       1 - mae/persistence  (0 = no better than persistence, 1 = perfect)
+  above_2m    same MAE restricted to rows where the observed peak >= 2 m,
+              i.e. the flood-relevant regime
+
+Only the P.1 gauge is verified by default: it is the one the city threshold
+is keyed to, and one station keeps the query cheap enough to run on request.
+"""
+
+import datetime
+import logging
+from typing import Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_STATION = "P.1"
+DEFAULT_HORIZON = 24
+MIN_VERIFIED = 24  # fewer than a day of verified hours is not a number
+
+
+def _sql_for(db_type: str) -> str:
+    """Join each issued forecast to the observed max over (as_of, as_of + h]."""
+    if db_type == "postgresql":
+        window_end = "f.as_of + (f.horizon_hours || ' hours')::interval"
+    elif db_type == "mysql":
+        window_end = "DATE_ADD(f.as_of, INTERVAL f.horizon_hours HOUR)"
+    else:  # sqlite
+        window_end = "datetime(f.as_of, '+' || f.horizon_hours || ' hours')"
+    return f"""
+        SELECT f.as_of, f.model_version, f.predicted_max_level, f.current_level,
+               (SELECT MAX(m.water_level) FROM water_measurements m
+                 JOIN stations s ON s.id = m.station_id
+                WHERE s.station_code = f.station_code
+                  AND m.timestamp > f.as_of AND m.timestamp <= {window_end}) AS observed_max,
+               (SELECT COUNT(m.water_level) FROM water_measurements m
+                 JOIN stations s ON s.id = m.station_id
+                WHERE s.station_code = f.station_code
+                  AND m.timestamp > f.as_of AND m.timestamp <= {window_end}) AS observed_n
+        FROM forecast_history f
+        WHERE f.station_code = :code AND f.horizon_hours = :horizon
+          AND f.source = 'model' AND f.predicted_max_level IS NOT NULL
+          AND f.as_of <= :verifiable_before
+        ORDER BY f.as_of
+    """
+
+
+def compute_skill(
+    engine,
+    db_type: str,
+    station_code: str = DEFAULT_STATION,
+    horizon_hours: int = DEFAULT_HORIZON,
+    now: Optional[datetime.datetime] = None,
+) -> Dict:
+    """Per-model-version verification of issued forecasts against observations."""
+    from sqlalchemy import text
+
+    now = now or datetime.datetime.now()
+    verifiable_before = now - datetime.timedelta(hours=horizon_hours)
+    with engine.connect() as conn:
+        rows = [
+            dict(r._mapping)
+            for r in conn.execute(
+                text(_sql_for(db_type)),
+                {
+                    "code": station_code,
+                    "horizon": horizon_hours,
+                    "verifiable_before": verifiable_before,
+                },
+            )
+        ]
+
+    def _ts(value):
+        # sqlite hands back strings; postgres/mysql give datetimes
+        if isinstance(value, datetime.datetime):
+            return value
+        return datetime.datetime.fromisoformat(str(value).replace(" ", "T"))
+
+    by_version: Dict[str, List[dict]] = {}
+    for r in rows:
+        r["as_of"] = _ts(r["as_of"])
+        # need most of the window observed, or the "max" is not the peak
+        if r["observed_max"] is None or (r["observed_n"] or 0) < horizon_hours * 0.75:
+            continue
+        by_version.setdefault(r["model_version"] or "unknown", []).append(r)
+
+    versions = []
+    for version, vrows in by_version.items():
+        pred = [float(r["predicted_max_level"]) for r in vrows]
+        obs = [float(r["observed_max"]) for r in vrows]
+        cur = [
+            float(r["current_level"]) if r["current_level"] is not None else None
+            for r in vrows
+        ]
+        err = [p - o for p, o in zip(pred, obs)]
+        mae = sum(abs(e) for e in err) / len(err)
+        bias = sum(err) / len(err)
+        pers_rows = [(c, o) for c, o in zip(cur, obs) if c is not None]
+        persistence = (
+            sum(abs(c - o) for c, o in pers_rows) / len(pers_rows)
+            if pers_rows
+            else None
+        )
+        high = [(p, o) for p, o in zip(pred, obs) if o >= 2.0]
+        versions.append(
+            {
+                "model_version": version,
+                "first_issued": min(r["as_of"] for r in vrows).isoformat(),
+                "last_issued": max(r["as_of"] for r in vrows).isoformat(),
+                "n": len(vrows),
+                "mae_m": round(mae, 3),
+                "bias_m": round(bias, 3),
+                "persistence_mae_m": (
+                    None if persistence is None else round(persistence, 3)
+                ),
+                "skill": (
+                    None if not persistence else round(1.0 - mae / persistence, 3)
+                ),
+                "above_2m_n": len(high),
+                "above_2m_mae_m": (
+                    round(sum(abs(p - o) for p, o in high) / len(high), 3)
+                    if high
+                    else None
+                ),
+                "enough_data": len(vrows) >= MIN_VERIFIED,
+            }
+        )
+    versions.sort(key=lambda v: v["first_issued"])
+
+    # Headline: current version vs the previous one that had enough data
+    current = versions[-1] if versions else None
+    previous = (
+        next((v for v in reversed(versions[:-1]) if v["enough_data"]), None)
+        if versions
+        else None
+    )
+    trend = None
+    if current and previous and current["enough_data"]:
+        trend = {
+            "previous_version": previous["model_version"],
+            "mae_delta_m": round(current["mae_m"] - previous["mae_m"], 3),
+            "skill_delta": (
+                None
+                if current["skill"] is None or previous["skill"] is None
+                else round(current["skill"] - previous["skill"], 3)
+            ),
+            "better": current["mae_m"] < previous["mae_m"],
+        }
+
+    return {
+        "station_code": station_code,
+        "horizon_hours": horizon_hours,
+        "verified_until": verifiable_before.isoformat(),
+        "min_verified": MIN_VERIFIED,
+        "versions": versions,
+        "current": current,
+        "trend": trend,
+    }