feat: codified backtests, honest docs, belt-and-braces serving, perf fixes

Retrained on the gap-filled DB (592k -> 976k rows) and re-examined the
flood backtests, now reproducible via scripts/backtest_render.py (renders
the three docs/img charts and gates on a >=12h 2024 first-alert lead —
currently failing by design and documented as such).

Findings, all documented in FLOOD_FORECASTING.md: the true 2024 crossing
was 24 Sep 17:00 (8h earlier than recorded; confirmed against the
independent HII sensor), the historical 24h-warning claim was partly a
missing-data artifact, and retrained warn classifiers collapse on the
filled grid (P.1 24h PR-AUC 0.900 -> 0.288) while regression MAE improves
(11.3 -> 10.5 cm). Serving therefore becomes max(classifier,
sigmoid(regression)) so alerting is never worse than the regression path;
metrics table, head-gating tiers, honest-limits and runbook expectations
all updated to the current model (hgb-v1+d2d0e65).

Perf, from Locust load testing (scripts/locustfile.py + load_test.py):
single-flight lock around /forecast inference (concurrent cache misses
previously each ran ~18s inference and starved the shared thread pool;
200-user run after: 105 rps, 0.01% errors), and /measurements/latest +
/health moved off the event loop (synchronous DB/network calls in async
handlers were stalling every request under load).

src/ml/predict.py

@@ -164,21 +164,24 @@ def _model_forecast(
         predicted_max = max(float(reg.predict(feature_row)[0]), current_level)
         sigma_h = bundle["sigma"].get(horizon_h, HEURISTIC_SIGMA)
 
+        # Belt-and-braces: the classifier head OR the regression-sigmoid path,
+        # whichever is more alarmed. The 2026-08-11 backtest showed a trained
+        # classifier staying silent through the 2024 record flood while the
+        # regression head tracked it — alerting must never be worse than the
+        # regression fallback.
         warn_head = (
             None if thresholds_stale else bundle["heads"].get(f"warn_{horizon_h}")
         )
+        p_warning = _sigmoid_probability(predicted_max, warn_thr, sigma_h)
         if warn_head is not None:
-            p_warning = float(warn_head.predict_proba(feature_row)[0][1])
-        else:
-            p_warning = _sigmoid_probability(predicted_max, warn_thr, sigma_h)
+            p_warning = max(p_warning, float(warn_head.predict_proba(feature_row)[0][1]))
 
         danger_head = (
             None if thresholds_stale else bundle["heads"].get(f"danger_{horizon_h}")
         )
+        p_danger = _sigmoid_probability(predicted_max, danger_thr, sigma_h)
         if danger_head is not None:
-            p_danger = float(danger_head.predict_proba(feature_row)[0][1])
-        else:
-            p_danger = _sigmoid_probability(predicted_max, danger_thr, sigma_h)
+            p_danger = max(p_danger, float(danger_head.predict_proba(feature_row)[0][1]))
 
         p_warning = _clip_probability(p_warning)
         p_danger = min(_clip_probability(p_danger), p_warning)

src/web_api.py

@@ -50,6 +50,7 @@ HISTORY_TTL = 300  # 5 minutes
 
 FORECAST_CACHE: Dict[str, tuple] = {}
 FORECAST_CACHE_LOCK = Lock()
+FORECAST_COMPUTE_LOCK = asyncio.Lock()  # single-flight for expensive inference
 FORECAST_TTL = 900  # 15 minutes
 
 DB_STATS_CACHE: Dict[str, tuple] = {}
@@ -359,8 +360,10 @@ async def get_health():
     if not health_manager:
         raise HTTPException(status_code=503, detail="Health manager not initialized")
 
-    # Run health checks (populates state read by get_health_summary)
-    health_manager.run_all_checks()
+    # Run health checks (populates state read by get_health_summary).
+    # In a thread: DatabaseHealthCheck and APIHealthCheck do blocking I/O and
+    # would otherwise stall the event loop for every other request.
+    await asyncio.to_thread(health_manager.run_all_checks)
     summary = health_manager.get_health_summary()
 
     return HealthResponse(**summary)
@@ -747,6 +750,19 @@ async def get_flood_forecasts():
         from .ml.predict import get_latest_forecasts
     except ImportError as error:
         raise HTTPException(status_code=503, detail=f"Forecasting unavailable: {error}")
+    # Single-flight: inference takes seconds; without this, N concurrent cache
+    # misses ran N full inferences and starved the thread pool (load test:
+    # /forecast timeouts at 10 concurrent clients rippled into every endpoint).
+    async with FORECAST_COMPUTE_LOCK:
+        with FORECAST_CACHE_LOCK:
+            cached = FORECAST_CACHE.get("all")
+            if cached and time.monotonic() - cached[0] < FORECAST_TTL:
+                return cached[1]
+        return await _compute_forecasts(get_latest_forecasts)
+
+
+async def _compute_forecasts(get_latest_forecasts):
+    now = time.monotonic()
     try:
         data = await asyncio.to_thread(get_latest_forecasts)
     except FileNotFoundError:
@@ -771,7 +787,9 @@ async def get_latest_measurements(limit: int = 100):
         raise HTTPException(status_code=503, detail="Database not available")
 
     try:
-        measurements = scraper.get_latest_data(limit=limit)
+        # In a thread: this is a synchronous DB query, and this is the most
+        # frequently hit endpoint — inline it would block the event loop.
+        measurements = await asyncio.to_thread(scraper.get_latest_data, limit)
 
         return [_to_measurement_response(m) for m in measurements]