perf: cache /measurements/latest; stale-on-error fallback for cached endpoints

The response caches (HII feeds, and now /measurements/latest at 45s TTL
— the endpoint every dashboard poll hits) share one helper,
_ttl_cached_stale: single-flight per key, empty results never cached,
and expired entries kept as a fallback. If a recompute fails (DB
unreachable), the last good response is served with an X-Data-Stale:
true header instead of a 5xx — during an outage the dashboard keeps
showing the last real readings with their honest timestamps. TTLs:
LATEST_CACHE_TTL_SECONDS (45), HII_CACHE_TTL_SECONDS (120).

src/web_api.py

@@ -14,7 +14,15 @@ from threading import Lock
 from typing import Any, Dict, List, Optional
 
 import requests
-from fastapi import BackgroundTasks, Depends, FastAPI, Header, HTTPException, Query
+from fastapi import (
+    BackgroundTasks,
+    Depends,
+    FastAPI,
+    Header,
+    HTTPException,
+    Query,
+    Response,
+)
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import FileResponse, HTMLResponse
 from fastapi.staticfiles import StaticFiles
@@ -641,39 +649,68 @@ def _hii_rows(sql: str, params: Dict[str, Any]) -> List[Dict[str, Any]]:
     return rows
 
 
-# Short-TTL cache for the HII feeds: the collector writes hourly, but these
-# endpoints carried ~30% of the load-test mix at 0.6-0.9 s per uncached query.
-# Process-local by design (single-worker deployment); the helper is the seam
-# where a shared backend (e.g. Redis) would slot in if we ever run multiple
-# workers. Thread-based single-flight keeps it event-loop-agnostic.
+# Short-TTL response caches with stale-on-error. Process-local by design
+# (single-worker deployment); _ttl_cached_stale is the seam where a shared
+# backend (e.g. Redis) would slot in if we ever run multiple workers.
+# Thread-based single-flight keeps it event-loop-agnostic. Expired entries are
+# kept as a fallback: if the recompute fails (typically the DB briefly
+# unreachable), the last good response is served instead of a 5xx — during a
+# flood, slightly stale readings with a visible timestamp beat an error page.
 HII_CACHE: Dict[str, Any] = {}
 HII_CACHE_LOCK = Lock()
 _HII_COMPUTE_LOCKS = {"rain": Lock(), "waterlevel": Lock()}
+LATEST_CACHE: Dict[str, Any] = {}
+LATEST_CACHE_LOCK = Lock()
+_LATEST_COMPUTE_LOCK = Lock()
 
 
-def _hii_cached(feed: str, hours: int, sql: str) -> List[Dict[str, Any]]:
-    key = f"{feed}:{hours}"
-    ttl = Config.HII_CACHE_TTL_SECONDS
-    now = time.monotonic()
-    with HII_CACHE_LOCK:
-        cached = HII_CACHE.get(key)
-        if cached and now - cached[0] < ttl:
-            return cached[1]
-    with _HII_COMPUTE_LOCKS[feed]:  # single-flight per feed
-        with HII_CACHE_LOCK:
-            cached = HII_CACHE.get(key)
-            if cached and time.monotonic() - cached[0] < ttl:
-                return cached[1]
+def _ttl_cached_stale(cache, cache_lock, compute_lock, key, ttl, compute):
+    """Return (value, is_stale). Single-flight; keeps expired entries as an
+    error fallback; never caches empty results so recovery is immediate."""
+    with cache_lock:
+        entry = cache.get(key)
+        if entry and time.monotonic() - entry[0] < ttl:
+            return entry[1], False
+    with compute_lock:
+        with cache_lock:
+            entry = cache.get(key)
+            if entry and time.monotonic() - entry[0] < ttl:
+                return entry[1], False
+        try:
+            value = compute()
+        except Exception as error:
+            if entry is not None:
+                age = int(time.monotonic() - entry[0])
+                logger.warning(
+                    f"{key}: recompute failed ({error}); serving {age}s-stale copy"
+                )
+                return entry[1], True
+            raise
+        if value:
+            with cache_lock:
+                cache[key] = (time.monotonic(), value)
+        return value, False
+
+
+def _hii_cached(feed: str, hours: int, sql: str):
+    def compute():
         cutoff = datetime.now() - timedelta(hours=hours)
-        rows = _hii_rows(sql, {"cutoff": cutoff})
-        if rows:  # never cache empty: recover immediately once data appears
-            with HII_CACHE_LOCK:
-                HII_CACHE[key] = (time.monotonic(), rows)
-        return rows
+        return _hii_rows(sql, {"cutoff": cutoff})
+
+    return _ttl_cached_stale(
+        HII_CACHE,
+        HII_CACHE_LOCK,
+        _HII_COMPUTE_LOCKS[feed],
+        f"{feed}:{hours}",
+        Config.HII_CACHE_TTL_SECONDS,
+        compute,
+    )
 
 
 @app.get("/api/hii/rainfall/latest")
-async def get_hii_rainfall_latest(hours: int = Query(26, ge=1, le=168)):
+async def get_hii_rainfall_latest(
+    response: Response, hours: int = Query(26, ge=1, le=168)
+):
     """Latest rainfall reading per HII station (Ping basin, collected hourly)."""
     increment_counter("api_requests", labels={"endpoint": "hii_rainfall"})
     sql = """
@@ -687,11 +724,16 @@ async def get_hii_rainfall_latest(hours: int = Query(26, ge=1, le=168)):
               GROUP BY station_id) latest
           ON latest.station_id = m.station_id AND latest.latest_ts = m.timestamp
     """
-    return await asyncio.to_thread(_hii_cached, "rain", hours, sql)
+    rows, stale = await asyncio.to_thread(_hii_cached, "rain", hours, sql)
+    if stale:
+        response.headers["X-Data-Stale"] = "true"
+    return rows
 
 
 @app.get("/api/hii/waterlevel/latest")
-async def get_hii_waterlevel_latest(hours: int = Query(26, ge=1, le=168)):
+async def get_hii_waterlevel_latest(
+    response: Response, hours: int = Query(26, ge=1, le=168)
+):
     """Latest water-level reading per HII station (Ping basin, m MSL)."""
     increment_counter("api_requests", labels={"endpoint": "hii_waterlevel"})
     sql = """
@@ -707,7 +749,10 @@ async def get_hii_waterlevel_latest(hours: int = Query(26, ge=1, le=168)):
               GROUP BY station_id) latest
           ON latest.station_id = m.station_id AND latest.latest_ts = m.timestamp
     """
-    return await asyncio.to_thread(_hii_cached, "waterlevel", hours, sql)
+    rows, stale = await asyncio.to_thread(_hii_cached, "waterlevel", hours, sql)
+    if stale:
+        response.headers["X-Data-Stale"] = "true"
+    return rows
 
 
 @app.get("/measurements/history/{station_code}")
@@ -807,7 +852,7 @@ async def _compute_forecasts(get_latest_forecasts):
 
 
 @app.get("/measurements/latest", response_model=List[MeasurementResponse])
-async def get_latest_measurements(limit: int = 100):
+async def get_latest_measurements(response: Response, limit: int = 100):
     """Get latest measurements from all stations"""
     increment_counter("api_requests", labels={"endpoint": "measurements_latest"})
 
@@ -815,11 +860,23 @@ async def get_latest_measurements(limit: int = 100):
     if not scraper or not scraper.db_adapter:
         raise HTTPException(status_code=503, detail="Database not available")
 
-    try:
-        # In a thread: this is a synchronous DB query, and this is the most
-        # frequently hit endpoint — inline it would block the event loop.
-        measurements = await asyncio.to_thread(scraper.get_latest_data, limit)
+    def compute():
+        return scraper.get_latest_data(limit=limit)
 
+    try:
+        # Cached + single-flight in a thread: this is the most frequently hit
+        # endpoint (every dashboard poll); the source data changes hourly.
+        measurements, stale = await asyncio.to_thread(
+            _ttl_cached_stale,
+            LATEST_CACHE,
+            LATEST_CACHE_LOCK,
+            _LATEST_COMPUTE_LOCK,
+            f"latest:{limit}",
+            Config.LATEST_CACHE_TTL_SECONDS,
+            compute,
+        )
+        if stale:
+            response.headers["X-Data-Stale"] = "true"
         return [_to_measurement_response(m) for m in measurements]
 
     except Exception as e: