feat: forecast precompute + issued-forecast archive + dashboard overlay

The collection-leader worker now precomputes forecasts after every
scrape cycle: primes the /forecast cache (users never trigger the
multi-second inference — its TTL rises to 4500s so the hourly refresh
always wins) and persists every issued forecast to a new
forecast_history table keyed by (as_of, station, horizon) with
predicted max level, warn/danger probabilities, current level, and
model_version. This is the operational record the backtests lacked —
predicted-vs-actual becomes a simple join instead of retraining
historical models.

GET /api/forecast/history/{station} serves the archive (hours or
start/end + horizon filters, 5-min edge cache), and the station history
chart overlays 'Model 24 h peak (as issued)' as a dashed violet line
once data accumulates.

src/web_api.py

@@ -60,7 +60,9 @@ HISTORY_TTL = 300  # 5 minutes
 FORECAST_CACHE: Dict[str, tuple] = {}
 FORECAST_CACHE_LOCK = Lock()
 FORECAST_COMPUTE_LOCK = asyncio.Lock()  # single-flight for expensive inference
-FORECAST_TTL = 900  # 15 minutes
+# The leader worker precomputes after every scrape cycle (hourly); the TTL just
+# needs to outlive one cycle so user requests never trigger inference themselves.
+FORECAST_TTL = int(os.getenv("FORECAST_TTL_SECONDS", "4500"))
 
 DB_STATS_CACHE: Dict[str, tuple] = {}
 DB_STATS_CACHE_LOCK = Lock()
@@ -159,6 +161,20 @@ async def lifespan(app: FastAPI):
     db_config = Config.get_database_config()
     app_state["scraper"] = EnhancedWaterMonitorScraper(db_config)
 
+    # Forecast history store (SQL only): records what the model predicted
+    try:
+        if db_config["type"] in ("sqlite", "postgresql", "mysql"):
+            from .forecast_history import ForecastHistoryStore
+
+            app_state["forecast_store"] = ForecastHistoryStore(
+                db_config["connection_string"], db_config["type"]
+            )
+        else:
+            app_state["forecast_store"] = None
+    except Exception as e:
+        app_state["forecast_store"] = None
+        logger.error(f"Forecast history store init failed: {e}")
+
     # Initialize HII/ThaiWater collector (rainfall + backup water level)
     try:
         from .hii_collector import create_collector_from_config
@@ -248,6 +264,32 @@ app.add_middleware(
 )
 
 
+async def _precompute_forecasts():
+    """Refresh the forecast cache and persist the issued forecasts (leader only)."""
+    try:
+        from .ml.predict import get_latest_forecasts
+    except ImportError:
+        return
+    try:
+        rows = await asyncio.to_thread(get_latest_forecasts)
+    except Exception as e:
+        logger.warning(f"Forecast precompute failed: {e}")
+        return
+    if not rows:
+        return
+    with FORECAST_CACHE_LOCK:
+        FORECAST_CACHE["all"] = (time.monotonic(), rows)
+    store = app_state.get("forecast_store")
+    if store:
+        try:
+            saved = await asyncio.to_thread(store.save_rows, rows)
+            logger.info(
+                f"Forecast precompute: cached {len(rows)} rows, persisted {saved}"
+            )
+        except Exception as e:
+            logger.warning(f"Forecast history save failed: {e}")
+
+
 async def background_scraping_task():
     """Background task for periodic data scraping"""
     while True:
@@ -309,6 +351,12 @@ async def background_scraping_task():
                         except Exception as e:
                             logger.error(f"HII collection failed: {e}")
 
+                    # Precompute forecasts on fresh data: primes the response
+                    # cache (user requests never pay for inference) and records
+                    # what the model predicted for later predicted-vs-actual
+                    # evaluation.
+                    await _precompute_forecasts()
+
                 app_state["is_scraping"] = False
 
             # Calculate next run time
@@ -366,6 +414,7 @@ def _send_umami_event(path: str, method: str, status: int, host: str, user_agent
 _CACHE_CONTROL_RULES = (
     ("/static/", "public, max-age=3600"),
     ("/measurements/latest", "public, max-age=30"),
+    ("/api/forecast/", "public, max-age=300"),
     ("/api/hii/", "public, max-age=60"),
     ("/measurements/history", "public, max-age=300"),
     ("/forecast", "public, max-age=120"),
@@ -1007,6 +1056,35 @@ async def _compute_forecasts(get_latest_forecasts):
     return data
 
 
+@app.get("/api/forecast/history/{station_code}")
+async def get_forecast_history(
+    station_code: str,
+    hours: int = Query(168, ge=1),
+    start: Optional[date] = Query(None, description="First day (overrides hours)"),
+    end: Optional[date] = Query(None, description="Last day, inclusive"),
+    horizon: Optional[int] = Query(None, ge=1, le=48),
+):
+    """Issued model forecasts for a station — what the model predicted, when.
+
+    Populated by the hourly precompute; enables predicted-vs-actual charts.
+    """
+    increment_counter("api_requests", labels={"endpoint": "forecast_history"})
+    store = app_state.get("forecast_store")
+    if not store:
+        return []
+    end_dt = (
+        datetime.combine(end, datetime.max.time()) if end else datetime.now()
+    )
+    start_dt = (
+        datetime.combine(start, datetime.min.time())
+        if start
+        else end_dt - timedelta(hours=hours)
+    )
+    return await asyncio.to_thread(
+        store.fetch, station_code, start_dt, end_dt, horizon
+    )
+
+
 @app.get("/measurements/latest", response_model=List[MeasurementResponse])
 async def get_latest_measurements(response: Response, limit: int = 100):
     """Get latest measurements from all stations"""