From 7befc82ff5275bdb1da2e47e4c8fc8304b519b0e Mon Sep 17 00:00:00 2001 From: grabowski Date: Mon, 10 Aug 2026 23:27:36 +0700 Subject: [PATCH] feat: database stats on the dashboard via GET /api/stats New SQLAdapter.get_database_stats() aggregates totals, station count, date range, and hourly-slot coverage in one query per dialect. The endpoint follows the existing 503-guard/to_thread/TTL-cache pattern; the dashboard gains a five-tile stats strip on the existing refresh cadence. Coverage denominator is hour-truncated so off-hour endpoints cannot push it past 100%; MySQL slot expression avoids % characters that would break under pyformat bind interpolation. --- src/database_adapters.py | 70 +++++++++++++++++++++++++++++++++++++++ src/static/dashboard.html | 27 +++++++++++++++ src/web_api.py | 43 ++++++++++++++++++++++++ 3 files changed, 140 insertions(+) diff --git a/src/database_adapters.py b/src/database_adapters.py index 2e0bc8b..0b9f837 100644 --- a/src/database_adapters.py +++ b/src/database_adapters.py @@ -56,6 +56,14 @@ class DatabaseAdapter(ABC): """ return None + def get_database_stats(self) -> Optional[Dict]: + """Summary statistics over stored measurements: total count, distinct + stations, first/last timestamp, and hourly-slot coverage. + + Returns None when the backend has no data or does not support the query. + """ + return None + # InfluxDB Adapter class InfluxDBAdapter(DatabaseAdapter): @@ -764,6 +772,68 @@ class SQLAdapter(DatabaseAdapter): ) return None + def get_database_stats(self) -> Optional[Dict]: + if not self.engine: + return None + + try: + from sqlalchemy import text + + if self.db_type == "sqlite": + slot_expr = "strftime('%Y-%m-%d %H', timestamp)" + elif self.db_type == "postgresql": + slot_expr = "TO_CHAR(timestamp, 'YYYY-MM-DD HH24')" + else: # MySQL + # %-free expression: a bare % inside text() breaks as soon as the + # query gains a bind parameter (pyformat interpolation) + slot_expr = "CONCAT(DATE(timestamp), ' ', HOUR(timestamp))" + + query = f""" + SELECT COUNT(*), + COUNT(DISTINCT station_id), + MIN(timestamp), + MAX(timestamp), + COUNT(DISTINCT {slot_expr}) + FROM water_measurements + """ + + with self.engine.connect() as conn: + row = conn.execute(text(query)).fetchone() + + if not row or not row[0]: + return None + + def to_datetime(value): + if isinstance(value, datetime.datetime): + return value + return datetime.datetime.fromisoformat(str(value)[:19]) + + first_ts = to_datetime(row[2]) + last_ts = to_datetime(row[3]) + # Truncate to the hour before differencing so the slot count matches + # the DISTINCT day-hour slots and coverage cannot exceed 100% + first_slot = first_ts.replace(minute=0, second=0, microsecond=0) + last_slot = last_ts.replace(minute=0, second=0, microsecond=0) + expected_hours = ( + int((last_slot - first_slot).total_seconds() // 3600) + 1 + ) + recorded_hours = int(row[4]) + coverage_percent = round(100.0 * recorded_hours / expected_hours, 1) + + return { + "total_measurements": int(row[0]), + "station_count": int(row[1]), + "first_timestamp": first_ts, + "last_timestamp": last_ts, + "recorded_hours": recorded_hours, + "expected_hours": expected_hours, + "coverage_percent": coverage_percent, + } + + except Exception as e: + logging.error(f"Error querying {self.db_type.upper()} stats: {e}") + return None + # VictoriaMetrics Adapter (using Prometheus format) class VictoriaMetricsAdapter(DatabaseAdapter): diff --git a/src/static/dashboard.html b/src/static/dashboard.html index faedc41..abb6789 100644 --- a/src/static/dashboard.html +++ b/src/static/dashboard.html @@ -255,6 +255,14 @@
+ + @@ -656,6 +664,7 @@ renderSummary(stations, readings); $('loading').style.display = 'none'; loadForecasts(); // non-blocking; the card stays hidden until models are deployed + loadDbStats(); // non-blocking; the strip stays hidden if /api/stats is unavailable } catch (error) { $('loading').style.display = 'none'; $('error').style.display = 'grid'; @@ -1017,6 +1026,24 @@ } } + async function loadDbStats() { + const strip = $('db-stats'); + try { + const response = await fetch('/api/stats'); + if (!response.ok) { strip.style.display = 'none'; return; } + const stats = await response.json(); + const fmtDate = (value) => new Date(value).toLocaleDateString([], { day: 'numeric', month: 'short', year: 'numeric' }); + $('db-total').textContent = Number(stats.total_measurements).toLocaleString(); + $('db-range').textContent = `${fmtDate(stats.first_timestamp)} – ${fmtDate(stats.last_timestamp)}`; + $('db-days').textContent = Number(stats.days_spanned).toLocaleString(); + $('db-stations').textContent = String(stats.station_count); + $('db-coverage').textContent = stats.coverage_percent == null ? '—' : `${Number(stats.coverage_percent).toFixed(1)}%`; + strip.style.display = 'grid'; + } catch (error) { + strip.style.display = 'none'; + } + } + $('refresh-button').addEventListener('click', loadDashboard); $('zones-toggle').addEventListener('click', toggleFloodZones); $('replay-2024').addEventListener('click', replayFlood2024); diff --git a/src/web_api.py b/src/web_api.py index 44eb0f3..aeb8e63 100644 --- a/src/web_api.py +++ b/src/web_api.py @@ -51,6 +51,10 @@ FORECAST_CACHE: Dict[str, tuple] = {} FORECAST_CACHE_LOCK = Lock() FORECAST_TTL = 900 # 15 minutes +DB_STATS_CACHE: Dict[str, tuple] = {} +DB_STATS_CACHE_LOCK = Lock() +DB_STATS_TTL = 300 # 5 minutes + # Admin API protection. Read/dashboard endpoints stay public; anything that # mutates state or leaks configuration requires the X-API-Key header matching # ADMIN_API_KEY. Secure by default: with no key configured, those endpoints @@ -630,6 +634,45 @@ async def get_station_measurements( raise HTTPException(status_code=500, detail=str(e)) +@app.get("/api/stats") +async def get_database_stats(): + """Get database coverage statistics (totals, date range, hourly coverage)""" + increment_counter("api_requests", labels={"endpoint": "api_stats"}) + + scraper = app_state["scraper"] + if not scraper or not scraper.db_adapter: + raise HTTPException(status_code=503, detail="Database not available") + + now = time.monotonic() + with DB_STATS_CACHE_LOCK: + cached = DB_STATS_CACHE.get("all") + if cached and now - cached[0] < DB_STATS_TTL: + return cached[1] + + try: + stats = await asyncio.to_thread(scraper.db_adapter.get_database_stats) + except Exception as e: + logger.error(f"Error fetching database stats: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + if stats is None: + raise HTTPException(status_code=503, detail="Database statistics unavailable") + + first_ts = stats["first_timestamp"] + last_ts = stats["last_timestamp"] + data = { + "total_measurements": stats["total_measurements"], + "station_count": stats["station_count"], + "first_timestamp": first_ts.isoformat(), + "last_timestamp": last_ts.isoformat(), + "days_spanned": (last_ts.date() - first_ts.date()).days + 1, + "coverage_percent": stats["coverage_percent"], + } + with DB_STATS_CACHE_LOCK: + DB_STATS_CACHE["all"] = (now, data) + return data + + @app.post("/scrape/trigger", dependencies=[Depends(require_admin_key)]) async def trigger_scraping(background_tasks: BackgroundTasks): """Trigger manual data scraping"""