feat: database stats on the dashboard via GET /api/stats
CI/CD Pipeline - Northern Thailand Ping River Monitor / Test Suite (3.11) (push) Failing after 23s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Build Docker Image (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Integration Test with Services (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Deploy to Staging (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Deploy to Production (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Performance Test (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Code Quality (push) Successful in 14s
Documentation / Validate Documentation (push) Failing after 8s
Documentation / Generate API Documentation (push) Successful in 9s
Documentation / Build Sphinx Documentation (push) Successful in 17s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Cleanup (push) Successful in 1s
Documentation / Documentation Summary (push) Successful in 3s

New SQLAdapter.get_database_stats() aggregates totals, station count,
date range, and hourly-slot coverage in one query per dialect. The
endpoint follows the existing 503-guard/to_thread/TTL-cache pattern;
the dashboard gains a five-tile stats strip on the existing refresh
cadence. Coverage denominator is hour-truncated so off-hour endpoints
cannot push it past 100%; MySQL slot expression avoids % characters
that would break under pyformat bind interpolation.
This commit is contained in:
2026-08-10 23:27:36 +07:00
parent 5e62ea529d
commit 7befc82ff5
3 changed files with 140 additions and 0 deletions
+70
View File
@@ -56,6 +56,14 @@ class DatabaseAdapter(ABC):
""" """
return None return None
def get_database_stats(self) -> Optional[Dict]:
"""Summary statistics over stored measurements: total count, distinct
stations, first/last timestamp, and hourly-slot coverage.
Returns None when the backend has no data or does not support the query.
"""
return None
# InfluxDB Adapter # InfluxDB Adapter
class InfluxDBAdapter(DatabaseAdapter): class InfluxDBAdapter(DatabaseAdapter):
@@ -764,6 +772,68 @@ class SQLAdapter(DatabaseAdapter):
) )
return None return None
def get_database_stats(self) -> Optional[Dict]:
if not self.engine:
return None
try:
from sqlalchemy import text
if self.db_type == "sqlite":
slot_expr = "strftime('%Y-%m-%d %H', timestamp)"
elif self.db_type == "postgresql":
slot_expr = "TO_CHAR(timestamp, 'YYYY-MM-DD HH24')"
else: # MySQL
# %-free expression: a bare % inside text() breaks as soon as the
# query gains a bind parameter (pyformat interpolation)
slot_expr = "CONCAT(DATE(timestamp), ' ', HOUR(timestamp))"
query = f"""
SELECT COUNT(*),
COUNT(DISTINCT station_id),
MIN(timestamp),
MAX(timestamp),
COUNT(DISTINCT {slot_expr})
FROM water_measurements
"""
with self.engine.connect() as conn:
row = conn.execute(text(query)).fetchone()
if not row or not row[0]:
return None
def to_datetime(value):
if isinstance(value, datetime.datetime):
return value
return datetime.datetime.fromisoformat(str(value)[:19])
first_ts = to_datetime(row[2])
last_ts = to_datetime(row[3])
# Truncate to the hour before differencing so the slot count matches
# the DISTINCT day-hour slots and coverage cannot exceed 100%
first_slot = first_ts.replace(minute=0, second=0, microsecond=0)
last_slot = last_ts.replace(minute=0, second=0, microsecond=0)
expected_hours = (
int((last_slot - first_slot).total_seconds() // 3600) + 1
)
recorded_hours = int(row[4])
coverage_percent = round(100.0 * recorded_hours / expected_hours, 1)
return {
"total_measurements": int(row[0]),
"station_count": int(row[1]),
"first_timestamp": first_ts,
"last_timestamp": last_ts,
"recorded_hours": recorded_hours,
"expected_hours": expected_hours,
"coverage_percent": coverage_percent,
}
except Exception as e:
logging.error(f"Error querying {self.db_type.upper()} stats: {e}")
return None
# VictoriaMetrics Adapter (using Prometheus format) # VictoriaMetrics Adapter (using Prometheus format)
class VictoriaMetricsAdapter(DatabaseAdapter): class VictoriaMetricsAdapter(DatabaseAdapter):
+27
View File
@@ -255,6 +255,14 @@
</div> </div>
<div style="height:260px;margin-top:14px;overflow:hidden;position:relative"><canvas id="history-chart" aria-label="Historical water level and discharge chart" style="display:block"></canvas></div> <div style="height:260px;margin-top:14px;overflow:hidden;position:relative"><canvas id="history-chart" aria-label="Historical water level and discharge chart" style="display:block"></canvas></div>
</section> </section>
<section class="stats" id="db-stats" aria-label="Database statistics" style="margin-top:14px;display:none;grid-template-columns:repeat(auto-fit,minmax(170px,1fr))">
<article class="stat"><div class="stat-label">Total datapoints</div><div class="stat-value" id="db-total"></div><div class="stat-note">measurements stored</div></article>
<article class="stat"><div class="stat-label">Date range</div><div class="stat-value" id="db-range" style="font-size:1.05rem;line-height:1.3;white-space:normal"></div><div class="stat-note">first to last record</div></article>
<article class="stat"><div class="stat-label">Days spanned</div><div class="stat-value" id="db-days"></div><div class="stat-note">between first and last</div></article>
<article class="stat"><div class="stat-label">Stations</div><div class="stat-value" id="db-stations"></div><div class="stat-note">distinct in database</div></article>
<article class="stat"><div class="stat-label">Coverage</div><div class="stat-value" id="db-coverage"></div><div class="stat-note">of hourly slots recorded</div></article>
</section>
</main> </main>
<script src="https://unpkg.com/leaflet@1.9.4/dist/leaflet.js" integrity="sha384-cxOPjt7s7Iz04uaHJceBmS+qpjv2JkIHNVcuOrM+YHwZOmJGBXI00mdUXEq65HTH" crossorigin="anonymous"></script> <script src="https://unpkg.com/leaflet@1.9.4/dist/leaflet.js" integrity="sha384-cxOPjt7s7Iz04uaHJceBmS+qpjv2JkIHNVcuOrM+YHwZOmJGBXI00mdUXEq65HTH" crossorigin="anonymous"></script>
@@ -656,6 +664,7 @@
renderSummary(stations, readings); renderSummary(stations, readings);
$('loading').style.display = 'none'; $('loading').style.display = 'none';
loadForecasts(); // non-blocking; the card stays hidden until models are deployed loadForecasts(); // non-blocking; the card stays hidden until models are deployed
loadDbStats(); // non-blocking; the strip stays hidden if /api/stats is unavailable
} catch (error) { } catch (error) {
$('loading').style.display = 'none'; $('loading').style.display = 'none';
$('error').style.display = 'grid'; $('error').style.display = 'grid';
@@ -1017,6 +1026,24 @@
} }
} }
async function loadDbStats() {
const strip = $('db-stats');
try {
const response = await fetch('/api/stats');
if (!response.ok) { strip.style.display = 'none'; return; }
const stats = await response.json();
const fmtDate = (value) => new Date(value).toLocaleDateString([], { day: 'numeric', month: 'short', year: 'numeric' });
$('db-total').textContent = Number(stats.total_measurements).toLocaleString();
$('db-range').textContent = `${fmtDate(stats.first_timestamp)} ${fmtDate(stats.last_timestamp)}`;
$('db-days').textContent = Number(stats.days_spanned).toLocaleString();
$('db-stations').textContent = String(stats.station_count);
$('db-coverage').textContent = stats.coverage_percent == null ? '—' : `${Number(stats.coverage_percent).toFixed(1)}%`;
strip.style.display = 'grid';
} catch (error) {
strip.style.display = 'none';
}
}
$('refresh-button').addEventListener('click', loadDashboard); $('refresh-button').addEventListener('click', loadDashboard);
$('zones-toggle').addEventListener('click', toggleFloodZones); $('zones-toggle').addEventListener('click', toggleFloodZones);
$('replay-2024').addEventListener('click', replayFlood2024); $('replay-2024').addEventListener('click', replayFlood2024);
+43
View File
@@ -51,6 +51,10 @@ FORECAST_CACHE: Dict[str, tuple] = {}
FORECAST_CACHE_LOCK = Lock() FORECAST_CACHE_LOCK = Lock()
FORECAST_TTL = 900 # 15 minutes FORECAST_TTL = 900 # 15 minutes
DB_STATS_CACHE: Dict[str, tuple] = {}
DB_STATS_CACHE_LOCK = Lock()
DB_STATS_TTL = 300 # 5 minutes
# Admin API protection. Read/dashboard endpoints stay public; anything that # Admin API protection. Read/dashboard endpoints stay public; anything that
# mutates state or leaks configuration requires the X-API-Key header matching # mutates state or leaks configuration requires the X-API-Key header matching
# ADMIN_API_KEY. Secure by default: with no key configured, those endpoints # ADMIN_API_KEY. Secure by default: with no key configured, those endpoints
@@ -630,6 +634,45 @@ async def get_station_measurements(
raise HTTPException(status_code=500, detail=str(e)) raise HTTPException(status_code=500, detail=str(e))
@app.get("/api/stats")
async def get_database_stats():
"""Get database coverage statistics (totals, date range, hourly coverage)"""
increment_counter("api_requests", labels={"endpoint": "api_stats"})
scraper = app_state["scraper"]
if not scraper or not scraper.db_adapter:
raise HTTPException(status_code=503, detail="Database not available")
now = time.monotonic()
with DB_STATS_CACHE_LOCK:
cached = DB_STATS_CACHE.get("all")
if cached and now - cached[0] < DB_STATS_TTL:
return cached[1]
try:
stats = await asyncio.to_thread(scraper.db_adapter.get_database_stats)
except Exception as e:
logger.error(f"Error fetching database stats: {e}")
raise HTTPException(status_code=500, detail=str(e))
if stats is None:
raise HTTPException(status_code=503, detail="Database statistics unavailable")
first_ts = stats["first_timestamp"]
last_ts = stats["last_timestamp"]
data = {
"total_measurements": stats["total_measurements"],
"station_count": stats["station_count"],
"first_timestamp": first_ts.isoformat(),
"last_timestamp": last_ts.isoformat(),
"days_spanned": (last_ts.date() - first_ts.date()).days + 1,
"coverage_percent": stats["coverage_percent"],
}
with DB_STATS_CACHE_LOCK:
DB_STATS_CACHE["all"] = (now, data)
return data
@app.post("/scrape/trigger", dependencies=[Depends(require_admin_key)]) @app.post("/scrape/trigger", dependencies=[Depends(require_admin_key)])
async def trigger_scraping(background_tasks: BackgroundTasks): async def trigger_scraping(background_tasks: BackgroundTasks):
"""Trigger manual data scraping""" """Trigger manual data scraping"""