feat: database stats on the dashboard via GET /api/stats
CI/CD Pipeline - Northern Thailand Ping River Monitor / Test Suite (3.11) (push) Failing after 23s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Build Docker Image (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Integration Test with Services (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Deploy to Staging (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Deploy to Production (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Performance Test (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Code Quality (push) Successful in 14s
Documentation / Validate Documentation (push) Failing after 8s
Documentation / Generate API Documentation (push) Successful in 9s
Documentation / Build Sphinx Documentation (push) Successful in 17s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Cleanup (push) Successful in 1s
Documentation / Documentation Summary (push) Successful in 3s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Test Suite (3.11) (push) Failing after 23s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Build Docker Image (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Integration Test with Services (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Deploy to Staging (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Deploy to Production (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Performance Test (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Code Quality (push) Successful in 14s
Documentation / Validate Documentation (push) Failing after 8s
Documentation / Generate API Documentation (push) Successful in 9s
Documentation / Build Sphinx Documentation (push) Successful in 17s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Cleanup (push) Successful in 1s
Documentation / Documentation Summary (push) Successful in 3s
New SQLAdapter.get_database_stats() aggregates totals, station count, date range, and hourly-slot coverage in one query per dialect. The endpoint follows the existing 503-guard/to_thread/TTL-cache pattern; the dashboard gains a five-tile stats strip on the existing refresh cadence. Coverage denominator is hour-truncated so off-hour endpoints cannot push it past 100%; MySQL slot expression avoids % characters that would break under pyformat bind interpolation.
This commit is contained in:
@@ -56,6 +56,14 @@ class DatabaseAdapter(ABC):
|
|||||||
"""
|
"""
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
def get_database_stats(self) -> Optional[Dict]:
|
||||||
|
"""Summary statistics over stored measurements: total count, distinct
|
||||||
|
stations, first/last timestamp, and hourly-slot coverage.
|
||||||
|
|
||||||
|
Returns None when the backend has no data or does not support the query.
|
||||||
|
"""
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
# InfluxDB Adapter
|
# InfluxDB Adapter
|
||||||
class InfluxDBAdapter(DatabaseAdapter):
|
class InfluxDBAdapter(DatabaseAdapter):
|
||||||
@@ -764,6 +772,68 @@ class SQLAdapter(DatabaseAdapter):
|
|||||||
)
|
)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
def get_database_stats(self) -> Optional[Dict]:
|
||||||
|
if not self.engine:
|
||||||
|
return None
|
||||||
|
|
||||||
|
try:
|
||||||
|
from sqlalchemy import text
|
||||||
|
|
||||||
|
if self.db_type == "sqlite":
|
||||||
|
slot_expr = "strftime('%Y-%m-%d %H', timestamp)"
|
||||||
|
elif self.db_type == "postgresql":
|
||||||
|
slot_expr = "TO_CHAR(timestamp, 'YYYY-MM-DD HH24')"
|
||||||
|
else: # MySQL
|
||||||
|
# %-free expression: a bare % inside text() breaks as soon as the
|
||||||
|
# query gains a bind parameter (pyformat interpolation)
|
||||||
|
slot_expr = "CONCAT(DATE(timestamp), ' ', HOUR(timestamp))"
|
||||||
|
|
||||||
|
query = f"""
|
||||||
|
SELECT COUNT(*),
|
||||||
|
COUNT(DISTINCT station_id),
|
||||||
|
MIN(timestamp),
|
||||||
|
MAX(timestamp),
|
||||||
|
COUNT(DISTINCT {slot_expr})
|
||||||
|
FROM water_measurements
|
||||||
|
"""
|
||||||
|
|
||||||
|
with self.engine.connect() as conn:
|
||||||
|
row = conn.execute(text(query)).fetchone()
|
||||||
|
|
||||||
|
if not row or not row[0]:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def to_datetime(value):
|
||||||
|
if isinstance(value, datetime.datetime):
|
||||||
|
return value
|
||||||
|
return datetime.datetime.fromisoformat(str(value)[:19])
|
||||||
|
|
||||||
|
first_ts = to_datetime(row[2])
|
||||||
|
last_ts = to_datetime(row[3])
|
||||||
|
# Truncate to the hour before differencing so the slot count matches
|
||||||
|
# the DISTINCT day-hour slots and coverage cannot exceed 100%
|
||||||
|
first_slot = first_ts.replace(minute=0, second=0, microsecond=0)
|
||||||
|
last_slot = last_ts.replace(minute=0, second=0, microsecond=0)
|
||||||
|
expected_hours = (
|
||||||
|
int((last_slot - first_slot).total_seconds() // 3600) + 1
|
||||||
|
)
|
||||||
|
recorded_hours = int(row[4])
|
||||||
|
coverage_percent = round(100.0 * recorded_hours / expected_hours, 1)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"total_measurements": int(row[0]),
|
||||||
|
"station_count": int(row[1]),
|
||||||
|
"first_timestamp": first_ts,
|
||||||
|
"last_timestamp": last_ts,
|
||||||
|
"recorded_hours": recorded_hours,
|
||||||
|
"expected_hours": expected_hours,
|
||||||
|
"coverage_percent": coverage_percent,
|
||||||
|
}
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logging.error(f"Error querying {self.db_type.upper()} stats: {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
# VictoriaMetrics Adapter (using Prometheus format)
|
# VictoriaMetrics Adapter (using Prometheus format)
|
||||||
class VictoriaMetricsAdapter(DatabaseAdapter):
|
class VictoriaMetricsAdapter(DatabaseAdapter):
|
||||||
|
|||||||
@@ -255,6 +255,14 @@
|
|||||||
</div>
|
</div>
|
||||||
<div style="height:260px;margin-top:14px;overflow:hidden;position:relative"><canvas id="history-chart" aria-label="Historical water level and discharge chart" style="display:block"></canvas></div>
|
<div style="height:260px;margin-top:14px;overflow:hidden;position:relative"><canvas id="history-chart" aria-label="Historical water level and discharge chart" style="display:block"></canvas></div>
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
|
<section class="stats" id="db-stats" aria-label="Database statistics" style="margin-top:14px;display:none;grid-template-columns:repeat(auto-fit,minmax(170px,1fr))">
|
||||||
|
<article class="stat"><div class="stat-label">Total datapoints</div><div class="stat-value" id="db-total">—</div><div class="stat-note">measurements stored</div></article>
|
||||||
|
<article class="stat"><div class="stat-label">Date range</div><div class="stat-value" id="db-range" style="font-size:1.05rem;line-height:1.3;white-space:normal">—</div><div class="stat-note">first to last record</div></article>
|
||||||
|
<article class="stat"><div class="stat-label">Days spanned</div><div class="stat-value" id="db-days">—</div><div class="stat-note">between first and last</div></article>
|
||||||
|
<article class="stat"><div class="stat-label">Stations</div><div class="stat-value" id="db-stations">—</div><div class="stat-note">distinct in database</div></article>
|
||||||
|
<article class="stat"><div class="stat-label">Coverage</div><div class="stat-value" id="db-coverage">—</div><div class="stat-note">of hourly slots recorded</div></article>
|
||||||
|
</section>
|
||||||
</main>
|
</main>
|
||||||
|
|
||||||
<script src="https://unpkg.com/leaflet@1.9.4/dist/leaflet.js" integrity="sha384-cxOPjt7s7Iz04uaHJceBmS+qpjv2JkIHNVcuOrM+YHwZOmJGBXI00mdUXEq65HTH" crossorigin="anonymous"></script>
|
<script src="https://unpkg.com/leaflet@1.9.4/dist/leaflet.js" integrity="sha384-cxOPjt7s7Iz04uaHJceBmS+qpjv2JkIHNVcuOrM+YHwZOmJGBXI00mdUXEq65HTH" crossorigin="anonymous"></script>
|
||||||
@@ -656,6 +664,7 @@
|
|||||||
renderSummary(stations, readings);
|
renderSummary(stations, readings);
|
||||||
$('loading').style.display = 'none';
|
$('loading').style.display = 'none';
|
||||||
loadForecasts(); // non-blocking; the card stays hidden until models are deployed
|
loadForecasts(); // non-blocking; the card stays hidden until models are deployed
|
||||||
|
loadDbStats(); // non-blocking; the strip stays hidden if /api/stats is unavailable
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
$('loading').style.display = 'none';
|
$('loading').style.display = 'none';
|
||||||
$('error').style.display = 'grid';
|
$('error').style.display = 'grid';
|
||||||
@@ -1017,6 +1026,24 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function loadDbStats() {
|
||||||
|
const strip = $('db-stats');
|
||||||
|
try {
|
||||||
|
const response = await fetch('/api/stats');
|
||||||
|
if (!response.ok) { strip.style.display = 'none'; return; }
|
||||||
|
const stats = await response.json();
|
||||||
|
const fmtDate = (value) => new Date(value).toLocaleDateString([], { day: 'numeric', month: 'short', year: 'numeric' });
|
||||||
|
$('db-total').textContent = Number(stats.total_measurements).toLocaleString();
|
||||||
|
$('db-range').textContent = `${fmtDate(stats.first_timestamp)} – ${fmtDate(stats.last_timestamp)}`;
|
||||||
|
$('db-days').textContent = Number(stats.days_spanned).toLocaleString();
|
||||||
|
$('db-stations').textContent = String(stats.station_count);
|
||||||
|
$('db-coverage').textContent = stats.coverage_percent == null ? '—' : `${Number(stats.coverage_percent).toFixed(1)}%`;
|
||||||
|
strip.style.display = 'grid';
|
||||||
|
} catch (error) {
|
||||||
|
strip.style.display = 'none';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
$('refresh-button').addEventListener('click', loadDashboard);
|
$('refresh-button').addEventListener('click', loadDashboard);
|
||||||
$('zones-toggle').addEventListener('click', toggleFloodZones);
|
$('zones-toggle').addEventListener('click', toggleFloodZones);
|
||||||
$('replay-2024').addEventListener('click', replayFlood2024);
|
$('replay-2024').addEventListener('click', replayFlood2024);
|
||||||
|
|||||||
@@ -51,6 +51,10 @@ FORECAST_CACHE: Dict[str, tuple] = {}
|
|||||||
FORECAST_CACHE_LOCK = Lock()
|
FORECAST_CACHE_LOCK = Lock()
|
||||||
FORECAST_TTL = 900 # 15 minutes
|
FORECAST_TTL = 900 # 15 minutes
|
||||||
|
|
||||||
|
DB_STATS_CACHE: Dict[str, tuple] = {}
|
||||||
|
DB_STATS_CACHE_LOCK = Lock()
|
||||||
|
DB_STATS_TTL = 300 # 5 minutes
|
||||||
|
|
||||||
# Admin API protection. Read/dashboard endpoints stay public; anything that
|
# Admin API protection. Read/dashboard endpoints stay public; anything that
|
||||||
# mutates state or leaks configuration requires the X-API-Key header matching
|
# mutates state or leaks configuration requires the X-API-Key header matching
|
||||||
# ADMIN_API_KEY. Secure by default: with no key configured, those endpoints
|
# ADMIN_API_KEY. Secure by default: with no key configured, those endpoints
|
||||||
@@ -630,6 +634,45 @@ async def get_station_measurements(
|
|||||||
raise HTTPException(status_code=500, detail=str(e))
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/stats")
|
||||||
|
async def get_database_stats():
|
||||||
|
"""Get database coverage statistics (totals, date range, hourly coverage)"""
|
||||||
|
increment_counter("api_requests", labels={"endpoint": "api_stats"})
|
||||||
|
|
||||||
|
scraper = app_state["scraper"]
|
||||||
|
if not scraper or not scraper.db_adapter:
|
||||||
|
raise HTTPException(status_code=503, detail="Database not available")
|
||||||
|
|
||||||
|
now = time.monotonic()
|
||||||
|
with DB_STATS_CACHE_LOCK:
|
||||||
|
cached = DB_STATS_CACHE.get("all")
|
||||||
|
if cached and now - cached[0] < DB_STATS_TTL:
|
||||||
|
return cached[1]
|
||||||
|
|
||||||
|
try:
|
||||||
|
stats = await asyncio.to_thread(scraper.db_adapter.get_database_stats)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error fetching database stats: {e}")
|
||||||
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
|
if stats is None:
|
||||||
|
raise HTTPException(status_code=503, detail="Database statistics unavailable")
|
||||||
|
|
||||||
|
first_ts = stats["first_timestamp"]
|
||||||
|
last_ts = stats["last_timestamp"]
|
||||||
|
data = {
|
||||||
|
"total_measurements": stats["total_measurements"],
|
||||||
|
"station_count": stats["station_count"],
|
||||||
|
"first_timestamp": first_ts.isoformat(),
|
||||||
|
"last_timestamp": last_ts.isoformat(),
|
||||||
|
"days_spanned": (last_ts.date() - first_ts.date()).days + 1,
|
||||||
|
"coverage_percent": stats["coverage_percent"],
|
||||||
|
}
|
||||||
|
with DB_STATS_CACHE_LOCK:
|
||||||
|
DB_STATS_CACHE["all"] = (now, data)
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
||||||
@app.post("/scrape/trigger", dependencies=[Depends(require_admin_key)])
|
@app.post("/scrape/trigger", dependencies=[Depends(require_admin_key)])
|
||||||
async def trigger_scraping(background_tasks: BackgroundTasks):
|
async def trigger_scraping(background_tasks: BackgroundTasks):
|
||||||
"""Trigger manual data scraping"""
|
"""Trigger manual data scraping"""
|
||||||
|
|||||||
Reference in New Issue
Block a user