feat: RID large-dam daily collector — Mae Ngat storage/inflow/outflow
POST app.rid.go.th/reservoir/api/dams (open, archive >=2009) collected hourly into rid_dams + rid_reservoir_daily; backfill script fetches only missing days so reruns repair holes and are safe alongside the live collector. /api/stats counts the new table via an engine fallback that works when HII collection is disabled. Mae Ngat (DAM_ID 200103) hit 113% usable capacity with ~19 MCM/day inflow in the Oct 2024 flood — candidate features for the next retrain.
This commit is contained in:
+63
-15
@@ -186,6 +186,17 @@ async def lifespan(app: FastAPI):
|
||||
app_state["hii_collector"] = None
|
||||
logger.error(f"HII collector initialization failed: {e}")
|
||||
|
||||
# Initialize RID reservoir collector (Mae Ngat + all large dams, daily)
|
||||
try:
|
||||
from .rid_reservoir import create_collector_from_config as create_rsv
|
||||
|
||||
app_state["reservoir_collector"] = create_rsv()
|
||||
if app_state["reservoir_collector"]:
|
||||
logger.info("RID reservoir collection enabled (large-dam daily status)")
|
||||
except Exception as e:
|
||||
app_state["reservoir_collector"] = None
|
||||
logger.error(f"Reservoir collector initialization failed: {e}")
|
||||
|
||||
# Initialize health checks
|
||||
health_manager = HealthCheckManager()
|
||||
health_manager.add_check(DatabaseHealthCheck(app_state["scraper"].db_adapter))
|
||||
@@ -372,6 +383,17 @@ async def background_scraping_task():
|
||||
except Exception as e:
|
||||
logger.error(f"HII collection failed: {e}")
|
||||
|
||||
# RID large-dam daily status (Mae Ngat storage/inflow/outflow)
|
||||
reservoir_collector = app_state.get("reservoir_collector")
|
||||
if reservoir_collector:
|
||||
try:
|
||||
rsv_saved = await asyncio.get_event_loop().run_in_executor(
|
||||
None, reservoir_collector.run_cycle
|
||||
)
|
||||
set_gauge("reservoir_rows_saved", rsv_saved)
|
||||
except Exception as e:
|
||||
logger.error(f"Reservoir collection failed: {e}")
|
||||
|
||||
# Persist the Open-Meteo catchment rain (observed tail +
|
||||
# 48h forecast) so the DB carries the weather context too
|
||||
await _persist_rain()
|
||||
@@ -798,6 +820,25 @@ def _hii_engine():
|
||||
return store.engine
|
||||
|
||||
|
||||
def _aux_stats_engine():
|
||||
"""Any available engine for counting auxiliary tables in /api/stats.
|
||||
|
||||
The HII and reservoir collectors are gated by independent config flags;
|
||||
either store's engine can run the guarded COUNT queries, so falling back
|
||||
keeps the stats honest when one collector is disabled.
|
||||
"""
|
||||
engine = _hii_engine()
|
||||
if engine is not None:
|
||||
return engine
|
||||
collector = app_state.get("reservoir_collector")
|
||||
if not collector:
|
||||
return None
|
||||
store = collector.store
|
||||
if not store.engine and not store.connect():
|
||||
return None
|
||||
return store.engine
|
||||
|
||||
|
||||
def _hii_rows(sql: str, params: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
"""Run a read query against the hii_* tables, JSON-normalizing numerics."""
|
||||
engine = _hii_engine()
|
||||
@@ -1222,21 +1263,26 @@ async def get_database_stats():
|
||||
except Exception as e:
|
||||
logger.warning(f"HII stats unavailable: {e}")
|
||||
|
||||
openmeteo_n = 0
|
||||
try: # table appears with the 2026-08 rain work; older DBs lack it
|
||||
engine = _hii_engine()
|
||||
if engine is not None:
|
||||
from sqlalchemy import text as _text
|
||||
# Tables that appear with 2026-08 feature work; older DBs lack them,
|
||||
# so each count is independently best-effort.
|
||||
extra_counts = {"openmeteo_rain": 0, "rid_reservoir_daily": 0}
|
||||
engine = _aux_stats_engine()
|
||||
if engine is not None:
|
||||
from sqlalchemy import text as _text
|
||||
|
||||
with engine.connect() as conn:
|
||||
openmeteo_n = int(
|
||||
conn.execute(
|
||||
_text("SELECT COUNT(*) FROM openmeteo_rain")
|
||||
).scalar()
|
||||
or 0
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
for table in extra_counts:
|
||||
try:
|
||||
with engine.connect() as conn:
|
||||
extra_counts[table] = int(
|
||||
conn.execute(
|
||||
_text(f"SELECT COUNT(*) FROM {table}")
|
||||
).scalar()
|
||||
or 0
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
openmeteo_n = extra_counts["openmeteo_rain"]
|
||||
reservoir_n = extra_counts["rid_reservoir_daily"]
|
||||
|
||||
def as_dt(value):
|
||||
if isinstance(value, str):
|
||||
@@ -1261,11 +1307,13 @@ async def get_database_stats():
|
||||
"total_measurements": stats["total_measurements"]
|
||||
+ rain_n
|
||||
+ wl_n
|
||||
+ openmeteo_n,
|
||||
+ openmeteo_n
|
||||
+ reservoir_n,
|
||||
"rid_measurements": stats["total_measurements"],
|
||||
"hii_rainfall_measurements": rain_n,
|
||||
"hii_waterlevel_measurements": wl_n,
|
||||
"openmeteo_rain_measurements": openmeteo_n,
|
||||
"reservoir_measurements": reservoir_n,
|
||||
"station_count": stats["station_count"] + hii_stations,
|
||||
"rid_station_count": stats["station_count"],
|
||||
"hii_station_count": hii_stations,
|
||||
|
||||
Reference in New Issue
Block a user