feat: RID large-dam daily collector — Mae Ngat storage/inflow/outflow

POST app.rid.go.th/reservoir/api/dams (open, archive >=2009) collected
hourly into rid_dams + rid_reservoir_daily; backfill script fetches only
missing days so reruns repair holes and are safe alongside the live
collector. /api/stats counts the new table via an engine fallback that
works when HII collection is disabled. Mae Ngat (DAM_ID 200103) hit 113%
usable capacity with ~19 MCM/day inflow in the Oct 2024 flood — candidate
features for the next retrain.
This commit is contained in:
2026-08-13 10:29:34 +07:00
parent 811af1625b
commit 6eafb353b1
7 changed files with 712 additions and 19 deletions
+63 -15
View File
@@ -186,6 +186,17 @@ async def lifespan(app: FastAPI):
app_state["hii_collector"] = None
logger.error(f"HII collector initialization failed: {e}")
# Initialize RID reservoir collector (Mae Ngat + all large dams, daily)
try:
from .rid_reservoir import create_collector_from_config as create_rsv
app_state["reservoir_collector"] = create_rsv()
if app_state["reservoir_collector"]:
logger.info("RID reservoir collection enabled (large-dam daily status)")
except Exception as e:
app_state["reservoir_collector"] = None
logger.error(f"Reservoir collector initialization failed: {e}")
# Initialize health checks
health_manager = HealthCheckManager()
health_manager.add_check(DatabaseHealthCheck(app_state["scraper"].db_adapter))
@@ -372,6 +383,17 @@ async def background_scraping_task():
except Exception as e:
logger.error(f"HII collection failed: {e}")
# RID large-dam daily status (Mae Ngat storage/inflow/outflow)
reservoir_collector = app_state.get("reservoir_collector")
if reservoir_collector:
try:
rsv_saved = await asyncio.get_event_loop().run_in_executor(
None, reservoir_collector.run_cycle
)
set_gauge("reservoir_rows_saved", rsv_saved)
except Exception as e:
logger.error(f"Reservoir collection failed: {e}")
# Persist the Open-Meteo catchment rain (observed tail +
# 48h forecast) so the DB carries the weather context too
await _persist_rain()
@@ -798,6 +820,25 @@ def _hii_engine():
return store.engine
def _aux_stats_engine():
"""Any available engine for counting auxiliary tables in /api/stats.
The HII and reservoir collectors are gated by independent config flags;
either store's engine can run the guarded COUNT queries, so falling back
keeps the stats honest when one collector is disabled.
"""
engine = _hii_engine()
if engine is not None:
return engine
collector = app_state.get("reservoir_collector")
if not collector:
return None
store = collector.store
if not store.engine and not store.connect():
return None
return store.engine
def _hii_rows(sql: str, params: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Run a read query against the hii_* tables, JSON-normalizing numerics."""
engine = _hii_engine()
@@ -1222,21 +1263,26 @@ async def get_database_stats():
except Exception as e:
logger.warning(f"HII stats unavailable: {e}")
openmeteo_n = 0
try: # table appears with the 2026-08 rain work; older DBs lack it
engine = _hii_engine()
if engine is not None:
from sqlalchemy import text as _text
# Tables that appear with 2026-08 feature work; older DBs lack them,
# so each count is independently best-effort.
extra_counts = {"openmeteo_rain": 0, "rid_reservoir_daily": 0}
engine = _aux_stats_engine()
if engine is not None:
from sqlalchemy import text as _text
with engine.connect() as conn:
openmeteo_n = int(
conn.execute(
_text("SELECT COUNT(*) FROM openmeteo_rain")
).scalar()
or 0
)
except Exception:
pass
for table in extra_counts:
try:
with engine.connect() as conn:
extra_counts[table] = int(
conn.execute(
_text(f"SELECT COUNT(*) FROM {table}")
).scalar()
or 0
)
except Exception:
pass
openmeteo_n = extra_counts["openmeteo_rain"]
reservoir_n = extra_counts["rid_reservoir_daily"]
def as_dt(value):
if isinstance(value, str):
@@ -1261,11 +1307,13 @@ async def get_database_stats():
"total_measurements": stats["total_measurements"]
+ rain_n
+ wl_n
+ openmeteo_n,
+ openmeteo_n
+ reservoir_n,
"rid_measurements": stats["total_measurements"],
"hii_rainfall_measurements": rain_n,
"hii_waterlevel_measurements": wl_n,
"openmeteo_rain_measurements": openmeteo_n,
"reservoir_measurements": reservoir_n,
"station_count": stats["station_count"] + hii_stations,
"rid_station_count": stats["station_count"],
"hii_station_count": hii_stations,