feat: RID large-dam daily collector — Mae Ngat storage/inflow/outflow

POST app.rid.go.th/reservoir/api/dams (open, archive >=2009) collected
hourly into rid_dams + rid_reservoir_daily; backfill script fetches only
missing days so reruns repair holes and are safe alongside the live
collector. /api/stats counts the new table via an engine fallback that
works when HII collection is disabled. Mae Ngat (DAM_ID 200103) hit 113%
usable capacity with ~19 MCM/day inflow in the Oct 2024 flood — candidate
features for the next retrain.
This commit is contained in:
2026-08-13 10:29:34 +07:00
parent 811af1625b
commit 6eafb353b1
7 changed files with 712 additions and 19 deletions
+3 -1
View File
@@ -26,7 +26,9 @@ def main(argv=None) -> int:
level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s"
)
if args.db_url:
connection_string, db_type = args.db_url, args.db_url.split(":", 1)[0]
# postgresql+psycopg2://... -> postgresql (driver suffix is not a dialect)
connection_string = args.db_url
db_type = args.db_url.split(":", 1)[0].split("+", 1)[0]
else:
cfg = Config.get_database_config()
if cfg["type"] not in ("sqlite", "postgresql", "mysql"):
+69
View File
@@ -0,0 +1,69 @@
#!/usr/bin/env python3
"""Backfill rid_reservoir_daily with RID large-dam history (Mae Ngat et al.).
One request per day against app.rid.go.th/reservoir/api/dams (archive reaches
back to at least 2009). Days already stored are skipped, so reruns only fetch
what is missing — safe alongside the hourly live collector, and a rerun
repairs holes left by transient failures.
Usage:
uv run scripts/backfill_rid_reservoir.py # missing days since 2018-08-01
uv run scripts/backfill_rid_reservoir.py --start 2015-01-01
uv run scripts/backfill_rid_reservoir.py --db-url postgresql://...
"""
import argparse
import datetime
import logging
import os
import sys
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
from src.config import Config
from src.rid_reservoir import RidReservoirStore, backfill
DEFAULT_START = datetime.date(2018, 8, 1) # start of the water_measurements grid
def main(argv=None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--start", type=datetime.date.fromisoformat, default=DEFAULT_START
)
parser.add_argument("--end", type=datetime.date.fromisoformat, default=None)
parser.add_argument("--db-url", default=None)
parser.add_argument("--throttle", type=float, default=0.4)
args = parser.parse_args(argv)
logging.basicConfig(
level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s"
)
if args.db_url:
# postgresql+psycopg2://... -> postgresql (driver suffix is not a dialect)
connection_string = args.db_url
db_type = args.db_url.split(":", 1)[0].split("+", 1)[0]
else:
cfg = Config.get_database_config()
if cfg["type"] not in ("sqlite", "postgresql", "mysql"):
print(f"requires a SQL database, got {cfg['type']}", file=sys.stderr)
return 1
connection_string, db_type = cfg["connection_string"], cfg["type"]
store = RidReservoirStore(connection_string, db_type)
if not store.connect():
print(
"database connection failed — check the connection string",
file=sys.stderr,
)
return 1
end = args.end or datetime.date.today()
span_days = (end - args.start).days + 1
missing = span_days - len(store.present_dates(args.start, end))
saved = backfill(store, args.start, end, throttle_seconds=args.throttle)
print(f"backfilled {saved} dam-day rows ({missing} days were missing)")
return 0 if saved or missing == 0 else 1
if __name__ == "__main__":
sys.exit(main())