#!/usr/bin/env python3 """Backfill rid_reservoir_daily with RID large-dam history (Mae Ngat et al.). One request per day against app.rid.go.th/reservoir/api/dams (archive reaches back to at least 2009). Days already stored are skipped, so reruns only fetch what is missing — safe alongside the hourly live collector, and a rerun repairs holes left by transient failures. Usage: uv run scripts/backfill_rid_reservoir.py # missing days since 2018-08-01 uv run scripts/backfill_rid_reservoir.py --start 2015-01-01 uv run scripts/backfill_rid_reservoir.py --db-url postgresql://... """ import argparse import datetime import logging import os import sys sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) from src.config import Config from src.rid_reservoir import RidReservoirStore, backfill DEFAULT_START = datetime.date(2018, 8, 1) # start of the water_measurements grid def main(argv=None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--start", type=datetime.date.fromisoformat, default=DEFAULT_START ) parser.add_argument("--end", type=datetime.date.fromisoformat, default=None) parser.add_argument("--db-url", default=None) parser.add_argument("--throttle", type=float, default=0.4) args = parser.parse_args(argv) logging.basicConfig( level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s" ) if args.db_url: # postgresql+psycopg2://... -> postgresql (driver suffix is not a dialect) connection_string = args.db_url db_type = args.db_url.split(":", 1)[0].split("+", 1)[0] else: cfg = Config.get_database_config() if cfg["type"] not in ("sqlite", "postgresql", "mysql"): print(f"requires a SQL database, got {cfg['type']}", file=sys.stderr) return 1 connection_string, db_type = cfg["connection_string"], cfg["type"] store = RidReservoirStore(connection_string, db_type) if not store.connect(): print( "database connection failed — check the connection string", file=sys.stderr, ) return 1 end = args.end or datetime.date.today() span_days = (end - args.start).days + 1 missing = span_days - len(store.present_dates(args.start, end)) saved = backfill(store, args.start, end, throttle_seconds=args.throttle) print(f"backfilled {saved} dam-day rows ({missing} days were missing)") return 0 if saved or missing == 0 else 1 if __name__ == "__main__": sys.exit(main())