feat: backfill hii_waterlevel from the HII waterlevel_graph archive
CI/CD Pipeline - Northern Thailand Ping River Monitor / Test Suite (3.11) (push) Failing after 31s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Build Docker Image (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Integration Test with Services (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Deploy to Staging (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Deploy to Production (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Performance Test (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Cleanup (push) Successful in 0s
Documentation / Documentation Summary (push) Successful in 2s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Code Quality (push) Successful in 13s
Documentation / Validate Documentation (push) Failing after 9s
Documentation / Generate API Documentation (push) Successful in 9s
Documentation / Build Sphinx Documentation (push) Successful in 15s

scripts/backfill_hii_waterlevel.py walks the api-v3 waterlevel_graph
endpoint (hourly wl_msl + discharge, archive back to ~2019) in full-year
windows per station and upserts into hii_waterlevel. Defaults to the
RID-mirror and key stations; --stations/--all/--start/--end/--chunk-days
override. History upserts touch only wl_msl and discharge so colliding
live-snapshot rows keep storage_percent/situation_level. Idempotent and
safe to re-run.
This commit is contained in:
2026-08-11 15:11:08 +07:00
parent 33d8baa8dd
commit d72496f404
5 changed files with 383 additions and 5 deletions
+39 -4
View File
@@ -162,18 +162,18 @@ class HiiClient:
self.session = session or requests.Session()
self.timeout = timeout
def _get(self, endpoint: str) -> Dict:
def get(self, endpoint: str, params: Optional[Dict] = None) -> Dict:
response = self.session.get(
f"{self.base_url}/{endpoint}", timeout=self.timeout
f"{self.base_url}/{endpoint}", params=params, timeout=self.timeout
)
response.raise_for_status()
return response.json()
def fetch_rain(self, basin_code: int = PING_BASIN_CODE) -> List[Dict]:
return parse_rain_records(self._get("rain_24h"), basin_code)
return parse_rain_records(self.get("rain_24h"), basin_code)
def fetch_waterlevel(self, basin_code: int = PING_BASIN_CODE) -> List[Dict]:
return parse_waterlevel_records(self._get("waterlevel_load"), basin_code)
return parse_waterlevel_records(self.get("waterlevel_load"), basin_code)
class HiiStore:
@@ -350,6 +350,41 @@ class HiiStore:
],
)
def save_waterlevel_history(self, station_id: int, rows: List[Dict]) -> int:
"""Upsert backfilled history rows, touching only wl_msl and discharge.
Live-snapshot rows for the same (station, hour) keep their extra
columns (storage_percent, situation_level, ...) untouched.
"""
if not rows:
return 0
if not self.engine and not self.connect():
return 0
from sqlalchemy import text
cols = "(station_id, timestamp, wl_msl, discharge)"
values = "(:station_id, :timestamp, :wl_msl, :discharge)"
if self.db_type == "mysql":
sql = (
f"INSERT INTO hii_waterlevel {cols} VALUES {values} "
"ON DUPLICATE KEY UPDATE wl_msl = VALUES(wl_msl), "
"discharge = VALUES(discharge)"
)
else: # sqlite (>=3.24) and postgresql share upsert syntax
sql = (
f"INSERT INTO hii_waterlevel {cols} VALUES {values} "
"ON CONFLICT (station_id, timestamp) DO UPDATE SET "
"wl_msl = EXCLUDED.wl_msl, discharge = EXCLUDED.discharge"
)
params = [{**row, "station_id": station_id} for row in rows]
try:
with self.engine.begin() as conn:
conn.execute(text(sql), params)
return len(params)
except Exception as e:
logger.error(f"HiiStore history save failed: {e}")
return 0
def _save(
self,
records: List[Dict],