feat: backfill hii_waterlevel from the HII waterlevel_graph archive
CI/CD Pipeline - Northern Thailand Ping River Monitor / Test Suite (3.11) (push) Failing after 31s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Build Docker Image (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Integration Test with Services (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Deploy to Staging (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Deploy to Production (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Performance Test (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Cleanup (push) Successful in 0s
Documentation / Documentation Summary (push) Successful in 2s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Code Quality (push) Successful in 13s
Documentation / Validate Documentation (push) Failing after 9s
Documentation / Generate API Documentation (push) Successful in 9s
Documentation / Build Sphinx Documentation (push) Successful in 15s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Test Suite (3.11) (push) Failing after 31s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Build Docker Image (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Integration Test with Services (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Deploy to Staging (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Deploy to Production (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Performance Test (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Cleanup (push) Successful in 0s
Documentation / Documentation Summary (push) Successful in 2s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Code Quality (push) Successful in 13s
Documentation / Validate Documentation (push) Failing after 9s
Documentation / Generate API Documentation (push) Successful in 9s
Documentation / Build Sphinx Documentation (push) Successful in 15s
scripts/backfill_hii_waterlevel.py walks the api-v3 waterlevel_graph endpoint (hourly wl_msl + discharge, archive back to ~2019) in full-year windows per station and upserts into hii_waterlevel. Defaults to the RID-mirror and key stations; --stations/--all/--start/--end/--chunk-days override. History upserts touch only wl_msl and discharge so colliding live-snapshot rows keep storage_percent/situation_level. Idempotent and safe to re-run.
This commit is contained in:
+39
-4
@@ -162,18 +162,18 @@ class HiiClient:
|
||||
self.session = session or requests.Session()
|
||||
self.timeout = timeout
|
||||
|
||||
def _get(self, endpoint: str) -> Dict:
|
||||
def get(self, endpoint: str, params: Optional[Dict] = None) -> Dict:
|
||||
response = self.session.get(
|
||||
f"{self.base_url}/{endpoint}", timeout=self.timeout
|
||||
f"{self.base_url}/{endpoint}", params=params, timeout=self.timeout
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
def fetch_rain(self, basin_code: int = PING_BASIN_CODE) -> List[Dict]:
|
||||
return parse_rain_records(self._get("rain_24h"), basin_code)
|
||||
return parse_rain_records(self.get("rain_24h"), basin_code)
|
||||
|
||||
def fetch_waterlevel(self, basin_code: int = PING_BASIN_CODE) -> List[Dict]:
|
||||
return parse_waterlevel_records(self._get("waterlevel_load"), basin_code)
|
||||
return parse_waterlevel_records(self.get("waterlevel_load"), basin_code)
|
||||
|
||||
|
||||
class HiiStore:
|
||||
@@ -350,6 +350,41 @@ class HiiStore:
|
||||
],
|
||||
)
|
||||
|
||||
def save_waterlevel_history(self, station_id: int, rows: List[Dict]) -> int:
|
||||
"""Upsert backfilled history rows, touching only wl_msl and discharge.
|
||||
|
||||
Live-snapshot rows for the same (station, hour) keep their extra
|
||||
columns (storage_percent, situation_level, ...) untouched.
|
||||
"""
|
||||
if not rows:
|
||||
return 0
|
||||
if not self.engine and not self.connect():
|
||||
return 0
|
||||
from sqlalchemy import text
|
||||
|
||||
cols = "(station_id, timestamp, wl_msl, discharge)"
|
||||
values = "(:station_id, :timestamp, :wl_msl, :discharge)"
|
||||
if self.db_type == "mysql":
|
||||
sql = (
|
||||
f"INSERT INTO hii_waterlevel {cols} VALUES {values} "
|
||||
"ON DUPLICATE KEY UPDATE wl_msl = VALUES(wl_msl), "
|
||||
"discharge = VALUES(discharge)"
|
||||
)
|
||||
else: # sqlite (>=3.24) and postgresql share upsert syntax
|
||||
sql = (
|
||||
f"INSERT INTO hii_waterlevel {cols} VALUES {values} "
|
||||
"ON CONFLICT (station_id, timestamp) DO UPDATE SET "
|
||||
"wl_msl = EXCLUDED.wl_msl, discharge = EXCLUDED.discharge"
|
||||
)
|
||||
params = [{**row, "station_id": station_id} for row in rows]
|
||||
try:
|
||||
with self.engine.begin() as conn:
|
||||
conn.execute(text(sql), params)
|
||||
return len(params)
|
||||
except Exception as e:
|
||||
logger.error(f"HiiStore history save failed: {e}")
|
||||
return 0
|
||||
|
||||
def _save(
|
||||
self,
|
||||
records: List[Dict],
|
||||
|
||||
Reference in New Issue
Block a user