feat: hourly HII/ThaiWater rainfall + backup water-level collection
CI/CD Pipeline - Northern Thailand Ping River Monitor / Test Suite (3.11) (push) Failing after 37s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Build Docker Image (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Integration Test with Services (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Deploy to Staging (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Deploy to Production (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Performance Test (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Code Quality (push) Successful in 16s
Documentation / Validate Documentation (push) Failing after 9s
Documentation / Generate API Documentation (push) Successful in 10s
Documentation / Build Sphinx Documentation (push) Successful in 15s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Cleanup (push) Successful in 0s
Documentation / Documentation Summary (push) Successful in 2s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Test Suite (3.11) (push) Failing after 37s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Build Docker Image (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Integration Test with Services (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Deploy to Staging (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Deploy to Production (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Performance Test (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Code Quality (push) Successful in 16s
Documentation / Validate Documentation (push) Failing after 9s
Documentation / Generate API Documentation (push) Successful in 10s
Documentation / Build Sphinx Documentation (push) Successful in 15s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Cleanup (push) Successful in 0s
Documentation / Documentation Summary (push) Successful in 2s
Poll the open api-v3.thaiwater.net public endpoints (rain_24h, waterlevel_load), filter to the Ping basin, and persist to new hii_rain_stations/hii_rainfall and hii_wl_stations/hii_waterlevel tables (auto-created; sqlite/postgresql/mysql). Water levels stay in their own tables since HII reports m MSL from a different station set; rid_code maps mirrors like ridhydro_P.1 to P.1 and offset_msl converts MSL to gauge datum. Runs every scraping cycle in web-api and continuous modes (hourly cadence even during 1-minute RID retry), one-shot via --collect-hii; docs/DATA_SOURCES.md catalogs all probed endpoints.
This commit is contained in:
+62
@@ -86,6 +86,19 @@ def run_continuous_monitoring():
|
||||
|
||||
alerting = WaterLevelAlertSystem()
|
||||
|
||||
# Initialize HII/ThaiWater collector (rainfall + backup water level)
|
||||
hii_collector = None
|
||||
try:
|
||||
from .hii_collector import create_collector_from_config
|
||||
|
||||
hii_collector = create_collector_from_config()
|
||||
if hii_collector:
|
||||
logger.info(
|
||||
"HII collection enabled (Ping-basin rainfall + water level)"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"HII collector initialization failed: {e}")
|
||||
|
||||
# Setup signal handlers
|
||||
setup_signal_handlers(scraper)
|
||||
|
||||
@@ -108,6 +121,14 @@ def run_continuous_monitoring():
|
||||
retry_mode = not initial_success
|
||||
last_successful_fetch = None if not initial_success else datetime.now()
|
||||
|
||||
last_hii_run = None
|
||||
if hii_collector:
|
||||
try:
|
||||
hii_collector.run_cycle()
|
||||
last_hii_run = datetime.now()
|
||||
except Exception as e:
|
||||
logger.error(f"HII collection failed: {e}")
|
||||
|
||||
if retry_mode:
|
||||
logger.warning("No data fetched in initial run - entering retry mode")
|
||||
next_run = datetime.now() + timedelta(minutes=1)
|
||||
@@ -126,6 +147,18 @@ def run_continuous_monitoring():
|
||||
logger.info("Running scheduled data collection...")
|
||||
success = scraper.run_scraping_cycle()
|
||||
|
||||
# HII feeds update hourly; keep collecting on that cadence even
|
||||
# when the RID scraper is in 1-minute retry mode.
|
||||
if hii_collector and (
|
||||
last_hii_run is None
|
||||
or current_time - last_hii_run >= timedelta(minutes=55)
|
||||
):
|
||||
try:
|
||||
hii_collector.run_cycle()
|
||||
last_hii_run = current_time
|
||||
except Exception as e:
|
||||
logger.error(f"HII collection failed: {e}")
|
||||
|
||||
if success:
|
||||
last_successful_fetch = current_time
|
||||
|
||||
@@ -180,6 +213,27 @@ def run_continuous_monitoring():
|
||||
return True
|
||||
|
||||
|
||||
def run_hii_collection():
|
||||
"""Run a single HII/ThaiWater collection cycle (rainfall + water level)"""
|
||||
try:
|
||||
Config.validate_config()
|
||||
from .hii_collector import create_collector_from_config
|
||||
|
||||
collector = create_collector_from_config()
|
||||
if not collector:
|
||||
logger.error(
|
||||
"HII collection unavailable (disabled via ENABLE_HII_COLLECTION "
|
||||
"or DB_TYPE is not a SQL database)"
|
||||
)
|
||||
return False
|
||||
|
||||
counts = collector.run_cycle()
|
||||
return counts["rainfall"] > 0 or counts["waterlevel"] > 0
|
||||
except Exception as e:
|
||||
logger.error(f"HII collection failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def run_gap_filling(days_back: Optional[int]):
|
||||
"""Run gap filling for missing data (days_back=None scans the whole range)"""
|
||||
if days_back is None:
|
||||
@@ -504,6 +558,12 @@ Examples:
|
||||
"--alert-test", action="store_true", help="Send test alert message to Matrix"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--collect-hii",
|
||||
action="store_true",
|
||||
help="Run one HII/ThaiWater collection cycle (rainfall + water level)",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--log-level",
|
||||
choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
|
||||
@@ -558,6 +618,8 @@ Examples:
|
||||
success = run_alert_check()
|
||||
elif args.alert_test:
|
||||
success = run_alert_test()
|
||||
elif args.collect_hii:
|
||||
success = run_hii_collection()
|
||||
else:
|
||||
success = run_continuous_monitoring()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user