#!/usr/bin/env python3 """ Enhanced Water Monitor Scraper with multiple database backend support """ import datetime import json import logging import os import time from typing import Dict, List, Optional import requests import schedule try: from .config import Config from .database_adapters import create_database_adapter from .logging_config import get_logger from .metrics import Timer, increment_counter, record_histogram, set_gauge from .rate_limiter import RateLimiter, RequestTracker from .validators import DataValidator except ImportError: # Handle case when running as standalone script from config import Config from database_adapters import create_database_adapter def get_logger(name): return logging.getLogger(name) def increment_counter(*args, **kwargs): pass def set_gauge(*args, **kwargs): pass def record_histogram(*args, **kwargs): pass class Timer: def __init__(self, *args, **kwargs): pass def __enter__(self): return self def __exit__(self, *args): pass class RateLimiter: def __init__(self, *args, **kwargs): pass def wait_if_needed(self): pass class RequestTracker: def __init__(self): pass def record_request(self, *args, **kwargs): pass class DataValidator: @staticmethod def validate_measurements(measurements): return measurements # Get logger instance logger = get_logger(__name__) class EnhancedWaterMonitorScraper: def __init__(self, db_config: Dict): """ Initialize scraper with database configuration Args: db_config: Database configuration dictionary """ self.api_url = "https://hyd-app-db.rid.go.th/webservice/getGroupHourlyWaterLevelReportAllHL.ashx" self.db_config = db_config.copy() # Make a copy to avoid modifying original self.db_adapter = None # Scheduler state tracking self.last_successful_update = None self.retry_mode = False self.next_hourly_check = None # Rate limiting and request tracking self.rate_limiter = RateLimiter(max_requests=10, time_window_seconds=60) self.request_tracker = RequestTracker() # HTTP session for API requests self.session = requests.Session() self.session.headers.update( { "User-Agent": Config.USER_AGENT, "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8", "Accept": "application/json, text/javascript, */*; q=0.01", "X-Requested-With": "XMLHttpRequest", } ) # Station mapping is persisted to a JSON file so that station CRUD via the # API survives restarts; on first run it is seeded from the bundled # defaults in data/stations.json. self.station_config_path = Config.STATION_CONFIG_PATH self.station_mapping = self._load_station_mapping() self.init_database() @staticmethod def _default_station_mapping_path() -> str: """Path to the bundled default station mapping shipped with the package.""" return os.path.join( os.path.dirname(os.path.abspath(__file__)), "data", "stations.json" ) def _load_station_mapping(self) -> Dict: """Load the station mapping, preferring the runtime-writable config file. Order of precedence: 1. The runtime config file (STATION_CONFIG_PATH) if it exists — this holds any changes made through the station CRUD API. 2. The bundled defaults in data/stations.json. """ for source in (self.station_config_path, self._default_station_mapping_path()): if source and os.path.exists(source): try: with open(source, encoding="utf-8") as f: mapping = json.load(f) logger.info(f"Loaded {len(mapping)} stations from {source}") return mapping except Exception as e: logger.error(f"Failed to load station mapping from {source}: {e}") logger.error( "No station mapping could be loaded; starting with an empty mapping" ) return {} def save_stations(self) -> bool: """Persist the current station mapping to the runtime config file. Written atomically (temp file + replace) so a crash mid-write cannot corrupt the existing configuration. """ path = self.station_config_path if not path: logger.warning( "STATION_CONFIG_PATH not set; station changes will not persist" ) return False try: tmp_path = f"{path}.tmp" with open(tmp_path, "w", encoding="utf-8") as f: json.dump(self.station_mapping, f, ensure_ascii=False, indent=2) f.write("\n") os.replace(tmp_path, path) logger.info(f"Persisted {len(self.station_mapping)} stations to {path}") return True except Exception as e: logger.error(f"Failed to persist station mapping to {path}: {e}") return False def init_database(self): """Initialize database connection""" try: # Extract db_type and pass remaining config as kwargs db_config_copy = self.db_config.copy() db_type = db_config_copy.pop("type") self.db_adapter = create_database_adapter(db_type, **db_config_copy) success = self.db_adapter.connect() if success: logger.info(f"Successfully connected to {db_type.upper()} database") set_gauge("database_connected", 1) increment_counter("database_connections_successful") else: logger.error(f"Failed to connect to {db_type.upper()} database") set_gauge("database_connected", 0) increment_counter("database_connections_failed") except Exception as e: logger.error(f"Error initializing database: {e}") set_gauge("database_connected", 0) increment_counter("database_connections_failed") self.db_adapter = None def fetch_water_data_for_date( self, target_date: datetime.datetime ) -> Optional[List[Dict]]: """Fetch water levels and discharge data from API for a specific date""" with Timer("api_request_duration"): try: logger.info( f"Starting data fetch from API for date: {target_date.strftime('%Y-%m-%d')}" ) # Rate limiting self.rate_limiter.wait_if_needed() # Create Thai format date (Buddhist calendar) thai_year = target_date.year + 543 thai_date = f"{target_date.day:02d}/{target_date.month:02d}/{thai_year}" # API parameters payload = { "DW[UtokID]": "1", "DW[BasinID]": "6", "DW[TimeCurrent]": thai_date, "_search": "false", "nd": str(int(time.time() * 1000)), "rows": "100", "page": "1", "sidx": "indexhourly", "sord": "asc", } logger.debug(f"API parameters: {payload}") # POST request to API start_time = time.time() response = self.session.post(self.api_url, data=payload, timeout=30) response_time = time.time() - start_time response.raise_for_status() # Record successful request self.request_tracker.record_request(True, response_time) increment_counter("api_requests_successful") record_histogram("api_response_time", response_time) # Parse JSON response try: json_data = response.json() logger.debug( f"API response received: {len(str(json_data))} characters" ) except ValueError as e: logger.error(f"Error parsing JSON response: {e}") self.request_tracker.record_request( False, response_time, "json_parse_error" ) increment_counter("api_requests_failed") return None water_data = [] # Parse JSON data if json_data and isinstance(json_data, dict) and "rows" in json_data: for row in json_data["rows"]: try: # Parse timestamp time_str = row.get("hourlytime", "") if not time_str: continue try: # Format: "1.00", "2.00", ..., "24.00" api_hour = int(float(time_str)) if api_hour < 1 or api_hour > 24: continue if api_hour == 24: # Hour 24 = midnight (00:00) of the next day data_time = target_date.replace( hour=0, minute=0, second=0, microsecond=0 ) data_time = data_time + datetime.timedelta(days=1) else: # Hours 1-23 = 01:00-23:00 of the same day data_time = target_date.replace( hour=api_hour, minute=0, second=0, microsecond=0 ) except (ValueError, IndexError): logger.warning(f"Could not parse timestamp: {time_str}") continue # Parse all water levels and discharge values station_count = 0 for station_num in range(1, 17): # Stations 1-16 wl_key = f"wlvalues{station_num}" q_key = f"qvalues{station_num}" qp_key = f"QPercent{station_num}" # Check if water level data exists (required) if wl_key in row: try: water_level = row[wl_key] # Skip if water level is None or invalid if water_level is None: continue # Convert water level to float (required) water_level = float(water_level) # Try to parse discharge data (optional) discharge = None discharge_percent = None if q_key in row: try: discharge_raw = row[q_key] if ( discharge_raw is not None and discharge_raw != "***" ): discharge = float(discharge_raw) # Only parse discharge percent if discharge is valid discharge_percent_raw = row.get( qp_key ) if ( discharge_percent_raw is not None ): try: discharge_percent = float( discharge_percent_raw ) except (ValueError, TypeError): discharge_percent = None else: logger.debug( "Skipping malformed discharge data for " f"station {station_num}: {discharge_raw}" ) except (ValueError, TypeError) as e: logger.debug( f"Could not parse discharge for station {station_num}: {e}" ) station_info = self.station_mapping.get( str(station_num), { "code": f"P.{19+station_num}", "thai_name": f"Station {station_num}", "english_name": f"Station {station_num}", }, ) water_data.append( { "timestamp": data_time, "station_id": station_num, "station_code": station_info["code"], "station_name_en": station_info[ "english_name" ], "station_name_th": station_info[ "thai_name" ], "latitude": station_info.get( "latitude" ), "longitude": station_info.get( "longitude" ), "geohash": station_info.get("geohash"), "water_level": water_level, "water_level_unit": "m", "discharge": discharge, "discharge_unit": "cms", "discharge_percent": discharge_percent, "status": "active", } ) station_count += 1 except (ValueError, TypeError) as e: logger.warning( f"Could not parse water level for station {station_num}: {e}" ) continue logger.debug( f"Processed {station_count} stations for time {time_str}" ) except Exception as e: logger.warning(f"Error processing data row: {e}") continue # Validate data water_data = DataValidator.validate_measurements(water_data) logger.info( f"Successfully fetched {len(water_data)} data points from API " f"for {target_date.strftime('%Y-%m-%d')}" ) return water_data except requests.RequestException as e: logger.error(f"Network error fetching API data: {e}") self.request_tracker.record_request(False, 0, "network_error") increment_counter("api_requests_failed") return None except Exception as e: logger.error(f"Unexpected error fetching API data: {e}") self.request_tracker.record_request(False, 0, "unexpected_error") increment_counter("api_requests_failed") return None def fetch_water_data(self) -> Optional[List[Dict]]: """Fetch water levels and discharge data from API with smart date selection""" current_time = datetime.datetime.now() # If it's past 01:00, try today's data first, then yesterday as fallback if current_time.hour >= 1: logger.info( "After 01:00 - trying today's data first, will fallback to yesterday if needed" ) # Try today's data first today_data = self.fetch_water_data_for_date(current_time) if today_data and len(today_data) > 0: logger.info( f"Successfully fetched {len(today_data)} data points for today" ) return today_data # Fallback to yesterday's data logger.info("No data available for today, trying yesterday's data") yesterday = current_time - datetime.timedelta(days=1) yesterday_data = self.fetch_water_data_for_date(yesterday) if yesterday_data and len(yesterday_data) > 0: logger.info( f"Successfully fetched {len(yesterday_data)} data points for yesterday" ) return yesterday_data logger.warning("No data available for today or yesterday") return None else: # Before 01:00 - only try yesterday's data (API likely hasn't updated yet) logger.info("Before 01:00 - fetching yesterday's data only") yesterday = current_time - datetime.timedelta(days=1) return self.fetch_water_data_for_date(yesterday) def save_to_database(self, water_data: List[Dict], max_retries: int = 3) -> bool: """Save water measurements to database with retry logic""" if not self.db_adapter: logger.error("Database adapter not initialized") return False if not water_data: logger.warning("No data to save") return False for attempt in range(max_retries): try: success = self.db_adapter.save_measurements(water_data) if success: logger.info( f"Successfully saved {len(water_data)} measurements to database" ) increment_counter("database_saves_successful") set_gauge("last_save_timestamp", time.time()) return True else: logger.warning(f"Save attempt {attempt + 1} failed, retrying...") except Exception as e: if "database is locked" in str(e).lower() and attempt < max_retries - 1: logger.warning( f"Database locked on attempt {attempt + 1}, retrying in {2 ** attempt} seconds..." ) time.sleep(2**attempt) # Exponential backoff continue else: logger.error( f"Error saving to database (attempt {attempt + 1}): {e}" ) if attempt == max_retries - 1: increment_counter("database_saves_failed") return False return False def get_latest_data(self, limit: int = 100) -> List[Dict]: """Get latest data from database""" if not self.db_adapter: return [] try: return self.db_adapter.get_latest_measurements(limit=limit) except Exception as e: logger.error(f"Error getting latest data: {e}") return [] def _check_data_freshness(self, water_data: List[Dict]) -> bool: """Check if the fetched data contains new data for the current hour""" if not water_data: return False current_time = datetime.datetime.now() current_hour = current_time.hour # Find the most recent timestamp in the data latest_timestamp = None for data_point in water_data: timestamp = data_point.get("timestamp") if timestamp and (latest_timestamp is None or timestamp > latest_timestamp): latest_timestamp = timestamp if latest_timestamp is None: logger.warning("No valid timestamps found in data") return False latest_hour = latest_timestamp.hour time_diff = current_time - latest_timestamp minutes_old = time_diff.total_seconds() / 60 logger.info( f"Current time: {current_time.strftime('%H:%M')}, Latest data: {latest_timestamp.strftime('%H:%M')}" ) logger.info( f"Current hour: {current_hour}, Latest data hour: {latest_hour}, Age: {minutes_old:.1f} minutes" ) # Strict check: we need data from the current hour # If it's 20:xx and we only have data up to 19:xx, that's stale - go to retry mode has_current_hour_data = latest_hour >= current_hour if not has_current_hour_data: logger.warning( f"No new data available - expected hour {current_hour}, got {latest_hour}" ) logger.warning("Switching to retry mode until new data becomes available") return False else: logger.info(f"Fresh data available for current hour {current_hour}") return True def run_scraping_cycle(self) -> bool: """Run a complete scraping cycle with freshness check""" logger.info("Starting scraping cycle...") try: # Fetch current data water_data = self.fetch_water_data() if water_data: # Check if data is fresh/recent is_fresh = self._check_data_freshness(water_data) if is_fresh: success = self.save_to_database(water_data) if success: logger.info( "Scraping cycle completed successfully with fresh data" ) increment_counter("scraping_cycles_successful") return True else: logger.error("Failed to save data") increment_counter("scraping_cycles_failed") return False else: # Data exists but is stale logger.warning( "Data fetched but is stale - treating as no fresh data available" ) increment_counter("scraping_cycles_failed") return False else: logger.warning("No data fetched") increment_counter("scraping_cycles_failed") return False except Exception as e: logger.error(f"Scraping cycle failed: {e}") increment_counter("scraping_cycles_failed") return False def fill_data_gaps(self, days_back: Optional[int] = None) -> int: """Fill gaps in stored data by re-fetching incomplete days. Args: days_back: How many days back to scan. None scans the whole data range, from the earliest measurement in the database to now. Detection is hour-granular where the backend supports it: any calendar day missing one or more hourly slots gets its source date(s) re-fetched. Because the API reports hour 24 as midnight of the next day, a missing 00:00 slot on day D is repaired by re-fetching day D-1. """ logger = get_logger(__name__) filled_count = 0 try: now = datetime.datetime.now() end_date = now.date() if days_back is not None: start_date = end_date - datetime.timedelta(days=days_back) else: if not self.db_adapter: logger.error("Database adapter not initialized") return 0 date_range = self.db_adapter.get_measurement_date_range() if not date_range: logger.error( "Database is empty or does not support range queries; " "use --import-historical to seed data first" ) return 0 start_date = date_range[0].date() logger.info(f"Checking for gaps from {start_date} to {end_date}") fetch_dates = self._find_gap_fetch_dates(start_date, end_date, now) if not fetch_dates: logger.info("No gaps found") return 0 logger.info(f"Found {len(fetch_dates)} day(s) needing a re-fetch") for fetch_date in sorted(fetch_dates): fetch_dt = datetime.datetime.combine(fetch_date, datetime.time.min) logger.info(f"Filling gap for date: {fetch_date}") data = self.fetch_water_data_for_date(fetch_dt) if data: if self.save_to_database(data): filled_count += len(data) logger.info( f"Filled {len(data)} measurements for {fetch_date}" ) else: logger.warning(f"Failed to save data for {fetch_date}") else: logger.warning(f"No data available for {fetch_date}") # Be respectful to the API time.sleep(1) except Exception as e: logger.error(f"Gap filling error: {e}") return filled_count def _find_gap_fetch_dates( self, start_date: datetime.date, end_date: datetime.date, now: datetime.datetime, ) -> List[datetime.date]: """Determine which source dates need re-fetching to fill gaps. Uses hour-granular detection when the adapter supports it, otherwise falls back to re-fetching only days with no data at all. """ hours_by_day = None if self.db_adapter: hours_by_day = self.db_adapter.get_recorded_hours_by_day( start_date, end_date ) fetch_dates = set() if hours_by_day is None: logger.info( "Backend does not support hour-granular gap detection; " "checking for fully-missing days only" ) current = start_date while current <= end_date: current_dt = datetime.datetime.combine(current, datetime.time.min) if not self._check_data_exists_for_date(current_dt): fetch_dates.add(current) current += datetime.timedelta(days=1) return sorted(fetch_dates) current = start_date while current <= end_date: if current == now.date(): # Today: only expect hours that have already passed expected_hours = set(range(0, now.hour)) else: expected_hours = set(range(24)) missing = expected_hours - hours_by_day.get(current, set()) if missing: # Hours 1-23 of day D come from fetching D; hour 0 comes from # the previous day's fetch (the API's "hour 24") if any(h >= 1 for h in missing): fetch_dates.add(current) if 0 in missing and current > start_date: fetch_dates.add(current - datetime.timedelta(days=1)) current += datetime.timedelta(days=1) return sorted(fetch_dates) def update_existing_data(self, days_back: int) -> int: """Update existing data with latest values for the specified number of days back""" logger = get_logger(__name__) updated_count = 0 try: # Calculate date range end_date = datetime.datetime.now() start_date = end_date - datetime.timedelta(days=days_back) logger.info(f"Updating data from {start_date.date()} to {end_date.date()}") # Iterate through each date in the range current_date = start_date while current_date <= end_date: logger.info(f"Updating data for date: {current_date.date()}") # Fetch fresh data for this date data = self.fetch_water_data_for_date(current_date) if data: # Save the data (this will update existing records) if self.save_to_database(data): updated_count += len(data) logger.info( f"Updated {len(data)} measurements for {current_date.date()}" ) else: logger.warning( f"Failed to update data for {current_date.date()}" ) else: logger.warning(f"No data available for {current_date.date()}") current_date += datetime.timedelta(days=1) except Exception as e: logger.error(f"Data update error: {e}") return updated_count def _check_data_exists_for_date(self, target_date: datetime.datetime) -> bool: """Check if data exists for a specific date""" try: if not self.db_adapter: return False # Get data for the specific date measurements = self.db_adapter.get_measurements_for_date(target_date) return len(measurements) > 0 except Exception as e: logger = get_logger(__name__) logger.debug(f"Error checking data existence: {e}") return False def import_historical_data( self, start_date: datetime.datetime, end_date: datetime.datetime, skip_existing: bool = True, ) -> int: """ Import historical data for a date range Args: start_date: Start date for historical import end_date: End date for historical import skip_existing: Skip dates that already have data (default: True) Returns: Number of data points imported """ logger.info( f"Starting historical data import from {start_date.date()} to {end_date.date()}" ) total_imported = 0 current_date = start_date while current_date <= end_date: try: # Check if data already exists for this date if skip_existing and self._check_data_exists_for_date(current_date): logger.info( f"Data already exists for {current_date.date()}, skipping..." ) current_date += datetime.timedelta(days=1) continue logger.info(f"Importing data for {current_date.date()}...") # Fetch data for this date data = self.fetch_water_data_for_date(current_date) if data: # Save to database if self.save_to_database(data): total_imported += len(data) logger.info( f"Successfully imported {len(data)} data points for {current_date.date()}" ) else: logger.warning(f"Failed to save data for {current_date.date()}") else: logger.warning(f"No data available for {current_date.date()}") # Add small delay to be respectful to the API time.sleep(1) except Exception as e: logger.error(f"Error importing data for {current_date.date()}: {e}") current_date += datetime.timedelta(days=1) logger.info( f"Historical import completed. Total data points imported: {total_imported}" ) return total_imported # Main execution for standalone usage if __name__ == "__main__": import argparse import sys logging.basicConfig( level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s", handlers=[logging.FileHandler("water_monitor.log"), logging.StreamHandler()], ) parser = argparse.ArgumentParser(description="Thailand Water Monitor") parser.add_argument("--test", action="store_true", help="Run single test cycle") args = parser.parse_args() # Default SQLite configuration db_config = {"type": "sqlite", "connection_string": "sqlite:///water_levels.db"} try: scraper = EnhancedWaterMonitorScraper(db_config) if args.test: logger.info("Running test cycle...") result = scraper.run_scraping_cycle() if result: logger.info("✅ Test completed successfully") sys.exit(0) else: logger.error("❌ Test failed") sys.exit(1) else: logger.info("Starting continuous monitoring...") schedule.every(1).hours.do(scraper.run_scraping_cycle) # Run initial cycle scraper.run_scraping_cycle() while True: schedule.run_pending() time.sleep(60) except KeyboardInterrupt: logger.info("Monitoring stopped by user") except Exception as e: logger.error(f"Error: {e}") sys.exit(1)