grabowski
5e62ea529d
Documentation / Generate API Documentation (push) Successful in 11s
Documentation / Build Sphinx Documentation (push) Successful in 17s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Cleanup (push) Successful in 1s
Documentation / Documentation Summary (push) Successful in 3s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Test Suite (3.11) (push) Failing after 29s
CI/CD Pipeline - Northern Thailand Ping River Monitor / Build Docker Image (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Integration Test with Services (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Deploy to Staging (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Deploy to Production (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Performance Test (push) Skipped
CI/CD Pipeline - Northern Thailand Ping River Monitor / Code Quality (push) Successful in 14s
Documentation / Validate Documentation (push) Failing after 9s
Gap detection is now hour-granular: days with partial data (missing hourly slots) are re-fetched, not just days with no rows at all. The API's hour-24-is-next-midnight quirk is handled by re-fetching day D-1 when day D is missing its 00:00 slot. SQL adapters gain single-query range and recorded-hours lookups; non-SQL backends fall back to the old day-granular check.
872 lines
35 KiB
Python
872 lines
35 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Enhanced Water Monitor Scraper with multiple database backend support
|
|
"""
|
|
|
|
import datetime
|
|
import json
|
|
import logging
|
|
import os
|
|
import time
|
|
from typing import Dict, List, Optional
|
|
|
|
import requests
|
|
import schedule
|
|
|
|
try:
|
|
from .config import Config
|
|
from .database_adapters import create_database_adapter
|
|
from .logging_config import get_logger
|
|
from .metrics import Timer, increment_counter, record_histogram, set_gauge
|
|
from .rate_limiter import RateLimiter, RequestTracker
|
|
from .validators import DataValidator
|
|
except ImportError:
|
|
# Handle case when running as standalone script
|
|
from config import Config
|
|
from database_adapters import create_database_adapter
|
|
|
|
def get_logger(name):
|
|
return logging.getLogger(name)
|
|
|
|
def increment_counter(*args, **kwargs):
|
|
pass
|
|
|
|
def set_gauge(*args, **kwargs):
|
|
pass
|
|
|
|
def record_histogram(*args, **kwargs):
|
|
pass
|
|
|
|
class Timer:
|
|
def __init__(self, *args, **kwargs):
|
|
pass
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *args):
|
|
pass
|
|
|
|
class RateLimiter:
|
|
def __init__(self, *args, **kwargs):
|
|
pass
|
|
|
|
def wait_if_needed(self):
|
|
pass
|
|
|
|
class RequestTracker:
|
|
def __init__(self):
|
|
pass
|
|
|
|
def record_request(self, *args, **kwargs):
|
|
pass
|
|
|
|
class DataValidator:
|
|
@staticmethod
|
|
def validate_measurements(measurements):
|
|
return measurements
|
|
|
|
|
|
# Get logger instance
|
|
logger = get_logger(__name__)
|
|
|
|
|
|
class EnhancedWaterMonitorScraper:
|
|
def __init__(self, db_config: Dict):
|
|
"""
|
|
Initialize scraper with database configuration
|
|
|
|
Args:
|
|
db_config: Database configuration dictionary
|
|
"""
|
|
self.api_url = "https://hyd-app-db.rid.go.th/webservice/getGroupHourlyWaterLevelReportAllHL.ashx"
|
|
self.db_config = db_config.copy() # Make a copy to avoid modifying original
|
|
self.db_adapter = None
|
|
|
|
# Scheduler state tracking
|
|
self.last_successful_update = None
|
|
self.retry_mode = False
|
|
self.next_hourly_check = None
|
|
|
|
# Rate limiting and request tracking
|
|
self.rate_limiter = RateLimiter(max_requests=10, time_window_seconds=60)
|
|
self.request_tracker = RequestTracker()
|
|
|
|
# HTTP session for API requests
|
|
self.session = requests.Session()
|
|
self.session.headers.update(
|
|
{
|
|
"User-Agent": Config.USER_AGENT,
|
|
"Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
|
|
"Accept": "application/json, text/javascript, */*; q=0.01",
|
|
"X-Requested-With": "XMLHttpRequest",
|
|
}
|
|
)
|
|
|
|
# Station mapping is persisted to a JSON file so that station CRUD via the
|
|
# API survives restarts; on first run it is seeded from the bundled
|
|
# defaults in data/stations.json.
|
|
self.station_config_path = Config.STATION_CONFIG_PATH
|
|
self.station_mapping = self._load_station_mapping()
|
|
|
|
self.init_database()
|
|
|
|
@staticmethod
|
|
def _default_station_mapping_path() -> str:
|
|
"""Path to the bundled default station mapping shipped with the package."""
|
|
return os.path.join(
|
|
os.path.dirname(os.path.abspath(__file__)), "data", "stations.json"
|
|
)
|
|
|
|
def _load_station_mapping(self) -> Dict:
|
|
"""Load the station mapping, preferring the runtime-writable config file.
|
|
|
|
Order of precedence:
|
|
1. The runtime config file (STATION_CONFIG_PATH) if it exists — this holds
|
|
any changes made through the station CRUD API.
|
|
2. The bundled defaults in data/stations.json.
|
|
"""
|
|
for source in (self.station_config_path, self._default_station_mapping_path()):
|
|
if source and os.path.exists(source):
|
|
try:
|
|
with open(source, encoding="utf-8") as f:
|
|
mapping = json.load(f)
|
|
logger.info(f"Loaded {len(mapping)} stations from {source}")
|
|
return mapping
|
|
except Exception as e:
|
|
logger.error(f"Failed to load station mapping from {source}: {e}")
|
|
|
|
logger.error(
|
|
"No station mapping could be loaded; starting with an empty mapping"
|
|
)
|
|
return {}
|
|
|
|
def save_stations(self) -> bool:
|
|
"""Persist the current station mapping to the runtime config file.
|
|
|
|
Written atomically (temp file + replace) so a crash mid-write cannot
|
|
corrupt the existing configuration.
|
|
"""
|
|
path = self.station_config_path
|
|
if not path:
|
|
logger.warning(
|
|
"STATION_CONFIG_PATH not set; station changes will not persist"
|
|
)
|
|
return False
|
|
try:
|
|
tmp_path = f"{path}.tmp"
|
|
with open(tmp_path, "w", encoding="utf-8") as f:
|
|
json.dump(self.station_mapping, f, ensure_ascii=False, indent=2)
|
|
f.write("\n")
|
|
os.replace(tmp_path, path)
|
|
logger.info(f"Persisted {len(self.station_mapping)} stations to {path}")
|
|
return True
|
|
except Exception as e:
|
|
logger.error(f"Failed to persist station mapping to {path}: {e}")
|
|
return False
|
|
|
|
def init_database(self):
|
|
"""Initialize database connection"""
|
|
try:
|
|
# Extract db_type and pass remaining config as kwargs
|
|
db_config_copy = self.db_config.copy()
|
|
db_type = db_config_copy.pop("type")
|
|
self.db_adapter = create_database_adapter(db_type, **db_config_copy)
|
|
success = self.db_adapter.connect()
|
|
|
|
if success:
|
|
logger.info(f"Successfully connected to {db_type.upper()} database")
|
|
set_gauge("database_connected", 1)
|
|
increment_counter("database_connections_successful")
|
|
else:
|
|
logger.error(f"Failed to connect to {db_type.upper()} database")
|
|
set_gauge("database_connected", 0)
|
|
increment_counter("database_connections_failed")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error initializing database: {e}")
|
|
set_gauge("database_connected", 0)
|
|
increment_counter("database_connections_failed")
|
|
self.db_adapter = None
|
|
|
|
def fetch_water_data_for_date(
|
|
self, target_date: datetime.datetime
|
|
) -> Optional[List[Dict]]:
|
|
"""Fetch water levels and discharge data from API for a specific date"""
|
|
with Timer("api_request_duration"):
|
|
try:
|
|
logger.info(
|
|
f"Starting data fetch from API for date: {target_date.strftime('%Y-%m-%d')}"
|
|
)
|
|
|
|
# Rate limiting
|
|
self.rate_limiter.wait_if_needed()
|
|
|
|
# Create Thai format date (Buddhist calendar)
|
|
thai_year = target_date.year + 543
|
|
thai_date = f"{target_date.day:02d}/{target_date.month:02d}/{thai_year}"
|
|
|
|
# API parameters
|
|
payload = {
|
|
"DW[UtokID]": "1",
|
|
"DW[BasinID]": "6",
|
|
"DW[TimeCurrent]": thai_date,
|
|
"_search": "false",
|
|
"nd": str(int(time.time() * 1000)),
|
|
"rows": "100",
|
|
"page": "1",
|
|
"sidx": "indexhourly",
|
|
"sord": "asc",
|
|
}
|
|
|
|
logger.debug(f"API parameters: {payload}")
|
|
|
|
# POST request to API
|
|
start_time = time.time()
|
|
response = self.session.post(self.api_url, data=payload, timeout=30)
|
|
response_time = time.time() - start_time
|
|
|
|
response.raise_for_status()
|
|
|
|
# Record successful request
|
|
self.request_tracker.record_request(True, response_time)
|
|
increment_counter("api_requests_successful")
|
|
record_histogram("api_response_time", response_time)
|
|
|
|
# Parse JSON response
|
|
try:
|
|
json_data = response.json()
|
|
logger.debug(
|
|
f"API response received: {len(str(json_data))} characters"
|
|
)
|
|
except ValueError as e:
|
|
logger.error(f"Error parsing JSON response: {e}")
|
|
self.request_tracker.record_request(
|
|
False, response_time, "json_parse_error"
|
|
)
|
|
increment_counter("api_requests_failed")
|
|
return None
|
|
|
|
water_data = []
|
|
|
|
# Parse JSON data
|
|
if json_data and isinstance(json_data, dict) and "rows" in json_data:
|
|
for row in json_data["rows"]:
|
|
try:
|
|
# Parse timestamp
|
|
time_str = row.get("hourlytime", "")
|
|
if not time_str:
|
|
continue
|
|
|
|
try:
|
|
# Format: "1.00", "2.00", ..., "24.00"
|
|
api_hour = int(float(time_str))
|
|
if api_hour < 1 or api_hour > 24:
|
|
continue
|
|
|
|
if api_hour == 24:
|
|
# Hour 24 = midnight (00:00) of the next day
|
|
data_time = target_date.replace(
|
|
hour=0, minute=0, second=0, microsecond=0
|
|
)
|
|
data_time = data_time + datetime.timedelta(days=1)
|
|
else:
|
|
# Hours 1-23 = 01:00-23:00 of the same day
|
|
data_time = target_date.replace(
|
|
hour=api_hour, minute=0, second=0, microsecond=0
|
|
)
|
|
|
|
except (ValueError, IndexError):
|
|
logger.warning(f"Could not parse timestamp: {time_str}")
|
|
continue
|
|
|
|
# Parse all water levels and discharge values
|
|
station_count = 0
|
|
for station_num in range(1, 17): # Stations 1-16
|
|
wl_key = f"wlvalues{station_num}"
|
|
q_key = f"qvalues{station_num}"
|
|
qp_key = f"QPercent{station_num}"
|
|
|
|
# Check if water level data exists (required)
|
|
if wl_key in row:
|
|
try:
|
|
water_level = row[wl_key]
|
|
|
|
# Skip if water level is None or invalid
|
|
if water_level is None:
|
|
continue
|
|
|
|
# Convert water level to float (required)
|
|
water_level = float(water_level)
|
|
|
|
# Try to parse discharge data (optional)
|
|
discharge = None
|
|
discharge_percent = None
|
|
|
|
if q_key in row:
|
|
try:
|
|
discharge_raw = row[q_key]
|
|
if (
|
|
discharge_raw is not None
|
|
and discharge_raw != "***"
|
|
):
|
|
discharge = float(discharge_raw)
|
|
|
|
# Only parse discharge percent if discharge is valid
|
|
discharge_percent_raw = row.get(
|
|
qp_key
|
|
)
|
|
if (
|
|
discharge_percent_raw
|
|
is not None
|
|
):
|
|
try:
|
|
discharge_percent = float(
|
|
discharge_percent_raw
|
|
)
|
|
except (ValueError, TypeError):
|
|
discharge_percent = None
|
|
else:
|
|
logger.debug(
|
|
"Skipping malformed discharge data for "
|
|
f"station {station_num}: {discharge_raw}"
|
|
)
|
|
except (ValueError, TypeError) as e:
|
|
logger.debug(
|
|
f"Could not parse discharge for station {station_num}: {e}"
|
|
)
|
|
|
|
station_info = self.station_mapping.get(
|
|
str(station_num),
|
|
{
|
|
"code": f"P.{19+station_num}",
|
|
"thai_name": f"Station {station_num}",
|
|
"english_name": f"Station {station_num}",
|
|
},
|
|
)
|
|
|
|
water_data.append(
|
|
{
|
|
"timestamp": data_time,
|
|
"station_id": station_num,
|
|
"station_code": station_info["code"],
|
|
"station_name_en": station_info[
|
|
"english_name"
|
|
],
|
|
"station_name_th": station_info[
|
|
"thai_name"
|
|
],
|
|
"latitude": station_info.get(
|
|
"latitude"
|
|
),
|
|
"longitude": station_info.get(
|
|
"longitude"
|
|
),
|
|
"geohash": station_info.get("geohash"),
|
|
"water_level": water_level,
|
|
"water_level_unit": "m",
|
|
"discharge": discharge,
|
|
"discharge_unit": "cms",
|
|
"discharge_percent": discharge_percent,
|
|
"status": "active",
|
|
}
|
|
)
|
|
|
|
station_count += 1
|
|
|
|
except (ValueError, TypeError) as e:
|
|
logger.warning(
|
|
f"Could not parse water level for station {station_num}: {e}"
|
|
)
|
|
continue
|
|
|
|
logger.debug(
|
|
f"Processed {station_count} stations for time {time_str}"
|
|
)
|
|
|
|
except Exception as e:
|
|
logger.warning(f"Error processing data row: {e}")
|
|
continue
|
|
|
|
# Validate data
|
|
water_data = DataValidator.validate_measurements(water_data)
|
|
|
|
logger.info(
|
|
f"Successfully fetched {len(water_data)} data points from API "
|
|
f"for {target_date.strftime('%Y-%m-%d')}"
|
|
)
|
|
return water_data
|
|
|
|
except requests.RequestException as e:
|
|
logger.error(f"Network error fetching API data: {e}")
|
|
self.request_tracker.record_request(False, 0, "network_error")
|
|
increment_counter("api_requests_failed")
|
|
return None
|
|
except Exception as e:
|
|
logger.error(f"Unexpected error fetching API data: {e}")
|
|
self.request_tracker.record_request(False, 0, "unexpected_error")
|
|
increment_counter("api_requests_failed")
|
|
return None
|
|
|
|
def fetch_water_data(self) -> Optional[List[Dict]]:
|
|
"""Fetch water levels and discharge data from API with smart date selection"""
|
|
current_time = datetime.datetime.now()
|
|
|
|
# If it's past 01:00, try today's data first, then yesterday as fallback
|
|
if current_time.hour >= 1:
|
|
logger.info(
|
|
"After 01:00 - trying today's data first, will fallback to yesterday if needed"
|
|
)
|
|
|
|
# Try today's data first
|
|
today_data = self.fetch_water_data_for_date(current_time)
|
|
if today_data and len(today_data) > 0:
|
|
logger.info(
|
|
f"Successfully fetched {len(today_data)} data points for today"
|
|
)
|
|
return today_data
|
|
|
|
# Fallback to yesterday's data
|
|
logger.info("No data available for today, trying yesterday's data")
|
|
yesterday = current_time - datetime.timedelta(days=1)
|
|
yesterday_data = self.fetch_water_data_for_date(yesterday)
|
|
if yesterday_data and len(yesterday_data) > 0:
|
|
logger.info(
|
|
f"Successfully fetched {len(yesterday_data)} data points for yesterday"
|
|
)
|
|
return yesterday_data
|
|
|
|
logger.warning("No data available for today or yesterday")
|
|
return None
|
|
else:
|
|
# Before 01:00 - only try yesterday's data (API likely hasn't updated yet)
|
|
logger.info("Before 01:00 - fetching yesterday's data only")
|
|
yesterday = current_time - datetime.timedelta(days=1)
|
|
return self.fetch_water_data_for_date(yesterday)
|
|
|
|
def save_to_database(self, water_data: List[Dict], max_retries: int = 3) -> bool:
|
|
"""Save water measurements to database with retry logic"""
|
|
if not self.db_adapter:
|
|
logger.error("Database adapter not initialized")
|
|
return False
|
|
|
|
if not water_data:
|
|
logger.warning("No data to save")
|
|
return False
|
|
|
|
for attempt in range(max_retries):
|
|
try:
|
|
success = self.db_adapter.save_measurements(water_data)
|
|
if success:
|
|
logger.info(
|
|
f"Successfully saved {len(water_data)} measurements to database"
|
|
)
|
|
increment_counter("database_saves_successful")
|
|
set_gauge("last_save_timestamp", time.time())
|
|
return True
|
|
else:
|
|
logger.warning(f"Save attempt {attempt + 1} failed, retrying...")
|
|
|
|
except Exception as e:
|
|
if "database is locked" in str(e).lower() and attempt < max_retries - 1:
|
|
logger.warning(
|
|
f"Database locked on attempt {attempt + 1}, retrying in {2 ** attempt} seconds..."
|
|
)
|
|
time.sleep(2**attempt) # Exponential backoff
|
|
continue
|
|
else:
|
|
logger.error(
|
|
f"Error saving to database (attempt {attempt + 1}): {e}"
|
|
)
|
|
if attempt == max_retries - 1:
|
|
increment_counter("database_saves_failed")
|
|
return False
|
|
|
|
return False
|
|
|
|
def get_latest_data(self, limit: int = 100) -> List[Dict]:
|
|
"""Get latest data from database"""
|
|
if not self.db_adapter:
|
|
return []
|
|
|
|
try:
|
|
return self.db_adapter.get_latest_measurements(limit=limit)
|
|
except Exception as e:
|
|
logger.error(f"Error getting latest data: {e}")
|
|
return []
|
|
|
|
def _check_data_freshness(self, water_data: List[Dict]) -> bool:
|
|
"""Check if the fetched data contains new data for the current hour"""
|
|
if not water_data:
|
|
return False
|
|
|
|
current_time = datetime.datetime.now()
|
|
current_hour = current_time.hour
|
|
|
|
# Find the most recent timestamp in the data
|
|
latest_timestamp = None
|
|
for data_point in water_data:
|
|
timestamp = data_point.get("timestamp")
|
|
if timestamp and (latest_timestamp is None or timestamp > latest_timestamp):
|
|
latest_timestamp = timestamp
|
|
|
|
if latest_timestamp is None:
|
|
logger.warning("No valid timestamps found in data")
|
|
return False
|
|
|
|
latest_hour = latest_timestamp.hour
|
|
time_diff = current_time - latest_timestamp
|
|
minutes_old = time_diff.total_seconds() / 60
|
|
|
|
logger.info(
|
|
f"Current time: {current_time.strftime('%H:%M')}, Latest data: {latest_timestamp.strftime('%H:%M')}"
|
|
)
|
|
logger.info(
|
|
f"Current hour: {current_hour}, Latest data hour: {latest_hour}, Age: {minutes_old:.1f} minutes"
|
|
)
|
|
|
|
# Strict check: we need data from the current hour
|
|
# If it's 20:xx and we only have data up to 19:xx, that's stale - go to retry mode
|
|
has_current_hour_data = latest_hour >= current_hour
|
|
|
|
if not has_current_hour_data:
|
|
logger.warning(
|
|
f"No new data available - expected hour {current_hour}, got {latest_hour}"
|
|
)
|
|
logger.warning("Switching to retry mode until new data becomes available")
|
|
return False
|
|
else:
|
|
logger.info(f"Fresh data available for current hour {current_hour}")
|
|
return True
|
|
|
|
def run_scraping_cycle(self) -> bool:
|
|
"""Run a complete scraping cycle with freshness check"""
|
|
logger.info("Starting scraping cycle...")
|
|
|
|
try:
|
|
# Fetch current data
|
|
water_data = self.fetch_water_data()
|
|
if water_data:
|
|
# Check if data is fresh/recent
|
|
is_fresh = self._check_data_freshness(water_data)
|
|
|
|
if is_fresh:
|
|
success = self.save_to_database(water_data)
|
|
if success:
|
|
logger.info(
|
|
"Scraping cycle completed successfully with fresh data"
|
|
)
|
|
increment_counter("scraping_cycles_successful")
|
|
return True
|
|
else:
|
|
logger.error("Failed to save data")
|
|
increment_counter("scraping_cycles_failed")
|
|
return False
|
|
else:
|
|
# Data exists but is stale
|
|
logger.warning(
|
|
"Data fetched but is stale - treating as no fresh data available"
|
|
)
|
|
increment_counter("scraping_cycles_failed")
|
|
return False
|
|
else:
|
|
logger.warning("No data fetched")
|
|
increment_counter("scraping_cycles_failed")
|
|
return False
|
|
|
|
except Exception as e:
|
|
logger.error(f"Scraping cycle failed: {e}")
|
|
increment_counter("scraping_cycles_failed")
|
|
return False
|
|
|
|
def fill_data_gaps(self, days_back: Optional[int] = None) -> int:
|
|
"""Fill gaps in stored data by re-fetching incomplete days.
|
|
|
|
Args:
|
|
days_back: How many days back to scan. None scans the whole data
|
|
range, from the earliest measurement in the database to now.
|
|
|
|
Detection is hour-granular where the backend supports it: any calendar
|
|
day missing one or more hourly slots gets its source date(s) re-fetched.
|
|
Because the API reports hour 24 as midnight of the next day, a missing
|
|
00:00 slot on day D is repaired by re-fetching day D-1.
|
|
"""
|
|
logger = get_logger(__name__)
|
|
filled_count = 0
|
|
|
|
try:
|
|
now = datetime.datetime.now()
|
|
end_date = now.date()
|
|
|
|
if days_back is not None:
|
|
start_date = end_date - datetime.timedelta(days=days_back)
|
|
else:
|
|
if not self.db_adapter:
|
|
logger.error("Database adapter not initialized")
|
|
return 0
|
|
date_range = self.db_adapter.get_measurement_date_range()
|
|
if not date_range:
|
|
logger.error(
|
|
"Database is empty or does not support range queries; "
|
|
"use --import-historical to seed data first"
|
|
)
|
|
return 0
|
|
start_date = date_range[0].date()
|
|
|
|
logger.info(f"Checking for gaps from {start_date} to {end_date}")
|
|
|
|
fetch_dates = self._find_gap_fetch_dates(start_date, end_date, now)
|
|
|
|
if not fetch_dates:
|
|
logger.info("No gaps found")
|
|
return 0
|
|
|
|
logger.info(f"Found {len(fetch_dates)} day(s) needing a re-fetch")
|
|
|
|
for fetch_date in sorted(fetch_dates):
|
|
fetch_dt = datetime.datetime.combine(fetch_date, datetime.time.min)
|
|
logger.info(f"Filling gap for date: {fetch_date}")
|
|
|
|
data = self.fetch_water_data_for_date(fetch_dt)
|
|
|
|
if data:
|
|
if self.save_to_database(data):
|
|
filled_count += len(data)
|
|
logger.info(
|
|
f"Filled {len(data)} measurements for {fetch_date}"
|
|
)
|
|
else:
|
|
logger.warning(f"Failed to save data for {fetch_date}")
|
|
else:
|
|
logger.warning(f"No data available for {fetch_date}")
|
|
|
|
# Be respectful to the API
|
|
time.sleep(1)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Gap filling error: {e}")
|
|
|
|
return filled_count
|
|
|
|
def _find_gap_fetch_dates(
|
|
self,
|
|
start_date: datetime.date,
|
|
end_date: datetime.date,
|
|
now: datetime.datetime,
|
|
) -> List[datetime.date]:
|
|
"""Determine which source dates need re-fetching to fill gaps.
|
|
|
|
Uses hour-granular detection when the adapter supports it, otherwise
|
|
falls back to re-fetching only days with no data at all.
|
|
"""
|
|
hours_by_day = None
|
|
if self.db_adapter:
|
|
hours_by_day = self.db_adapter.get_recorded_hours_by_day(
|
|
start_date, end_date
|
|
)
|
|
|
|
fetch_dates = set()
|
|
|
|
if hours_by_day is None:
|
|
logger.info(
|
|
"Backend does not support hour-granular gap detection; "
|
|
"checking for fully-missing days only"
|
|
)
|
|
current = start_date
|
|
while current <= end_date:
|
|
current_dt = datetime.datetime.combine(current, datetime.time.min)
|
|
if not self._check_data_exists_for_date(current_dt):
|
|
fetch_dates.add(current)
|
|
current += datetime.timedelta(days=1)
|
|
return sorted(fetch_dates)
|
|
|
|
current = start_date
|
|
while current <= end_date:
|
|
if current == now.date():
|
|
# Today: only expect hours that have already passed
|
|
expected_hours = set(range(0, now.hour))
|
|
else:
|
|
expected_hours = set(range(24))
|
|
|
|
missing = expected_hours - hours_by_day.get(current, set())
|
|
|
|
if missing:
|
|
# Hours 1-23 of day D come from fetching D; hour 0 comes from
|
|
# the previous day's fetch (the API's "hour 24")
|
|
if any(h >= 1 for h in missing):
|
|
fetch_dates.add(current)
|
|
if 0 in missing and current > start_date:
|
|
fetch_dates.add(current - datetime.timedelta(days=1))
|
|
|
|
current += datetime.timedelta(days=1)
|
|
|
|
return sorted(fetch_dates)
|
|
|
|
def update_existing_data(self, days_back: int) -> int:
|
|
"""Update existing data with latest values for the specified number of days back"""
|
|
logger = get_logger(__name__)
|
|
updated_count = 0
|
|
|
|
try:
|
|
# Calculate date range
|
|
end_date = datetime.datetime.now()
|
|
start_date = end_date - datetime.timedelta(days=days_back)
|
|
|
|
logger.info(f"Updating data from {start_date.date()} to {end_date.date()}")
|
|
|
|
# Iterate through each date in the range
|
|
current_date = start_date
|
|
while current_date <= end_date:
|
|
logger.info(f"Updating data for date: {current_date.date()}")
|
|
|
|
# Fetch fresh data for this date
|
|
data = self.fetch_water_data_for_date(current_date)
|
|
|
|
if data:
|
|
# Save the data (this will update existing records)
|
|
if self.save_to_database(data):
|
|
updated_count += len(data)
|
|
logger.info(
|
|
f"Updated {len(data)} measurements for {current_date.date()}"
|
|
)
|
|
else:
|
|
logger.warning(
|
|
f"Failed to update data for {current_date.date()}"
|
|
)
|
|
else:
|
|
logger.warning(f"No data available for {current_date.date()}")
|
|
|
|
current_date += datetime.timedelta(days=1)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Data update error: {e}")
|
|
|
|
return updated_count
|
|
|
|
def _check_data_exists_for_date(self, target_date: datetime.datetime) -> bool:
|
|
"""Check if data exists for a specific date"""
|
|
try:
|
|
if not self.db_adapter:
|
|
return False
|
|
|
|
# Get data for the specific date
|
|
measurements = self.db_adapter.get_measurements_for_date(target_date)
|
|
return len(measurements) > 0
|
|
|
|
except Exception as e:
|
|
logger = get_logger(__name__)
|
|
logger.debug(f"Error checking data existence: {e}")
|
|
return False
|
|
|
|
def import_historical_data(
|
|
self,
|
|
start_date: datetime.datetime,
|
|
end_date: datetime.datetime,
|
|
skip_existing: bool = True,
|
|
) -> int:
|
|
"""
|
|
Import historical data for a date range
|
|
|
|
Args:
|
|
start_date: Start date for historical import
|
|
end_date: End date for historical import
|
|
skip_existing: Skip dates that already have data (default: True)
|
|
|
|
Returns:
|
|
Number of data points imported
|
|
"""
|
|
logger.info(
|
|
f"Starting historical data import from {start_date.date()} to {end_date.date()}"
|
|
)
|
|
|
|
total_imported = 0
|
|
current_date = start_date
|
|
|
|
while current_date <= end_date:
|
|
try:
|
|
# Check if data already exists for this date
|
|
if skip_existing and self._check_data_exists_for_date(current_date):
|
|
logger.info(
|
|
f"Data already exists for {current_date.date()}, skipping..."
|
|
)
|
|
current_date += datetime.timedelta(days=1)
|
|
continue
|
|
|
|
logger.info(f"Importing data for {current_date.date()}...")
|
|
|
|
# Fetch data for this date
|
|
data = self.fetch_water_data_for_date(current_date)
|
|
|
|
if data:
|
|
# Save to database
|
|
if self.save_to_database(data):
|
|
total_imported += len(data)
|
|
logger.info(
|
|
f"Successfully imported {len(data)} data points for {current_date.date()}"
|
|
)
|
|
else:
|
|
logger.warning(f"Failed to save data for {current_date.date()}")
|
|
else:
|
|
logger.warning(f"No data available for {current_date.date()}")
|
|
|
|
# Add small delay to be respectful to the API
|
|
time.sleep(1)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error importing data for {current_date.date()}: {e}")
|
|
|
|
current_date += datetime.timedelta(days=1)
|
|
|
|
logger.info(
|
|
f"Historical import completed. Total data points imported: {total_imported}"
|
|
)
|
|
return total_imported
|
|
|
|
|
|
# Main execution for standalone usage
|
|
if __name__ == "__main__":
|
|
import argparse
|
|
import sys
|
|
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s - %(levelname)s - %(message)s",
|
|
handlers=[logging.FileHandler("water_monitor.log"), logging.StreamHandler()],
|
|
)
|
|
|
|
parser = argparse.ArgumentParser(description="Thailand Water Monitor")
|
|
parser.add_argument("--test", action="store_true", help="Run single test cycle")
|
|
args = parser.parse_args()
|
|
|
|
# Default SQLite configuration
|
|
db_config = {"type": "sqlite", "connection_string": "sqlite:///water_levels.db"}
|
|
|
|
try:
|
|
scraper = EnhancedWaterMonitorScraper(db_config)
|
|
|
|
if args.test:
|
|
logger.info("Running test cycle...")
|
|
result = scraper.run_scraping_cycle()
|
|
if result:
|
|
logger.info("✅ Test completed successfully")
|
|
sys.exit(0)
|
|
else:
|
|
logger.error("❌ Test failed")
|
|
sys.exit(1)
|
|
else:
|
|
logger.info("Starting continuous monitoring...")
|
|
schedule.every(1).hours.do(scraper.run_scraping_cycle)
|
|
|
|
# Run initial cycle
|
|
scraper.run_scraping_cycle()
|
|
|
|
while True:
|
|
schedule.run_pending()
|
|
time.sleep(60)
|
|
|
|
except KeyboardInterrupt:
|
|
logger.info("Monitoring stopped by user")
|
|
except Exception as e:
|
|
logger.error(f"Error: {e}")
|
|
sys.exit(1)
|