Implement smart date selection for data fetching
- Add intelligent date selection based on current time - Before 01:00: fetch yesterday's data only (API not updated yet) - After 01:00: try today's data first, fallback to yesterday if needed - Improve data availability by adapting to API update patterns - Add comprehensive logging for date selection decisions This ensures optimal data fetching regardless of the time of day: - Early morning (00:00-00:59): fetches yesterday (reliable) - Rest of day (01:00-23:59): tries today first, falls back to yesterday 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -407,9 +407,34 @@ class EnhancedWaterMonitorScraper:
|
||||
return None
|
||||
|
||||
def fetch_water_data(self) -> Optional[List[Dict]]:
|
||||
"""Fetch water levels and discharge data from API for yesterday (API has 1-day delay)"""
|
||||
yesterday = datetime.datetime.now() - datetime.timedelta(days=1)
|
||||
return self.fetch_water_data_for_date(yesterday)
|
||||
"""Fetch water levels and discharge data from API with smart date selection"""
|
||||
current_time = datetime.datetime.now()
|
||||
|
||||
# If it's past 01:00, try today's data first, then yesterday as fallback
|
||||
if current_time.hour >= 1:
|
||||
logger.info("After 01:00 - trying today's data first, will fallback to yesterday if needed")
|
||||
|
||||
# Try today's data first
|
||||
today_data = self.fetch_water_data_for_date(current_time)
|
||||
if today_data and len(today_data) > 0:
|
||||
logger.info(f"Successfully fetched {len(today_data)} data points for today")
|
||||
return today_data
|
||||
|
||||
# Fallback to yesterday's data
|
||||
logger.info("No data available for today, trying yesterday's data")
|
||||
yesterday = current_time - datetime.timedelta(days=1)
|
||||
yesterday_data = self.fetch_water_data_for_date(yesterday)
|
||||
if yesterday_data and len(yesterday_data) > 0:
|
||||
logger.info(f"Successfully fetched {len(yesterday_data)} data points for yesterday")
|
||||
return yesterday_data
|
||||
|
||||
logger.warning("No data available for today or yesterday")
|
||||
return None
|
||||
else:
|
||||
# Before 01:00 - only try yesterday's data (API likely hasn't updated yet)
|
||||
logger.info("Before 01:00 - fetching yesterday's data only")
|
||||
yesterday = current_time - datetime.timedelta(days=1)
|
||||
return self.fetch_water_data_for_date(yesterday)
|
||||
|
||||
def save_to_database(self, water_data: List[Dict], max_retries: int = 3) -> bool:
|
||||
"""Save water measurements to database with retry logic"""
|
||||
|
Reference in New Issue
Block a user