Implement smart date selection for data fetching

- Add intelligent date selection based on current time
- Before 01:00: fetch yesterday's data only (API not updated yet)
- After 01:00: try today's data first, fallback to yesterday if needed
- Improve data availability by adapting to API update patterns
- Add comprehensive logging for date selection decisions

This ensures optimal data fetching regardless of the time of day:
- Early morning (00:00-00:59): fetches yesterday (reliable)
- Rest of day (01:00-23:59): tries today first, falls back to yesterday

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
2025-09-28 18:30:53 +07:00
parent 4cc792157f
commit 6846091522

View File

@@ -407,9 +407,34 @@ class EnhancedWaterMonitorScraper:
return None
def fetch_water_data(self) -> Optional[List[Dict]]:
"""Fetch water levels and discharge data from API for yesterday (API has 1-day delay)"""
yesterday = datetime.datetime.now() - datetime.timedelta(days=1)
return self.fetch_water_data_for_date(yesterday)
"""Fetch water levels and discharge data from API with smart date selection"""
current_time = datetime.datetime.now()
# If it's past 01:00, try today's data first, then yesterday as fallback
if current_time.hour >= 1:
logger.info("After 01:00 - trying today's data first, will fallback to yesterday if needed")
# Try today's data first
today_data = self.fetch_water_data_for_date(current_time)
if today_data and len(today_data) > 0:
logger.info(f"Successfully fetched {len(today_data)} data points for today")
return today_data
# Fallback to yesterday's data
logger.info("No data available for today, trying yesterday's data")
yesterday = current_time - datetime.timedelta(days=1)
yesterday_data = self.fetch_water_data_for_date(yesterday)
if yesterday_data and len(yesterday_data) > 0:
logger.info(f"Successfully fetched {len(yesterday_data)} data points for yesterday")
return yesterday_data
logger.warning("No data available for today or yesterday")
return None
else:
# Before 01:00 - only try yesterday's data (API likely hasn't updated yet)
logger.info("Before 01:00 - fetching yesterday's data only")
yesterday = current_time - datetime.timedelta(days=1)
return self.fetch_water_data_for_date(yesterday)
def save_to_database(self, water_data: List[Dict], max_retries: int = 3) -> bool:
"""Save water measurements to database with retry logic"""