From 684609152297e13282ca0436f6e620d81ff76174 Mon Sep 17 00:00:00 2001 From: grabowski Date: Sun, 28 Sep 2025 18:30:53 +0700 Subject: [PATCH] Implement smart date selection for data fetching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add intelligent date selection based on current time - Before 01:00: fetch yesterday's data only (API not updated yet) - After 01:00: try today's data first, fallback to yesterday if needed - Improve data availability by adapting to API update patterns - Add comprehensive logging for date selection decisions This ensures optimal data fetching regardless of the time of day: - Early morning (00:00-00:59): fetches yesterday (reliable) - Rest of day (01:00-23:59): tries today first, falls back to yesterday 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- src/water_scraper_v3.py | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/src/water_scraper_v3.py b/src/water_scraper_v3.py index 2e8652a..6ed3ba9 100644 --- a/src/water_scraper_v3.py +++ b/src/water_scraper_v3.py @@ -407,9 +407,34 @@ class EnhancedWaterMonitorScraper: return None def fetch_water_data(self) -> Optional[List[Dict]]: - """Fetch water levels and discharge data from API for yesterday (API has 1-day delay)""" - yesterday = datetime.datetime.now() - datetime.timedelta(days=1) - return self.fetch_water_data_for_date(yesterday) + """Fetch water levels and discharge data from API with smart date selection""" + current_time = datetime.datetime.now() + + # If it's past 01:00, try today's data first, then yesterday as fallback + if current_time.hour >= 1: + logger.info("After 01:00 - trying today's data first, will fallback to yesterday if needed") + + # Try today's data first + today_data = self.fetch_water_data_for_date(current_time) + if today_data and len(today_data) > 0: + logger.info(f"Successfully fetched {len(today_data)} data points for today") + return today_data + + # Fallback to yesterday's data + logger.info("No data available for today, trying yesterday's data") + yesterday = current_time - datetime.timedelta(days=1) + yesterday_data = self.fetch_water_data_for_date(yesterday) + if yesterday_data and len(yesterday_data) > 0: + logger.info(f"Successfully fetched {len(yesterday_data)} data points for yesterday") + return yesterday_data + + logger.warning("No data available for today or yesterday") + return None + else: + # Before 01:00 - only try yesterday's data (API likely hasn't updated yet) + logger.info("Before 01:00 - fetching yesterday's data only") + yesterday = current_time - datetime.timedelta(days=1) + return self.fetch_water_data_for_date(yesterday) def save_to_database(self, water_data: List[Dict], max_retries: int = 3) -> bool: """Save water measurements to database with retry logic"""