Add historical data import functionality
- Add import_historical_data() method to EnhancedWaterMonitorScraper - Support date range imports with Buddhist calendar API format - Add CLI arguments --import-historical and --force-overwrite - Include API rate limiting and skip existing data option - Enable importing years of historical water level data 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -576,6 +576,58 @@ class EnhancedWaterMonitorScraper:
|
||||
logger.debug(f"Error checking data existence: {e}")
|
||||
return False
|
||||
|
||||
def import_historical_data(self, start_date: datetime.datetime, end_date: datetime.datetime,
|
||||
skip_existing: bool = True) -> int:
|
||||
"""
|
||||
Import historical data for a date range
|
||||
|
||||
Args:
|
||||
start_date: Start date for historical import
|
||||
end_date: End date for historical import
|
||||
skip_existing: Skip dates that already have data (default: True)
|
||||
|
||||
Returns:
|
||||
Number of data points imported
|
||||
"""
|
||||
logger.info(f"Starting historical data import from {start_date.date()} to {end_date.date()}")
|
||||
|
||||
total_imported = 0
|
||||
current_date = start_date
|
||||
|
||||
while current_date <= end_date:
|
||||
try:
|
||||
# Check if data already exists for this date
|
||||
if skip_existing and self._check_data_exists_for_date(current_date):
|
||||
logger.info(f"Data already exists for {current_date.date()}, skipping...")
|
||||
current_date += datetime.timedelta(days=1)
|
||||
continue
|
||||
|
||||
logger.info(f"Importing data for {current_date.date()}...")
|
||||
|
||||
# Fetch data for this date
|
||||
data = self.fetch_water_data_for_date(current_date)
|
||||
|
||||
if data:
|
||||
# Save to database
|
||||
if self.save_to_database(data):
|
||||
total_imported += len(data)
|
||||
logger.info(f"Successfully imported {len(data)} data points for {current_date.date()}")
|
||||
else:
|
||||
logger.warning(f"Failed to save data for {current_date.date()}")
|
||||
else:
|
||||
logger.warning(f"No data available for {current_date.date()}")
|
||||
|
||||
# Add small delay to be respectful to the API
|
||||
time.sleep(1)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error importing data for {current_date.date()}: {e}")
|
||||
|
||||
current_date += datetime.timedelta(days=1)
|
||||
|
||||
logger.info(f"Historical import completed. Total data points imported: {total_imported}")
|
||||
return total_imported
|
||||
|
||||
# Main execution for standalone usage
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
|
||||
Reference in New Issue
Block a user