Add historical data import functionality

- Add import_historical_data() method to EnhancedWaterMonitorScraper
- Support date range imports with Buddhist calendar API format
- Add CLI arguments --import-historical and --force-overwrite
- Include API rate limiting and skip existing data option
- Enable importing years of historical water level data

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
2025-09-28 14:46:54 +07:00
co-authored by Claude
parent bd812ca5ca
commit 0ff58ecb13
2 changed files with 115 additions and 6 deletions
+52
View File
@@ -576,6 +576,58 @@ class EnhancedWaterMonitorScraper:
logger.debug(f"Error checking data existence: {e}")
return False
def import_historical_data(self, start_date: datetime.datetime, end_date: datetime.datetime,
skip_existing: bool = True) -> int:
"""
Import historical data for a date range
Args:
start_date: Start date for historical import
end_date: End date for historical import
skip_existing: Skip dates that already have data (default: True)
Returns:
Number of data points imported
"""
logger.info(f"Starting historical data import from {start_date.date()} to {end_date.date()}")
total_imported = 0
current_date = start_date
while current_date <= end_date:
try:
# Check if data already exists for this date
if skip_existing and self._check_data_exists_for_date(current_date):
logger.info(f"Data already exists for {current_date.date()}, skipping...")
current_date += datetime.timedelta(days=1)
continue
logger.info(f"Importing data for {current_date.date()}...")
# Fetch data for this date
data = self.fetch_water_data_for_date(current_date)
if data:
# Save to database
if self.save_to_database(data):
total_imported += len(data)
logger.info(f"Successfully imported {len(data)} data points for {current_date.date()}")
else:
logger.warning(f"Failed to save data for {current_date.date()}")
else:
logger.warning(f"No data available for {current_date.date()}")
# Add small delay to be respectful to the API
time.sleep(1)
except Exception as e:
logger.error(f"Error importing data for {current_date.date()}: {e}")
current_date += datetime.timedelta(days=1)
logger.info(f"Historical import completed. Total data points imported: {total_imported}")
return total_imported
# Main execution for standalone usage
if __name__ == "__main__":
import argparse