Fix malformed discharge data handling to preserve water level data
- Change data parsing logic to make discharge data optional - Water level data is now saved even when discharge values are malformed (e.g., "***") - Handle malformed discharge values gracefully with null instead of skipping entire record - Add specific handling for "***" discharge values from API - Improve data completeness by not discarding valid water level measurements Before: Entire station record was skipped if discharge was malformed After: Water level data is preserved, discharge set to null for malformed values Example fix: - wlvalues8: 1.6 (valid) + qvalues8: "***" (malformed) - Before: No record saved - After: Record saved with water_level=1.6, discharge=null 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -338,21 +338,39 @@ class EnhancedWaterMonitorScraper:
|
|||||||
q_key = f'qvalues{station_num}'
|
q_key = f'qvalues{station_num}'
|
||||||
qp_key = f'QPercent{station_num}'
|
qp_key = f'QPercent{station_num}'
|
||||||
|
|
||||||
# Check if both water level and discharge data exist
|
# Check if water level data exists (required)
|
||||||
if wl_key in row and q_key in row:
|
if wl_key in row:
|
||||||
try:
|
try:
|
||||||
water_level = row[wl_key]
|
water_level = row[wl_key]
|
||||||
discharge = row[q_key]
|
|
||||||
discharge_percent = row.get(qp_key)
|
|
||||||
|
|
||||||
# Skip if values are None or invalid
|
# Skip if water level is None or invalid
|
||||||
if water_level is None or discharge is None:
|
if water_level is None:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Convert to float
|
# Convert water level to float (required)
|
||||||
water_level = float(water_level)
|
water_level = float(water_level)
|
||||||
discharge = float(discharge)
|
|
||||||
discharge_percent = float(discharge_percent) if discharge_percent is not None else None
|
# Try to parse discharge data (optional)
|
||||||
|
discharge = None
|
||||||
|
discharge_percent = None
|
||||||
|
|
||||||
|
if q_key in row:
|
||||||
|
try:
|
||||||
|
discharge_raw = row[q_key]
|
||||||
|
if discharge_raw is not None and discharge_raw != "***":
|
||||||
|
discharge = float(discharge_raw)
|
||||||
|
|
||||||
|
# Only parse discharge percent if discharge is valid
|
||||||
|
discharge_percent_raw = row.get(qp_key)
|
||||||
|
if discharge_percent_raw is not None:
|
||||||
|
try:
|
||||||
|
discharge_percent = float(discharge_percent_raw)
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
discharge_percent = None
|
||||||
|
else:
|
||||||
|
logger.debug(f"Skipping malformed discharge data for station {station_num}: {discharge_raw}")
|
||||||
|
except (ValueError, TypeError) as e:
|
||||||
|
logger.debug(f"Could not parse discharge for station {station_num}: {e}")
|
||||||
|
|
||||||
station_info = self.station_mapping.get(str(station_num), {
|
station_info = self.station_mapping.get(str(station_num), {
|
||||||
'code': f'P.{19+station_num}',
|
'code': f'P.{19+station_num}',
|
||||||
@@ -380,7 +398,7 @@ class EnhancedWaterMonitorScraper:
|
|||||||
station_count += 1
|
station_count += 1
|
||||||
|
|
||||||
except (ValueError, TypeError) as e:
|
except (ValueError, TypeError) as e:
|
||||||
logger.warning(f"Could not parse data for station {station_num}: {e}")
|
logger.warning(f"Could not parse water level for station {station_num}: {e}")
|
||||||
continue
|
continue
|
||||||
|
|
||||||
logger.debug(f"Processed {station_count} stations for time {time_str}")
|
logger.debug(f"Processed {station_count} stations for time {time_str}")
|
||||||
|
Reference in New Issue
Block a user