Fix malformed discharge data handling to preserve water level data

- Change data parsing logic to make discharge data optional
- Water level data is now saved even when discharge values are malformed (e.g., "***")
- Handle malformed discharge values gracefully with null instead of skipping entire record
- Add specific handling for "***" discharge values from API
- Improve data completeness by not discarding valid water level measurements

Before: Entire station record was skipped if discharge was malformed
After: Water level data is preserved, discharge set to null for malformed values

Example fix:
- wlvalues8: 1.6 (valid) + qvalues8: "***" (malformed)
- Before: No record saved
- After: Record saved with water_level=1.6, discharge=null

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
2025-09-28 18:41:39 +07:00
parent 6846091522
commit cc5c4522b8

View File

@@ -338,21 +338,39 @@ class EnhancedWaterMonitorScraper:
q_key = f'qvalues{station_num}' q_key = f'qvalues{station_num}'
qp_key = f'QPercent{station_num}' qp_key = f'QPercent{station_num}'
# Check if both water level and discharge data exist # Check if water level data exists (required)
if wl_key in row and q_key in row: if wl_key in row:
try: try:
water_level = row[wl_key] water_level = row[wl_key]
discharge = row[q_key]
discharge_percent = row.get(qp_key)
# Skip if values are None or invalid # Skip if water level is None or invalid
if water_level is None or discharge is None: if water_level is None:
continue continue
# Convert to float # Convert water level to float (required)
water_level = float(water_level) water_level = float(water_level)
discharge = float(discharge)
discharge_percent = float(discharge_percent) if discharge_percent is not None else None # Try to parse discharge data (optional)
discharge = None
discharge_percent = None
if q_key in row:
try:
discharge_raw = row[q_key]
if discharge_raw is not None and discharge_raw != "***":
discharge = float(discharge_raw)
# Only parse discharge percent if discharge is valid
discharge_percent_raw = row.get(qp_key)
if discharge_percent_raw is not None:
try:
discharge_percent = float(discharge_percent_raw)
except (ValueError, TypeError):
discharge_percent = None
else:
logger.debug(f"Skipping malformed discharge data for station {station_num}: {discharge_raw}")
except (ValueError, TypeError) as e:
logger.debug(f"Could not parse discharge for station {station_num}: {e}")
station_info = self.station_mapping.get(str(station_num), { station_info = self.station_mapping.get(str(station_num), {
'code': f'P.{19+station_num}', 'code': f'P.{19+station_num}',
@@ -380,7 +398,7 @@ class EnhancedWaterMonitorScraper:
station_count += 1 station_count += 1
except (ValueError, TypeError) as e: except (ValueError, TypeError) as e:
logger.warning(f"Could not parse data for station {station_num}: {e}") logger.warning(f"Could not parse water level for station {station_num}: {e}")
continue continue
logger.debug(f"Processed {station_count} stations for time {time_str}") logger.debug(f"Processed {station_count} stations for time {time_str}")