grabowskiandClaude
60e70c2192
- Make discharge field optional in data validator - Remove discharge from required fields list - Add explicit null check for discharge before float conversion - Prevent "float() argument must be a string or a real number, not 'NoneType'" errors - Allow records with valid water levels but malformed/null discharge data This completes the malformed data handling fix by updating the validator to match the parser's new behavior of allowing null discharge values. Before: Validator rejected records with null discharge After: Validator accepts records with null discharge, validates only if present 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
121 lines
4.8 KiB
Python
121 lines
4.8 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Data validation utilities for water monitoring system
|
|
"""
|
|
|
|
from typing import List, Dict, Any, Optional
|
|
from datetime import datetime
|
|
import logging
|
|
from .exceptions import DataValidationError
|
|
from .models import WaterMeasurement, StationInfo
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
class DataValidator:
|
|
"""Validates water measurement data"""
|
|
|
|
# Reasonable ranges for water measurements
|
|
WATER_LEVEL_MIN = -10.0 # meters
|
|
WATER_LEVEL_MAX = 50.0 # meters
|
|
DISCHARGE_MIN = 0.0 # cms
|
|
DISCHARGE_MAX = 10000.0 # cms
|
|
DISCHARGE_PERCENT_MIN = 0.0
|
|
DISCHARGE_PERCENT_MAX = 200.0 # Allow some overflow
|
|
|
|
@classmethod
|
|
def validate_measurement(cls, measurement: Dict[str, Any]) -> bool:
|
|
"""Validate a single measurement"""
|
|
try:
|
|
# Check required fields (discharge is now optional)
|
|
required_fields = ['timestamp', 'station_id', 'water_level']
|
|
for field in required_fields:
|
|
if field not in measurement:
|
|
logger.warning(f"Missing required field: {field}")
|
|
return False
|
|
|
|
# Validate timestamp
|
|
if not isinstance(measurement['timestamp'], datetime):
|
|
logger.warning(f"Invalid timestamp type: {type(measurement['timestamp'])}")
|
|
return False
|
|
|
|
# Validate water level (required)
|
|
if measurement['water_level'] is None:
|
|
logger.warning("Water level cannot be None")
|
|
return False
|
|
water_level = float(measurement['water_level'])
|
|
if not (cls.WATER_LEVEL_MIN <= water_level <= cls.WATER_LEVEL_MAX):
|
|
logger.warning(f"Water level out of range: {water_level}")
|
|
return False
|
|
|
|
# Validate discharge (optional - can be None)
|
|
discharge_value = measurement.get('discharge')
|
|
if discharge_value is not None:
|
|
discharge = float(discharge_value)
|
|
if not (cls.DISCHARGE_MIN <= discharge <= cls.DISCHARGE_MAX):
|
|
logger.warning(f"Discharge out of range: {discharge}")
|
|
return False
|
|
|
|
# Validate discharge percent if present
|
|
if measurement.get('discharge_percent') is not None:
|
|
discharge_percent = float(measurement['discharge_percent'])
|
|
if not (cls.DISCHARGE_PERCENT_MIN <= discharge_percent <= cls.DISCHARGE_PERCENT_MAX):
|
|
logger.warning(f"Discharge percent out of range: {discharge_percent}")
|
|
return False
|
|
|
|
# Validate station ID
|
|
station_id = measurement['station_id']
|
|
if not isinstance(station_id, int) or station_id < 1 or station_id > 16:
|
|
logger.warning(f"Invalid station ID: {station_id}")
|
|
return False
|
|
|
|
return True
|
|
|
|
except (ValueError, TypeError) as e:
|
|
logger.warning(f"Data validation error: {e}")
|
|
return False
|
|
|
|
@classmethod
|
|
def validate_measurements(cls, measurements: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
|
"""Validate and filter a list of measurements"""
|
|
valid_measurements = []
|
|
invalid_count = 0
|
|
|
|
for measurement in measurements:
|
|
if cls.validate_measurement(measurement):
|
|
valid_measurements.append(measurement)
|
|
else:
|
|
invalid_count += 1
|
|
|
|
if invalid_count > 0:
|
|
logger.warning(f"Filtered out {invalid_count} invalid measurements")
|
|
|
|
return valid_measurements
|
|
|
|
@classmethod
|
|
def validate_station_info(cls, station_info: Dict[str, Any]) -> bool:
|
|
"""Validate station information"""
|
|
try:
|
|
required_fields = ['station_id', 'station_code', 'thai_name', 'english_name']
|
|
for field in required_fields:
|
|
if field not in station_info or not station_info[field]:
|
|
logger.warning(f"Missing or empty station field: {field}")
|
|
return False
|
|
|
|
# Validate coordinates if present
|
|
if station_info.get('latitude') is not None:
|
|
lat = float(station_info['latitude'])
|
|
if not (-90 <= lat <= 90):
|
|
logger.warning(f"Invalid latitude: {lat}")
|
|
return False
|
|
|
|
if station_info.get('longitude') is not None:
|
|
lon = float(station_info['longitude'])
|
|
if not (-180 <= lon <= 180):
|
|
logger.warning(f"Invalid longitude: {lon}")
|
|
return False
|
|
|
|
return True
|
|
|
|
except (ValueError, TypeError) as e:
|
|
logger.warning(f"Station validation error: {e}")
|
|
return False |