#!/usr/bin/env python3 """ Data validation utilities for water monitoring system """ import logging from datetime import datetime from typing import Any, Dict, List, Optional from .exceptions import DataValidationError from .models import StationInfo, WaterMeasurement logger = logging.getLogger(__name__) class DataValidator: """Validates water measurement data""" # Reasonable ranges for water measurements WATER_LEVEL_MIN = -10.0 # meters WATER_LEVEL_MAX = 50.0 # meters DISCHARGE_MIN = 0.0 # cms DISCHARGE_MAX = 10000.0 # cms DISCHARGE_PERCENT_MIN = 0.0 # % of channel capacity. Major floods genuinely exceed 200% (Oct 2024 peaked # at 226%); values beyond 500% are treated as data errors. DISCHARGE_PERCENT_MAX = 500.0 @classmethod def validate_measurement(cls, measurement: Dict[str, Any]) -> bool: """Validate a single measurement""" try: # Check required fields (discharge is now optional) required_fields = ["timestamp", "station_id", "water_level"] for field in required_fields: if field not in measurement: logger.warning(f"Missing required field: {field}") return False # Validate timestamp if not isinstance(measurement["timestamp"], datetime): logger.warning( f"Invalid timestamp type: {type(measurement['timestamp'])}" ) return False # Validate water level (required) if measurement["water_level"] is None: logger.warning("Water level cannot be None") return False water_level = float(measurement["water_level"]) if not (cls.WATER_LEVEL_MIN <= water_level <= cls.WATER_LEVEL_MAX): logger.warning(f"Water level out of range: {water_level}") return False # Validate discharge (optional - can be None) discharge_value = measurement.get("discharge") if discharge_value is not None: discharge = float(discharge_value) if not (cls.DISCHARGE_MIN <= discharge <= cls.DISCHARGE_MAX): logger.warning(f"Discharge out of range: {discharge}") return False # Validate discharge percent if present. This is an auxiliary field: # an implausible value must not cost us the water level and discharge # (rejecting rows here silently deleted the Oct 2024 flood peaks), so # out-of-range percents are nulled and the measurement is kept. if measurement.get("discharge_percent") is not None: discharge_percent = float(measurement["discharge_percent"]) if not ( cls.DISCHARGE_PERCENT_MIN <= discharge_percent <= cls.DISCHARGE_PERCENT_MAX ): logger.warning( f"Discharge percent out of range ({discharge_percent}); " "keeping measurement with discharge_percent=None" ) measurement["discharge_percent"] = None # Validate station ID station_id = measurement["station_id"] if not isinstance(station_id, int) or station_id < 1 or station_id > 16: logger.warning(f"Invalid station ID: {station_id}") return False return True except (ValueError, TypeError) as e: logger.warning(f"Data validation error: {e}") return False @classmethod def validate_measurements( cls, measurements: List[Dict[str, Any]] ) -> List[Dict[str, Any]]: """Validate and filter a list of measurements""" valid_measurements = [] invalid_count = 0 for measurement in measurements: if cls.validate_measurement(measurement): valid_measurements.append(measurement) else: invalid_count += 1 if invalid_count > 0: logger.warning(f"Filtered out {invalid_count} invalid measurements") return valid_measurements @classmethod def validate_station_info(cls, station_info: Dict[str, Any]) -> bool: """Validate station information""" try: required_fields = [ "station_id", "station_code", "thai_name", "english_name", ] for field in required_fields: if field not in station_info or not station_info[field]: logger.warning(f"Missing or empty station field: {field}") return False # Validate coordinates if present if station_info.get("latitude") is not None: lat = float(station_info["latitude"]) if not (-90 <= lat <= 90): logger.warning(f"Invalid latitude: {lat}") return False if station_info.get("longitude") is not None: lon = float(station_info["longitude"]) if not (-180 <= lon <= 180): logger.warning(f"Invalid longitude: {lon}") return False return True except (ValueError, TypeError) as e: logger.warning(f"Station validation error: {e}") return False